From 5a85291a0d433a7050fa955a26bac27fd766faa1 Mon Sep 17 00:00:00 2001 From: frozenhelium Date: Wed, 12 Nov 2025 18:08:12 +0545 Subject: [PATCH] feat(translation): connect with cacheppuccino --- .github/workflows/ci.yml | 1 + app/env.ts | 3 + app/package.json | 16 +- .../translatte/commands/applyMigrations.ts | 4 +- .../translatte/commands/clearServerStrings.ts | 12 +- .../commands/exportServerStringsToExcel.ts | 4 +- .../translatte/commands/generateMigration.ts | 4 +- .../translatte/commands/lintMigrations.ts | 28 ++ .../commands/listMigrations.test.ts | 12 +- .../translatte/commands/mergeMigrations.ts | 156 +-------- .../translatte/commands/pushMigration.ts | 230 ------------ .../translatte/commands/pushMigrationsToGo.ts | 267 ++++++++++++++ .../commands/pushMigrationsToIfrc.ts | 327 ++++++++++++++++++ .../commands/pushStringsFromExcel.ts | 313 +++++++++++------ .../commands/pushStringsFromExcelToIfrc.ts | 48 +++ app/scripts/translatte/main.ts | 117 ++++++- app/scripts/translatte/utils.ts | 237 ++++++++++++- app/src/config.ts | 2 + app/src/utils/resolveUrl.ts | 20 +- app/src/utils/restRequest/go.ts | 33 +- app/src/utils/restRequest/index.ts | 32 +- app/src/views/RootLayout/index.tsx | 105 +++--- nginx-serve/Dockerfile | 1 + nginx-serve/apply-config.sh | 1 + nginx-serve/helm/templates/configmap.yaml | 1 + nginx-serve/helm/values-test.yaml | 1 + nginx-serve/helm/values.yaml | 1 + pnpm-lock.yaml | 73 +++- 28 files changed, 1414 insertions(+), 635 deletions(-) create mode 100644 app/scripts/translatte/commands/lintMigrations.ts delete mode 100644 app/scripts/translatte/commands/pushMigration.ts create mode 100644 app/scripts/translatte/commands/pushMigrationsToGo.ts create mode 100644 app/scripts/translatte/commands/pushMigrationsToIfrc.ts create mode 100644 app/scripts/translatte/commands/pushStringsFromExcelToIfrc.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ccf89ddfe..f292d3249c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,7 @@ env: APP_RISK_API_ENDPOINT: 'https://go-risk-staging.northeurope.cloudapp.azure.com/api/v1/' APP_TINY_API_KEY: 'dummy-api-key' APP_TITLE: 'IFRC Go Test' + APP_TRANSLATION_API_ENDPOINT: 'https://cacheppuccino-alpha-1.ifrc-go.dev.togglecorp.com/' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/app/env.ts b/app/env.ts index d0e7ea635c..e1897ad90a 100644 --- a/app/env.ts +++ b/app/env.ts @@ -16,6 +16,9 @@ export default defineConfig({ return value as ('production' | 'staging' | 'testing' | `alpha-${number}` | 'development' | 'APP_ENVIRONMENT_PLACEHOLDER'); }, APP_API_ENDPOINT: Schema.string({ format: 'url', protocol: true, tld: false }), + + APP_TRANSLATION_API_ENDPOINT: Schema.string({ format: 'url', protocol: true, tld: false }), + APP_ADMIN_URL: Schema.string.optional({ format: 'url', protocol: true, tld: false }), APP_MAPBOX_ACCESS_TOKEN: Schema.string(), APP_TINY_API_KEY: Schema.string(), diff --git a/app/package.json b/app/package.json index 20aff0b263..e195339311 100644 --- a/app/package.json +++ b/app/package.json @@ -13,12 +13,15 @@ "translatte": "tsx scripts/translatte/main.ts", "translatte:generate": "pnpm translatte generate-migration ../translationMigrations ./src/**/i18n.json ../packages/ui/src/**/i18n.json", "translatte:lint": "pnpm translatte lint ./src/**/i18n.json ../packages/ui/src/**/i18n.json", - "initialize:type": "mkdir -p generated/ && pnpm initialize:type:go-api && pnpm initialize:type:risk-api", + "translatte:lint-migrations": "pnpm translatte lint-migrations ../translationMigrations", + "initialize:type": "mkdir -p generated/ && pnpm initialize:type:go-api && pnpm initialize:type:risk-api && pnpm initialize:type:translations", "initialize:type:go-api": "test -f ./generated/types.ts && true || cp types.stub.ts ./generated/types.ts", "initialize:type:risk-api": "test -f ./generated/riskTypes.ts && true || cp types.stub.ts ./generated/riskTypes.ts", - "generate:type": "pnpm generate:type:go-api && pnpm generate:type:risk-api", + "initialize:type:translations": "test -f ./generated/translationTypes.ts && true || cp types.stub.ts ./generated/translationTypes.ts", + "generate:type": "pnpm generate:type:go-api && pnpm generate:type:risk-api && pnpm generate:type:translations", "generate:type:go-api": "openapi-typescript ../go-api/assets/openapi-schema.yaml -o ./generated/types.ts --alphabetize", - "generate:type:risk-api": "dotenv -- cross-var openapi-typescript ../go-risk-module-api/openapi-schema.yaml -o ./generated/riskTypes.ts --alphabetize", + "generate:type:risk-api": "openapi-typescript ../go-risk-module-api/openapi-schema.yaml -o ./generated/riskTypes.ts --alphabetize", + "generate:type:translations": "dotenv -- cross-var openapi-typescript \"%APP_TRANSLATION_API_ENDPOINT%openapi.json\" -o ./generated/translationTypes.ts --alphabetize", "prestart": "pnpm initialize:type", "start": "pnpm -F @ifrc-go/ui build && vite", "prebuild": "pnpm initialize:type", @@ -29,7 +32,7 @@ "prelint:js": "pnpm initialize:type", "lint:js": "eslint src", "lint:css": "stylelint \"./src/**/*.css\"", - "lint:translation": "pnpm translatte:lint", + "lint:translation": "pnpm translatte:lint && pnpm translatte:lint-migrations", "lint": "pnpm lint:js && pnpm lint:css && pnpm lint:translation", "lint:fix": "pnpm lint:js --fix && pnpm lint:css --fix", "test": "vitest", @@ -48,7 +51,7 @@ "@togglecorp/toggle-request": "^1.0.0-beta.3", "@turf/bbox": "^6.5.0", "@turf/buffer": "^6.5.0", - "exceljs": "^4.3.0", + "exceljs": "^4.4.0", "file-saver": "^2.0.5", "html-to-image": "^1.11.11", "mapbox-gl": "^1.13.0", @@ -57,7 +60,8 @@ "react": "^18.2.0", "react-dom": "^18.2.0", "react-router-dom": "^6.18.0", - "sanitize-html": "^2.10.0" + "sanitize-html": "^2.10.0", + "xlsx": "^0.18.5" }, "devDependencies": { "@eslint/eslintrc": "^3.2.0", diff --git a/app/scripts/translatte/commands/applyMigrations.ts b/app/scripts/translatte/commands/applyMigrations.ts index dbcb8b39e5..43bdc7343e 100644 --- a/app/scripts/translatte/commands/applyMigrations.ts +++ b/app/scripts/translatte/commands/applyMigrations.ts @@ -129,7 +129,7 @@ async function applyMigrations( const migrationFilesAttrs = await getMigrationFilesAttrs(projectPath, migrationFilePath); const selectedMigrationFilesAttrs = from - ? migrationFilesAttrs.filter((item) => (item.migrationName > from)) + ? migrationFilesAttrs.filter((item) => (item.migrationFileName > from)) : migrationFilesAttrs; console.info(`Found ${selectedMigrationFilesAttrs.length} migration files`); @@ -139,7 +139,7 @@ async function applyMigrations( } const selectedMigrations = await readMigrations( - selectedMigrationFilesAttrs.map((migration) => migration.fileName), + selectedMigrationFilesAttrs.map((migration) => migration.filePath), ); const lastMigration = selectedMigrations[selectedMigrations.length - 1]; diff --git a/app/scripts/translatte/commands/clearServerStrings.ts b/app/scripts/translatte/commands/clearServerStrings.ts index 6df5d923b5..d368922403 100644 --- a/app/scripts/translatte/commands/clearServerStrings.ts +++ b/app/scripts/translatte/commands/clearServerStrings.ts @@ -1,11 +1,11 @@ -import { listToGroupList } from "@togglecorp/fujs"; +import { isTruthyString, listToGroupList } from "@togglecorp/fujs"; import { fetchServerState, postLanguageStrings, writeFilePromisify } from "../utils"; async function clearServerStrings(apiUrl: string, authToken: string) { const serverStrings = await fetchServerState(apiUrl, authToken); const bulkActions = listToGroupList( - serverStrings, + serverStrings.filter(({ page_name }) => isTruthyString(page_name)), ({ language }) => language, ({ key, page_name }) => ({ action: "delete" as const, @@ -19,7 +19,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { response: object, }[] = []; - console.log('Pusing delete actions for en...') + console.log('Pushing delete actions for en...') const enResponse = await postLanguageStrings( 'en', bulkActions.en, @@ -31,7 +31,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { logs.push({ responseFor: 'en', response: enResponseJson }); - console.log('Pusing delete actions for fr...') + console.log('Pushing delete actions for fr...') const frResponse = await postLanguageStrings( 'fr', bulkActions.fr, @@ -42,7 +42,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { const frResponseJson = await frResponse.json(); logs.push({ responseFor: 'fr', response: frResponseJson }); - console.log('Pusing delete actions for es...') + console.log('Pushing delete actions for es...') const esResponse = await postLanguageStrings( 'es', bulkActions.es, @@ -52,7 +52,7 @@ async function clearServerStrings(apiUrl: string, authToken: string) { const esResponseJson = await esResponse.json(); logs.push({ responseFor: 'es', response: esResponseJson }); - console.log('Pusing delete actions for ar...') + console.log('Pushing delete actions for ar...') const arResponse = await postLanguageStrings( 'ar', bulkActions.ar, diff --git a/app/scripts/translatte/commands/exportServerStringsToExcel.ts b/app/scripts/translatte/commands/exportServerStringsToExcel.ts index bcfa7f8e93..73b8f966f1 100644 --- a/app/scripts/translatte/commands/exportServerStringsToExcel.ts +++ b/app/scripts/translatte/commands/exportServerStringsToExcel.ts @@ -1,7 +1,7 @@ import xlsx from 'exceljs'; import { fetchServerState } from "../utils"; -import { isFalsyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { isFalsyString, isTruthyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; async function exportServerStringsToExcel( apiUrl: string, @@ -35,7 +35,7 @@ async function exportServerStringsToExcel( const keyGroupedStrings = mapToList( listToGroupList( - serverStrings, + serverStrings.filter(({ page_name, key }) => isTruthyString(page_name) && isTruthyString(key)), ({ page_name, key }) => `${page_name}:${key}`, ), (list) => { diff --git a/app/scripts/translatte/commands/generateMigration.ts b/app/scripts/translatte/commands/generateMigration.ts index f42e6525d5..516a9b51ce 100644 --- a/app/scripts/translatte/commands/generateMigration.ts +++ b/app/scripts/translatte/commands/generateMigration.ts @@ -132,7 +132,7 @@ async function generate( const selectedMigrationFilesAttrs = migrationFilesAttrs; console.info(`Found ${selectedMigrationFilesAttrs.length} migration files`); const selectedMigrations = await readMigrations( - selectedMigrationFilesAttrs.map((migration) => migration.fileName), + selectedMigrationFilesAttrs.map((migration) => migration.filePath), ); const mergedMigrationActions = merge( selectedMigrations.map((migration) => migration.content), @@ -170,7 +170,7 @@ async function generate( const lastMigration = migrationFilesAttrs[migrationFilesAttrs.length - 1]; const migrationContent: MigrationFileContent = { - parent: lastMigration?.migrationName, + parent: lastMigration?.migrationFileName, actions: migrationActionItems, } diff --git a/app/scripts/translatte/commands/lintMigrations.ts b/app/scripts/translatte/commands/lintMigrations.ts new file mode 100644 index 0000000000..474e4821cc --- /dev/null +++ b/app/scripts/translatte/commands/lintMigrations.ts @@ -0,0 +1,28 @@ +import { listToGroupList } from '@togglecorp/fujs'; +import { getMigrationFilesAttrs } from '../utils'; + +async function lintMigrations(projectPath: string, path: string) { + const migrationFileAttrs = await getMigrationFilesAttrs(projectPath, path); + console.info(`Found ${migrationFileAttrs.length} migration files.`); + + const migrationGroups = listToGroupList( + migrationFileAttrs, + ({ migrationName }) => migrationName, + ); + + const duplicates = Object.values(migrationGroups).filter((group) => group.length > 1); + + if (duplicates.length > 0) { + const duplicateStr = duplicates.map((duplicate) => ( + duplicate.map(({ migrationName }) => migrationName).join(' <> ') + )).join('\n'); + + console.info(duplicateStr); + + throw `Error: found divirging migrations!`; + } + + console.info('All good! No divirging migrations!'); +} + +export default lintMigrations; diff --git a/app/scripts/translatte/commands/listMigrations.test.ts b/app/scripts/translatte/commands/listMigrations.test.ts index 5fa7a3699e..bf3e236a0d 100644 --- a/app/scripts/translatte/commands/listMigrations.test.ts +++ b/app/scripts/translatte/commands/listMigrations.test.ts @@ -37,12 +37,12 @@ testWithTmpDir('test listMigrations', async ({ tmpdir }) => { (await getMigrationFilesAttrs( tmpdir, 'migrations', - )).map((item) => ({ ...item, fileName: undefined })), + )).map(({ migrationFileName, num, timestamp }) => ({ migrationFileName, num, timestamp })), ).toEqual([ - { migrationName: '001-1000000000000.json', num: '001', timestamp: '1000000000000' }, - { migrationName: '002-1000000000000.json', num: '002', timestamp: '1000000000000' }, - { migrationName: '003-1000000000000.json', num: '003', timestamp: '1000000000000' }, - { migrationName: '004-1000000000000.json', num: '004', timestamp: '1000000000000' }, - { migrationName: '005-1000000000000.json', num: '005', timestamp: '1000000000000' }, + { migrationFileName: '001-1000000000000.json', num: '001', timestamp: '1000000000000' }, + { migrationFileName: '002-1000000000000.json', num: '002', timestamp: '1000000000000' }, + { migrationFileName: '003-1000000000000.json', num: '003', timestamp: '1000000000000' }, + { migrationFileName: '004-1000000000000.json', num: '004', timestamp: '1000000000000' }, + { migrationFileName: '005-1000000000000.json', num: '005', timestamp: '1000000000000' }, ]); }); diff --git a/app/scripts/translatte/commands/mergeMigrations.ts b/app/scripts/translatte/commands/mergeMigrations.ts index 4be6b0f0ab..9eadb0f313 100644 --- a/app/scripts/translatte/commands/mergeMigrations.ts +++ b/app/scripts/translatte/commands/mergeMigrations.ts @@ -1,158 +1,12 @@ -import { listToMap, isDefined } from '@togglecorp/fujs'; - import { MigrationActionItem, MigrationFileContent } from '../types'; import { - concat, - removeUndefinedKeys, getMigrationFilesAttrs, readMigrations, removeFiles, - writeFilePromisify + writeFilePromisify, + mergeMigrationActionItems } from '../utils'; -function getCanonicalKey( - item: MigrationActionItem, - opts: { useNewKey: boolean }, -) { - if (opts.useNewKey && item.action === 'update') { - return concat( - item.newNamespace ?? item.namespace, - item.newKey ?? item.key, - ); - } - return concat( - item.namespace, - item.key, - ); -} - -function mergeMigrationActionItems( - prevMigrationActionItems: MigrationActionItem[], - nextMigrationActionItems: MigrationActionItem[], -) { - interface PrevMappings { - [key: string]: MigrationActionItem, - } - - const prevCanonicalKeyMappings: PrevMappings = listToMap( - prevMigrationActionItems, - (item) => getCanonicalKey(item, { useNewKey: true }), - (item) => item, - ); - - interface NextMappings { - [key: string]: MigrationActionItem | null, - } - - const nextMappings = nextMigrationActionItems.reduce( - (acc, nextMigrationActionItem) => { - const canonicalKey = getCanonicalKey(nextMigrationActionItem, { useNewKey: false }) - - const prevItemWithCanonicalKey = prevCanonicalKeyMappings[canonicalKey]; - // const prevItemWithKey = prevKeyMappings[nextMigrationActionItem.key]; - - if (!prevItemWithCanonicalKey) { - return { - ...acc, - [canonicalKey]: nextMigrationActionItem, - }; - } - - if (prevItemWithCanonicalKey.action === 'add' && nextMigrationActionItem.action === 'add') { - throw `Action 'add' already exists for '${canonicalKey}'`; - } - if (prevItemWithCanonicalKey.action === 'add' && nextMigrationActionItem.action === 'remove') { - return { - ...acc, - [canonicalKey]: null, - }; - } - if (prevItemWithCanonicalKey.action === 'add' && nextMigrationActionItem.action === 'update') { - const newKey = nextMigrationActionItem.newKey - ?? prevItemWithCanonicalKey.key; - const newNamespace = nextMigrationActionItem.newNamespace - ?? prevItemWithCanonicalKey.namespace; - - const newMigrationItem = removeUndefinedKeys({ - action: 'add', - namespace: newNamespace, - key: newKey, - value: nextMigrationActionItem.newValue - ?? prevItemWithCanonicalKey.value, - }); - - const newCanonicalKey = getCanonicalKey(newMigrationItem, { useNewKey: true }); - if (acc[newCanonicalKey] !== undefined && acc[newCanonicalKey] !== null) { - throw `Action 'update' cannot be applied to '${newCanonicalKey}' as the key already exists`; - } - - return { - ...acc, - // Setting null so that we remove them on the mappings. - // No need to set null, if we have already overridden with other value - [canonicalKey]: acc[canonicalKey] === undefined || acc[canonicalKey] === null - ? null - : acc[canonicalKey], - [newCanonicalKey]: newMigrationItem, - } - } - if (prevItemWithCanonicalKey.action === 'remove' && nextMigrationActionItem.action === 'add') { - return { - ...acc, - [canonicalKey]: removeUndefinedKeys({ - action: 'update', - namespace: prevItemWithCanonicalKey.namespace, - key: prevItemWithCanonicalKey.key, - newValue: nextMigrationActionItem.value, - }) - }; - } - if (prevItemWithCanonicalKey.action === 'remove' && nextMigrationActionItem.action === 'remove') { - // pass - return acc; - } - if (prevItemWithCanonicalKey.action === 'remove' && nextMigrationActionItem.action === 'update') { - throw `Action 'update' cannot be applied to '${canonicalKey}' after action 'remove'`; - } - if (prevItemWithCanonicalKey.action === 'update' && nextMigrationActionItem.action === 'add') { - throw `Action 'add' cannot be applied to '${canonicalKey}' after action 'update'`; - } - if (prevItemWithCanonicalKey.action === 'update' && nextMigrationActionItem.action === 'update') { - return { - ...acc, - [canonicalKey]: removeUndefinedKeys({ - action: 'update', - namespace: prevItemWithCanonicalKey.namespace, - key: prevItemWithCanonicalKey.key, - newNamespace: nextMigrationActionItem.newNamespace ?? prevItemWithCanonicalKey.newNamespace, - newKey: nextMigrationActionItem.newKey ?? prevItemWithCanonicalKey.newKey, - newValue: nextMigrationActionItem.newValue ?? prevItemWithCanonicalKey.newValue, - }), - }; - } - if (prevItemWithCanonicalKey.action === 'update' && nextMigrationActionItem.action === 'remove') { - return { - ...acc, - [canonicalKey]: removeUndefinedKeys({ - action: 'remove', - namespace: prevItemWithCanonicalKey.namespace, - key: prevItemWithCanonicalKey.key, - }), - }; - } - return acc; - }, - {}, - ); - - const finalMappings = { - ...prevCanonicalKeyMappings, - ...nextMappings, - }; - - return Object.values(finalMappings).filter(isDefined); -} - export function merge(migrationFileContents: MigrationFileContent[]) { const migrationActionItems = migrationFileContents.reduce( (acc, migrationActionItem) => { @@ -174,7 +28,7 @@ async function mergeMigrations( ) { const migrationFilesAttrs = await getMigrationFilesAttrs(projectPath, path); const selectedMigrationFilesAttrs = migrationFilesAttrs.filter( - (item) => (item.migrationName >= from && item.migrationName <= to) + (item) => (item.migrationFileName >= from && item.migrationFileName <= to) ); console.info(`Found ${selectedMigrationFilesAttrs.length} migration files`); @@ -182,13 +36,13 @@ async function mergeMigrations( throw 'There should be atleast 2 migration files'; } const selectedMigrations = await readMigrations( - selectedMigrationFilesAttrs.map((migration) => migration.fileName), + selectedMigrationFilesAttrs.map((migration) => migration.filePath), ); const firstMigration= selectedMigrations[0]; const lastMigration = selectedMigrations[selectedMigrations.length - 1]; - const selectedMigrationsFileNames = selectedMigrationFilesAttrs.map((migration) => migration.fileName); + const selectedMigrationsFileNames = selectedMigrationFilesAttrs.map((migration) => migration.filePath); const mergedMigrationContent = { actions: merge(selectedMigrations.map((migration) => migration.content)), diff --git a/app/scripts/translatte/commands/pushMigration.ts b/app/scripts/translatte/commands/pushMigration.ts deleted file mode 100644 index bce31ef3a7..0000000000 --- a/app/scripts/translatte/commands/pushMigration.ts +++ /dev/null @@ -1,230 +0,0 @@ -import { isDefined, isFalsyString, isNotDefined, listToGroupList, listToMap, mapToMap } from "@togglecorp/fujs"; -import { Language, MigrationActionItem, ServerActionItem } from "../types"; -import { fetchServerState, getCombinedKey, languages, postLanguageStrings, readMigrations, writeFilePromisify } from "../utils"; -import { Md5 } from "ts-md5"; - -async function pushMigration(migrationFilePath: string, apiUrl: string, authToken: string) { - const serverStrings = await fetchServerState(apiUrl, authToken); - - const serverStringMapByCombinedKey = mapToMap( - listToGroupList( - serverStrings, - ({ key, page_name }) => getCombinedKey(key, page_name), - ), - (key) => key, - (list) => listToMap( - list, - ({ language }) => language, - ) - ); - - const migrations = await readMigrations( - [migrationFilePath] - ); - - const actions = migrations[0].content.actions; - - - function getItemsForNamespaceUpdate(actionItem: MigrationActionItem, language: Language) { - if (actionItem.action !== 'update') { - return undefined; - } - - if (isNotDefined(actionItem.newNamespace)) { - return undefined; - } - - const oldCombinedKey = getCombinedKey( - actionItem.key, - actionItem.namespace, - ); - - const oldStringItem = serverStringMapByCombinedKey[oldCombinedKey]?.[language]; - - if (isNotDefined(oldStringItem) || isFalsyString(oldStringItem.value)) { - return undefined; - } - - return [ - { - action: 'delete' as const, - key: actionItem.key, - page_name: actionItem.namespace, - }, - { - action: 'set' as const, - key: actionItem.key, - page_name: actionItem.newNamespace, - value: oldStringItem.value, - hash: oldStringItem.hash, - }, - ]; - } - - function getItemsForKeyUpdate(actionItem: MigrationActionItem, language: Language) { - if (actionItem.action !== 'update') { - return undefined; - } - - if (isNotDefined(actionItem.newKey)) { - return undefined; - } - - const oldCombinedKey = getCombinedKey( - actionItem.key, - actionItem.namespace, - ); - - const oldStringItem = serverStringMapByCombinedKey[oldCombinedKey]?.[language]; - - if (isNotDefined(oldStringItem) || isFalsyString(oldStringItem.value)) { - return undefined; - } - - return [ - { - action: 'delete' as const, - key: actionItem.key, - page_name: actionItem.namespace, - }, - { - action: 'set' as const, - key: actionItem.newKey, - page_name: actionItem.namespace, - value: oldStringItem.value, - hash: oldStringItem.hash, - }, - ]; - } - - const serverActions = listToMap( - languages.map((language) => { - const serverActionsForCurrentLanguage = actions.flatMap((actionItem) => { - if (language === 'en') { - if (actionItem.action === 'add') { - return { - action: 'set' as const, - key: actionItem.key, - page_name: actionItem.namespace, - value: actionItem.value, - hash: Md5.hashStr(actionItem.value), - } - } - - if (actionItem.action === 'remove') { - return { - action: 'delete' as const, - key: actionItem.key, - page_name: actionItem.namespace, - } - } - - if (isDefined(actionItem.newNamespace)) { - return getItemsForNamespaceUpdate(actionItem, language); - } - - if (isDefined(actionItem.newKey)) { - return getItemsForKeyUpdate(actionItem, language); - } - - if (isDefined(actionItem.newValue)) { - return { - action: 'set' as const, - key: actionItem.key, - page_name: actionItem.namespace, - value: actionItem.newValue, - hash: Md5.hashStr(actionItem.newValue), - } - } - } else { - if (actionItem.action === 'remove') { - return { - action: 'delete' as const, - key: actionItem.key, - page_name: actionItem.namespace, - } - } - - if (actionItem.action === 'update') { - if (isDefined(actionItem.newNamespace)) { - return getItemsForNamespaceUpdate(actionItem, language); - } - - if (isDefined(actionItem.newKey)) { - return getItemsForKeyUpdate(actionItem, language); - } - } - } - - return undefined; - }).filter(isDefined); - - return { - language, - actions: serverActionsForCurrentLanguage, - } - }), - ({ language }) => language, - ); - - await writeFilePromisify( - `server-actions.json`, - JSON.stringify(serverActions, null, 2), - 'utf8', - ); - - const logs: { - responseFor: string, - response: object, - }[] = []; - - async function applyAction(lang: Language, actions: ServerActionItem[]) { - console.log(`Pusing actions for ${lang}...`) - const response = await postLanguageStrings( - lang, - actions, - apiUrl, - authToken, - ); - const responseJson = await response.json(); - logs.push({ responseFor: 'en', response: responseJson }); - - /* - const setActions = actions.filter(({ action }) => action === 'set'); - const deleteActions = actions.filter(({ action }) => action === 'delete'); - - console.log(`Pusing deleted actions for ${lang}...`) - const deleteResponse = await postLanguageStrings( - lang, - deleteActions, - apiUrl, - authToken, - ); - const deleteResponseJson = await deleteResponse.json(); - logs.push({ responseFor: 'delete en', response: deleteResponseJson }); - - console.log(`Pusing set actions for ${lang}...`) - const setResponse = await postLanguageStrings( - lang, - setActions, - apiUrl, - authToken, - ); - const setResponseJson = await setResponse.json(); - logs.push({ responseFor: 'set en', response: setResponseJson }); - */ - } - - await applyAction(serverActions.en.language, serverActions.en.actions); - await applyAction(serverActions.fr.language, serverActions.fr.actions); - await applyAction(serverActions.es.language, serverActions.es.actions); - await applyAction(serverActions.ar.language, serverActions.ar.actions); - - await writeFilePromisify( - `push-migration-logs.json`, - JSON.stringify(logs, null, 2), - 'utf8', - ); -} - -export default pushMigration; diff --git a/app/scripts/translatte/commands/pushMigrationsToGo.ts b/app/scripts/translatte/commands/pushMigrationsToGo.ts new file mode 100644 index 0000000000..7cefa69a88 --- /dev/null +++ b/app/scripts/translatte/commands/pushMigrationsToGo.ts @@ -0,0 +1,267 @@ +import xlsx from 'xlsx'; +import { Md5 } from 'ts-md5'; +import { compareString, isFalsyString, isNotDefined, isTruthyString, listToGroupList, listToMap, mapToList } from "@togglecorp/fujs"; +import { fetchServerState, getMigrationFilesAttrs, mergeMigrationActionItems, readMigrations } from "../utils"; +import { MigrationActionItem, MigrationFileContent } from "../types"; + +type Translation = { + page: string; + key: string; + en: string; + hash: string; + fr: string | undefined; + es: string | undefined; + ar: string | undefined; +} + +const META_PAGE_NAME = '__meta'; +const LAST_MIGRATION_KEY_NAME = 'lastClientMigration'; + +function getCombinedKey(page: string, key: string) { + return `${page}:${key}`; +} + +function applyMigrationActions( + translations: Translation[], + migrationActions: MigrationFileContent['actions'], +): Translation[] { + const translationsMap = listToMap( + translations, + (item) => getCombinedKey(item.page, item.key), + ); + + migrationActions.map((migration) => { + const combinedKey = getCombinedKey(migration.namespace, migration.key); + + if (migration.action === 'remove') { + delete translationsMap[combinedKey]; + } else if (migration.action === 'add') { + const { + key, + namespace, + value, + } = migration; + + const hash = Md5.hashStr(value); + + translationsMap[combinedKey] = { + page: namespace, + key, + en: value, + hash, + fr: undefined, + es: undefined, + ar: undefined, + } satisfies Translation; + } else { + const { + key, + namespace, + // value, + newValue, + newKey, + newNamespace, + } = migration; + + const oldTranslation = translationsMap[combinedKey]; + + if (isTruthyString(newKey)) { + if (isNotDefined(oldTranslation)) { + console.info(`Update key error: cannot find translation for ${combinedKey}`); + return; + } + + const newCombinedKey = getCombinedKey(namespace, newKey); + + translationsMap[newCombinedKey] = { + ...oldTranslation, + key: newKey, + page: namespace, + } satisfies Translation; + + delete translationsMap[combinedKey]; + } else if (isTruthyString(newNamespace)) { + if (isNotDefined(oldTranslation)) { + console.info(`Update namespace error: cannot find translation for ${combinedKey}`); + return; + } + + const newCombinedKey = getCombinedKey(newNamespace, key); + translationsMap[newCombinedKey] = { + ...oldTranslation, + key, + page: newNamespace, + } satisfies Translation; + + delete translationsMap[combinedKey]; + } else { + if (isFalsyString(newValue)) { + console.info('Update value error: new value is not defined'); + return; + } + + const hash = Md5.hashStr(newValue); + + translationsMap[combinedKey] = { + key, + page: namespace, + en: newValue, + hash, + fr: undefined, + es: undefined, + ar: undefined, + } satisfies Translation; + } + } + }); + + return mapToList(translationsMap); +} + +async function pushMigrationsToGo( + projectPath: string, + migrationDirPath: string, + apiUrl: string, + authToken: string, +) { + const migrationFilesAttrs = await getMigrationFilesAttrs(projectPath, migrationDirPath); + + const serverState = await fetchServerState(apiUrl); + + const translations: Translation[] = Object.values( + listToGroupList( + serverState, + ({ page_name, key }) => getCombinedKey(page_name, key), + undefined, + (groupedList) => { + const languageMappedStrings = listToMap( + groupedList, + ({ language }) => language, + ); + + return { + page: languageMappedStrings.en.page_name, + key: languageMappedStrings.en.key, + en: languageMappedStrings.en.value, + hash: languageMappedStrings.en.hash, + fr: languageMappedStrings.fr?.value, + es: languageMappedStrings.es?.value, + ar: languageMappedStrings.ar?.value, + } satisfies Translation + } + ), + ); + + if (translations.length === 0) { + console.info('Cannot find any translations in the server'); + // FIXME: the case for first upload is not handled here + return; + } + + const groupedTranslations = listToMap( + translations ?? [], + ({ page, key }) => getCombinedKey(page, key), + ); + + const lastMigrationCombinedKey = getCombinedKey(META_PAGE_NAME, LAST_MIGRATION_KEY_NAME); + const lastMigrationName = groupedTranslations[lastMigrationCombinedKey]?.en; + + if (isFalsyString(lastMigrationName)) { + console.info('Cannot find the last applied migration in the remote system!'); + return; + } + + console.info('Last applied migration:', lastMigrationName); + + const attrIndex = migrationFilesAttrs.findIndex( + ({ migrationName }) => migrationName === lastMigrationName, + ); + + if (attrIndex === -1) { + console.info('Cannot find the last applied migration in local system!'); + return; + } + + const remainingMigrationFilesAttr = migrationFilesAttrs.slice(attrIndex + 1); + + if (remainingMigrationFilesAttr.length === 0) { + console.info('No migrations left to apply!'); + return; + } + + console.info(`Found ${remainingMigrationFilesAttr.length} migrations to apply!`); + + const migrations = await readMigrations( + remainingMigrationFilesAttr.map(({ filePath }) => filePath), + ); + + + const migrationActions = migrations.reduce( + (acc, migration) => ( + mergeMigrationActionItems(acc, migration.content.actions) + ), + [], + ); + + console.info(`Applying ${migrationActions.length} actions`); + + const newTranslations = applyMigrationActions( + translations, + migrationActions, + ); + + newTranslations.sort((a, b) => { + return compareString(a.page, b.page) || compareString(a.key, b.key); + }); + + const newLatestMigrationName = remainingMigrationFilesAttr[remainingMigrationFilesAttr.length - 1].migrationName; + + const lastMigrationMetaKeyIndex = newTranslations.findIndex((translation) => ( + getCombinedKey(translation.page, translation.key) === lastMigrationCombinedKey + )); + + const lastMigrationMetaItem: Translation = { + page: META_PAGE_NAME, + key: LAST_MIGRATION_KEY_NAME, + en: newLatestMigrationName, + hash: newLatestMigrationName, + fr: undefined, + es: undefined, + ar: undefined, + } + + if (lastMigrationMetaKeyIndex=== -1) { + newTranslations.unshift(lastMigrationMetaItem); + } else { + newTranslations.splice(lastMigrationMetaKeyIndex, 1, lastMigrationMetaItem); + } + + const newWorksheet = xlsx.utils.json_to_sheet(newTranslations.map((translation) => ({ + namespace: translation.page, + key: translation.key, + en: translation.en, + fr: translation.fr, + es: translation.es, + ar: translation.ar, + }))); + const newWorkbook = xlsx.utils.book_new(); + xlsx.utils.book_append_sheet( + newWorkbook, + newWorksheet, + newLatestMigrationName, + ); + + const now = new Date(); + const yyyy = now.getFullYear(); + const mm = (now.getMonth() + 1).toString().padStart(2, '0'); + const dd = now.getDate().toString().padStart(2, '0'); + const HH = now.getHours().toString().padStart(2, '0'); + const MM = now.getMinutes().toString().padStart(2, '0'); + + const fileName = `migrated-strings-${newLatestMigrationName}--${yyyy}-${mm}-${dd}--${HH}-${MM}.xlsx`; + console.info(`Writing to ${fileName}`); + + await xlsx.writeFile(newWorkbook, fileName); +} + +export default pushMigrationsToGo; diff --git a/app/scripts/translatte/commands/pushMigrationsToIfrc.ts b/app/scripts/translatte/commands/pushMigrationsToIfrc.ts new file mode 100644 index 0000000000..e39e9a4ce2 --- /dev/null +++ b/app/scripts/translatte/commands/pushMigrationsToIfrc.ts @@ -0,0 +1,327 @@ +// NOTE: we are using sheetJS instead of excelJS here +// because for some reason excelJS cannot parse exports +// from IFRC translation service +import xlsx from 'xlsx'; + +import { getMigrationFilesAttrs, mergeMigrationActionItems, readMigrations, resolveUrl } from "../utils"; +import { isDefined, isFalsyString, isNotDefined, isTruthyString, listToMap, mapToList } from '@togglecorp/fujs'; +import { MigrationActionItem, MigrationFileContent } from '../types'; + +/* +export function isTranslatedTemplateValid( + source: string, + translation: string, +): boolean { + const extract = (s: string): Set => { + const set = new Set(); + const re = /\{([^{}]+)\}/g; + let match: RegExpExecArray | null; + + while ((match = re.exec(s)) !== null) { + const key = match[1].trim(); + if (key) set.add(key); + } + return set; + }; + + const sourceVars = extract(source); + const translationVars = extract(translation); + + if (sourceVars.size !== translationVars.size) return false; + + for (const v of sourceVars) { + if (!translationVars.has(v)) return false; + } + + return true; +} +*/ + +// FIXME: get this from params +const applicationId = 18; + +async function fetchTranslations(ifrcApiUrl: string, ifrcApiKey: string) { + const endpoint = resolveUrl(ifrcApiUrl, `api/Application/${applicationId}/Translation/export`); + + const headers: RequestInit['headers'] = { + // 'Accept': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + // 'Accept': 'application/octet-stream', + 'X-API-KEY': ifrcApiKey, + } + + const promise = fetch( + endpoint, + { + method: 'GET', + headers, + } + ); + + return promise; +} + +async function pushTranslations(blob: Blob, ifrcApiUrl: string, ifrcApiKey: string) { + const endpoint = resolveUrl(ifrcApiUrl, `api/Application/${applicationId}/Translation/fullappimport`); + + console.info('posting to', endpoint); + + const formData = new FormData(); + formData.append('files', blob, 'translations.xlsx'); + + const headers: RequestInit['headers'] = { + 'Accept': 'application/json', + 'X-API-KEY': ifrcApiKey, + } + + const promise = fetch( + endpoint, + { + method: 'POST', + headers, + body: formData, + } + ); + + return promise; +} + +type Translation = { + page: string; + key: string; + en: string; + fr: string | undefined; + es: string | undefined; + ar: string | undefined; +} + +function applyMigrationActions( + translations: Translation[], + migrationActions: MigrationFileContent['actions'], +): Translation[] { + const translationsMap = listToMap( + translations, + (item) => getCombinedKey(item.page, item.key), + ); + + migrationActions.map((migration) => { + const combinedKey = getCombinedKey(migration.namespace, migration.key); + + if (migration.action === 'remove') { + delete translationsMap[combinedKey]; + } else if (migration.action === 'add') { + const { + key, + namespace, + value, + } = migration; + + translationsMap[combinedKey] = { + page: namespace, + key, + en: value, + fr: undefined, + es: undefined, + ar: undefined, + } satisfies Translation; + } else { + const { + key, + namespace, + // value, + newValue, + newKey, + newNamespace, + } = migration; + + const oldTranslation = translationsMap[combinedKey]; + + if (isTruthyString(newKey)) { + if (isNotDefined(oldTranslation)) { + console.info(`Update key error: cannot find translation for ${combinedKey}`); + return; + } + + const newCombinedKey = getCombinedKey(namespace, newKey); + + translationsMap[newCombinedKey] = { + ...oldTranslation, + key: newKey, + page: namespace, + } satisfies Translation; + + delete translationsMap[combinedKey]; + } else if (isTruthyString(newNamespace)) { + if (isNotDefined(oldTranslation)) { + console.info(`Update namespace error: cannot find translation for ${combinedKey}`); + return; + } + + const newCombinedKey = getCombinedKey(newNamespace, key); + translationsMap[newCombinedKey] = { + ...oldTranslation, + key, + page: newNamespace, + } satisfies Translation; + + delete translationsMap[combinedKey]; + } else { + if (isFalsyString(newValue)) { + console.info('Update value error: new value is not defined'); + return; + } + + translationsMap[combinedKey] = { + key, + page: namespace, + en: newValue, + fr: undefined, + es: undefined, + ar: undefined, + } satisfies Translation; + } + } + }); + + return mapToList(translationsMap); +} + +const META_PAGE_NAME = '__meta'; +const LAST_MIGRATION_KEY_NAME = 'lastClientMigration'; + +function getCombinedKey(page: string, key: string) { + return `${page}:${key}`; +} + +async function pushMigrationsToIfrc( + projectPath: string, + migrationDirPath: string, + apiUrl: string, + apiKey: string, +) { + const migrationFilesAttrs = await getMigrationFilesAttrs(projectPath, migrationDirPath); + + const fetchResult = await fetchTranslations(apiUrl, apiKey); + const arrayBuffer = await fetchResult.arrayBuffer(); + const workbook= xlsx.read(arrayBuffer); + const firstSheetData = xlsx.utils.sheet_to_json( + workbook.Sheets[workbook.SheetNames[0]], + ) as Partial[] | undefined; + + const translations = firstSheetData?.map((row) => { + const { + page, + key, + en, + fr, + es, + ar, + } = row; + + if (isFalsyString(page) || isFalsyString(key) || isFalsyString(en)) { + return undefined; + } + + return { + ...row, + page, + key, + en, + fr, + es, + ar, + } satisfies Translation; + }).filter(isDefined) ?? []; + + if (translations.length === 0) { + console.info('Cannot find any translations in the server'); + // FIXME: the case for first upload is not handled here + return; + } + + const groupedTranslations = listToMap( + translations ?? [], + ({ page, key }) => getCombinedKey(page, key), + ); + + const lastMigrationCombinedKey = getCombinedKey(META_PAGE_NAME, LAST_MIGRATION_KEY_NAME); + const lastMigrationName = groupedTranslations[lastMigrationCombinedKey]?.en; + + if (isFalsyString(lastMigrationName)) { + console.info('Cannot find the last applied migration in the remote system!'); + return; + } + + console.info('Last applied migration:', lastMigrationName); + + const attrIndex = migrationFilesAttrs.findIndex( + ({ migrationName }) => migrationName === lastMigrationName, + ); + + if (attrIndex === -1) { + console.info('Cannot find the last applied migration in local system!'); + return; + } + + const remainingMigrationFilesAttr = migrationFilesAttrs.slice(attrIndex + 1); + + if (remainingMigrationFilesAttr.length === 0) { + console.info('No migrations left to apply!'); + return; + } + + console.info(`Found ${remainingMigrationFilesAttr.length} migrations to apply!`); + + const migrations = await readMigrations( + remainingMigrationFilesAttr.map(({ filePath }) => filePath), + ); + + const migrationActions = migrations.reduce( + (acc, migration) => ( + mergeMigrationActionItems(acc, migration.content.actions) + ), + [], + ); + + const newTranslations = applyMigrationActions( + translations, + migrationActions, + ); + + const newLatestMigrationName = remainingMigrationFilesAttr[remainingMigrationFilesAttr.length - 1].migrationName; + + const lastMigrationMetaIndex = newTranslations.findIndex( + ({ page, key }) => getCombinedKey(page, key) === getCombinedKey(META_PAGE_NAME, LAST_MIGRATION_KEY_NAME) + ); + + const metaItem = { + page: META_PAGE_NAME, + key: LAST_MIGRATION_KEY_NAME, + en: newLatestMigrationName, + fr: undefined, + es: undefined, + ar: undefined, + } satisfies Translation; + + if (lastMigrationMetaIndex === -1) { + newTranslations.push(metaItem); + } else { + newTranslations.splice(lastMigrationMetaIndex, 1, metaItem); + } + + const newWorksheet = xlsx.utils.json_to_sheet(newTranslations); + const newWorkbook = xlsx.utils.book_new(); + xlsx.utils.book_append_sheet( + newWorkbook, + newWorksheet, + newLatestMigrationName, + ); + + const u8 = xlsx.write(newWorkbook, { bookType: 'xlsx', type: 'buffer' }); + const blob = new Blob([u8], { type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" }) + + const pushResult = await pushTranslations(blob, apiUrl, apiKey); + + console.info(await pushResult.text()); +} + +export default pushMigrationsToIfrc; diff --git a/app/scripts/translatte/commands/pushStringsFromExcel.ts b/app/scripts/translatte/commands/pushStringsFromExcel.ts index b9da7b38b5..5ca3197db9 100644 --- a/app/scripts/translatte/commands/pushStringsFromExcel.ts +++ b/app/scripts/translatte/commands/pushStringsFromExcel.ts @@ -1,25 +1,120 @@ -import xlsx from 'exceljs'; +import xlsx, { CellValue, Row } from 'exceljs'; import { Md5 } from 'ts-md5'; -import { isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs'; +import { encodeDate, isDefined, isFalsyString, isNotDefined, listToMap } from '@togglecorp/fujs'; import { Language, ServerActionItem } from '../types'; -import { postLanguageStrings } from '../utils'; +import { postLanguageStrings, writeFilePromisify } from '../utils'; + +type Translation = { + key: string; + namespace: string; + en: string; + fr: string | undefined; + es: string | undefined; + ar: string | undefined; + hash: string; +} -async function pushStringsFromExcel(importFilePath: string, apiUrl: string, accessToken: string) { - const workbook = new xlsx.Workbook(); +function resolveCellValue(cellValue: CellValue) { + if (isNotDefined(cellValue)) { + return undefined; + } + + if ( + typeof cellValue === 'number' + || typeof cellValue === 'string' + || typeof cellValue === 'boolean' + ) { + return cellValue; + } + + if (cellValue instanceof Date) { + return encodeDate(cellValue); + } + + if ('error' in cellValue) { + return undefined; + } + + if ('richText' in cellValue) { + return cellValue.richText.map(({ text }) => text).join(''); + } + + if ('hyperlink' in cellValue) { + const MAIL_IDENTIFIER = 'mailto:'; + if (cellValue.hyperlink.startsWith(MAIL_IDENTIFIER)) { + return cellValue.hyperlink.substring(MAIL_IDENTIFIER.length); + } + + return cellValue.hyperlink; + } + + if (isNotDefined(cellValue.result)) { + return undefined; + } + + if (typeof cellValue.result === 'object' && 'error' in cellValue.result) { + return undefined; + } + + // Formula result + return resolveCellValue(cellValue.result); +} + +function getStringValueFromCellValue(cellValue: CellValue | undefined) { + if (isNotDefined(cellValue)) { + return undefined; + } + + const resolvedValue = resolveCellValue(cellValue); + + if (isNotDefined(resolvedValue)) { + return undefined; + } + + const stringValue = String(resolvedValue); - await workbook.xlsx.readFile(importFilePath); + if (isFalsyString(stringValue.trim())) { + return undefined; + } + + return stringValue; +} + +function getCellValueFromRow(row: Row, columnIndex: number | undefined) { + if (isNotDefined(row) || isNotDefined(columnIndex)) { + return undefined; + } + + const cellValue = row.getCell(columnIndex).value; + + return getStringValueFromCellValue(cellValue); +} + +async function getExcelTranslations(excelFilePath: string) { + const workbook = new xlsx.Workbook(); + await workbook.xlsx.readFile(excelFilePath); const firstSheet = workbook.worksheets[0]; - const columns = firstSheet.columns.map( - (column) => { - const key = column.values?.[1]?.toString(); - if (isNotDefined(key)) { - return undefined; - } - return { key, column: column.number } + + const columns: { + key: string; + column: number | undefined; + }[] = []; + + for (let i = 0; i < firstSheet.columnCount; i++) { + const column = firstSheet.columns[i]; + const key = column.values?.[1]?.toString(); + + if (isNotDefined(key)) { + return; } - ).filter(isDefined); + + columns.push({ + key: key.toLowerCase(), + column: column.number, + }) + } const columnMap = listToMap( columns, @@ -27,116 +122,116 @@ async function pushStringsFromExcel(importFilePath: string, apiUrl: string, acce ({ column }) => column, ); - const strings: { - key: string; - namespace: string; - language: Language; - value: string; - hash: string; - }[] = []; + // Remove header row + firstSheet.spliceRows(1, 1); - firstSheet.eachRow( - (row) => { - const keyColumn = columnMap['key']; - const key = isDefined(keyColumn) ? row.getCell(keyColumn).value?.toString() : undefined; + const KEY = 'key'; + const NAMESPACE = 'namespace'; + const EN = 'en'; + const FR = 'fr'; + const ES = 'es'; + const AR = 'ar'; - const namespaceColumn = columnMap['namespace']; - const namespace = isDefined(namespaceColumn) ? row.getCell(namespaceColumn).value?.toString() : undefined; + const translations: Translation[] = []; - if (isNotDefined(key) || isNotDefined(namespace)) { - return; - } + firstSheet.eachRow((row) => { + const key = getCellValueFromRow(row, columnMap[KEY]); + const namespace = getCellValueFromRow(row, columnMap[NAMESPACE]); - const enColumn = columnMap['en']; - const en = isDefined(enColumn) ? row.getCell(enColumn).value?.toString() : undefined; + if (isFalsyString(key) || isFalsyString(namespace)) { + return; + } - const arColumn = columnMap['ar']; - const ar = isDefined(arColumn) ? row.getCell(arColumn).value?.toString() : undefined; + const en = getCellValueFromRow(row, columnMap[EN]); - const frColumn = columnMap['fr']; - const fr = isDefined(frColumn) ? row.getCell(frColumn).value?.toString() : undefined; + if (isFalsyString(en)) { + return; + } - const esColumn = columnMap['es']; - const es = isDefined(esColumn) ? row.getCell(esColumn).value?.toString() : undefined; + const fr = getCellValueFromRow(row, columnMap[FR]); + const es = getCellValueFromRow(row, columnMap[ES]); + const ar = getCellValueFromRow(row, columnMap[AR]); - if (isNotDefined(en)) { - return; - } + const hash = Md5.hashStr(String(en)); - const hash = Md5.hashStr(en); - - strings.push({ - key, - namespace, - language: 'en', - value: en, - hash, - }); - - if (isDefined(ar)) { - strings.push({ - key, - namespace, - language: 'ar', - value: ar, - hash, - }); - } + translations.push({ + key, + namespace, + en, + fr, + es, + ar, + hash, + }); + }); - if (isDefined(fr)) { - strings.push({ - key, - namespace, - language: 'fr', - value: fr, - hash, - }); - } + return translations; +} - if (isDefined(es)) { - strings.push({ - key, - namespace, - language: 'es', - value: es, - hash, - }); +async function pushStringsFromExcel(importFilePath: string, apiUrl: string, accessToken: string) { + const translations = await getExcelTranslations(importFilePath); + + if (isNotDefined(translations)) { + console.info('Could not process the given excel file', importFilePath); + return; + } + + console.info(`Found ${translations.length} rows`); + + const applicableLanguages: Language[] = ['en', 'fr', 'es', 'ar']; + + const actionsByLanguage = listToMap( + applicableLanguages, + (lang) => lang, + (lang) => translations.map((translation) => { + const languageValue = translation[lang]; + + if (isNotDefined(languageValue)) { + return undefined; } - } - ); - const languageGroupedActions = mapToList( - listToGroupList( - strings, - ({ language }) => language, - (languageString) => { - const serverAction: ServerActionItem = { - action: 'set', - key: languageString.key, - page_name: languageString.namespace, - value: languageString.value, - hash: languageString.hash, - } - - return serverAction; - }, - ), - (actions, language) => ({ - language: language as Language, - actions, - }) + return { + action: 'set', + key: translation.key, + page_name: translation.namespace, + value: languageValue, + hash: translation.hash, + } satisfies ServerActionItem; + }).filter(isDefined), ); - const postPromises = languageGroupedActions.map( - (languageStrings) => postLanguageStrings( - languageStrings.language, - languageStrings.actions, - apiUrl, - accessToken, - ) - ) - - await Promise.all(postPromises); + for (let i = 0; i < applicableLanguages.length; i++) { + const language = applicableLanguages[i]; + const actions = actionsByLanguage[language]; + + if (actions.length > 0) { + console.log(`posting ${actions.length} ${language} actions...`); + const result = await postLanguageStrings( + language, + actions, + apiUrl, + accessToken, + ); + + try { + const resultJson = await result.json(); + + await writeFilePromisify( + `/tmp/push-strings-from-excel-response-${language}.json`, + JSON.stringify(resultJson, null, 2), + 'utf8', + ); + } catch { + const resultText = await result.text(); + + await writeFilePromisify( + `/tmp/push-strings-from-excel-response-${language}.log`, + JSON.stringify(resultText, null, 2), + 'utf8', + ); + } + } + } } export default pushStringsFromExcel; diff --git a/app/scripts/translatte/commands/pushStringsFromExcelToIfrc.ts b/app/scripts/translatte/commands/pushStringsFromExcelToIfrc.ts new file mode 100644 index 0000000000..fac205ac6d --- /dev/null +++ b/app/scripts/translatte/commands/pushStringsFromExcelToIfrc.ts @@ -0,0 +1,48 @@ +import { readFileSync } from "fs"; +import { resolveUrl } from "../utils"; + +// FIXME: get this from params +const applicationId = 18; + +async function fullAppImport(importFilePath: string, ifrcApiUrl: string, ifrcApiKey: string) { + const endpoint = resolveUrl(ifrcApiUrl, `Application/${applicationId}/Translation/fullappimport`); + const translationFile = readFileSync(importFilePath); + const uint8FileData = new Uint8Array(translationFile); + const blob = new Blob([uint8FileData], { + type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + }); + + const formData = new FormData(); + formData.append('files', blob, 'translations.xlsx'); + + const headers: RequestInit['headers'] = { + 'Accept': 'application/json', + 'X-API-KEY': ifrcApiKey, + } + + const promise = fetch( + endpoint, + { + method: 'POST', + headers, + body: formData, + } + ); + + return promise; +} + +async function pushStringsFromExcelToIfrc(importFilePath: string, apiUrl: string, apiKey: string) { + const response = await fullAppImport(importFilePath, apiUrl, apiKey); + + try { + const responseJson = await response.json(); + console.info(responseJson); + } catch(e) { + console.info(e); + const responseText = await response.text(); + console.info(responseText); + } +} + +export default pushStringsFromExcelToIfrc; diff --git a/app/scripts/translatte/main.ts b/app/scripts/translatte/main.ts index f8dd65a23f..266073b28b 100644 --- a/app/scripts/translatte/main.ts +++ b/app/scripts/translatte/main.ts @@ -10,10 +10,13 @@ import mergeMigrations from './commands/mergeMigrations'; import applyMigrations from './commands/applyMigrations'; import generateMigration from './commands/generateMigration'; import exportMigration from './commands/exportMigration'; -import pushMigration from './commands/pushMigration'; import pushStringsFromExcel from './commands/pushStringsFromExcel'; import exportServerStringsToExcel from './commands/exportServerStringsToExcel'; import clearServerStrings from './commands/clearServerStrings'; +import pushMigrationsToIfrc from './commands/pushMigrationsToIfrc'; +import pushStringsFromExcelToIfrc from './commands/pushStringsFromExcelToIfrc'; +import lintMigrations from './commands/lintMigrations'; +import pushMigrationsToGo from './commands/pushMigrationsToGo'; const currentDir = cwd(); @@ -42,6 +45,22 @@ yargs(hideBin(process.argv)) await lint(currentDir, argv.TRANSLATION_FILE as string[], argv.fix as boolean | undefined); }, ) + .command( + 'lint-migrations ', + 'Lint migration files for divirging migrations', + (yargs) => { + yargs.positional('MIGRATION_DIR_PATH', { + type: 'string', + describe: 'Read the files from TRANSLATION_FILE', + }); + }, + async (argv) => { + await lintMigrations( + currentDir, + argv.MIGRATION_DIR_PATH as string, + ); + }, + ) .command( 'list-migrations ', 'List migration files', @@ -193,38 +212,38 @@ yargs(hideBin(process.argv)) }, ) .command( - 'push-migration ', - 'Push migration file to the server', + 'push-strings-from-excel ', + 'Import migration from excel file and push it to server', (yargs) => { - yargs.positional('MIGRATION_FILE_PATH', { + yargs.positional('IMPORT_FILE_PATH', { type: 'string', - describe: 'Find the migration file on MIGRATION_FILE_PATH', + describe: 'Find the import file on IMPORT_FILE_PATH', }); yargs.options({ - 'api-url': { - type: 'string', - describe: 'URL for the API server', - require: true, - }, 'auth-token': { type: 'string', describe: 'Authentication token to access the API server', require: true, }, + 'api-url': { + type: 'string', + describe: 'URL for the API server', + require: true, + } }); }, async (argv) => { - const migrationFilePath = (argv.MIGRATION_FILE_PATH as string); + const importFilePath = (argv.IMPORT_FILE_PATH as string); - await pushMigration( - migrationFilePath, + await pushStringsFromExcel( + importFilePath, argv.apiUrl as string, argv.authToken as string, ); }, ) .command( - 'push-strings-from-excel ', + 'push-strings-from-excel-to-ifrc ', 'Import migration from excel file and push it to server', (yargs) => { yargs.positional('IMPORT_FILE_PATH', { @@ -232,9 +251,9 @@ yargs(hideBin(process.argv)) describe: 'Find the import file on IMPORT_FILE_PATH', }); yargs.options({ - 'auth-token': { + 'api-key': { type: 'string', - describe: 'Authentication token to access the API server', + describe: 'API key to access the API server', require: true, }, 'api-url': { @@ -247,13 +266,77 @@ yargs(hideBin(process.argv)) async (argv) => { const importFilePath = (argv.IMPORT_FILE_PATH as string); - await pushStringsFromExcel( + await pushStringsFromExcelToIfrc( importFilePath, argv.apiUrl as string, + argv.apiKey as string, + ); + }, + ) + .command( + 'push-migrations-to-go ', + 'Push migrations to GO API', + (yargs) => { + yargs.positional('MIGRATION_DIR_PATH', { + type: 'string', + describe: 'Find the import file on MIGRATION_DIR_PATH', + }); + yargs.options({ + 'auth-token': { + type: 'string', + describe: 'Authentication token to access the API server', + require: true, + }, + 'api-url': { + type: 'string', + describe: 'URL for the API server', + require: true, + } + }); + }, + async (argv) => { + const migrationDirPath = (argv.MIGRATION_DIR_PATH as string); + + await pushMigrationsToGo( + currentDir, + migrationDirPath, + argv.apiUrl as string, argv.authToken as string, ); }, ) + .command( + 'push-migrations-to-ifrc ', + 'Push migrations to IFRC translations service', + (yargs) => { + yargs.positional('MIGRATION_DIR_PATH', { + type: 'string', + describe: 'Find the import file on MIGRATION_DIR_PATH', + }); + yargs.options({ + 'api-key': { + type: 'string', + describe: 'Authentication token to access the API server', + require: true, + }, + 'api-url': { + type: 'string', + describe: 'URL for the API server', + require: true, + } + }); + }, + async (argv) => { + const migrationDirPath = (argv.MIGRATION_DIR_PATH as string); + + await pushMigrationsToIfrc( + currentDir, + migrationDirPath, + argv.apiUrl as string, + argv.apiKey as string, + ); + }, + ) .command( 'export-server-strings ', 'Export server strings to excel file', diff --git a/app/scripts/translatte/utils.ts b/app/scripts/translatte/utils.ts index 697fe3569f..4ec85314c1 100644 --- a/app/scripts/translatte/utils.ts +++ b/app/scripts/translatte/utils.ts @@ -18,6 +18,7 @@ import { Language, ServerActionItem, SourceStringItem, + MigrationActionItem, } from './types'; const readFilePromisify = promisify(readFile); @@ -26,17 +27,21 @@ const unlinkPromisify = promisify(unlink); // Utilities -export function getCombinedKey(key: string, namespace: string) { - return `${namespace}:${key}`; -} - -function resolveUrl(from: string, to: string) { - const resolvedUrl = new URL(to, new URL(from, 'resolve://')); - if (resolvedUrl.protocol === 'resolve:') { - const { pathname, search, hash } = resolvedUrl; - return pathname + search + hash; +export function resolveUrl(base: string, endpoint: string): string { + // If endpoint is already an absolute URL, return it as-is + if (/^https?:\/\//i.test(endpoint)) { + return endpoint; } - return resolvedUrl.toString(); + + // Ensure base ends with a slash + const normalizedBase = base.endsWith('/') ? base : `${base}/`; + + // Remove leading slash from endpoint to avoid URL() resetting the path + const normalizedEndpoint = endpoint.startsWith('/') + ? endpoint.slice(1) + : endpoint; + + return new URL(normalizedEndpoint, normalizedBase).toString(); } async function fetchLanguageStrings(language: Language, apiUrl: string, authToken?: string) { @@ -255,11 +260,11 @@ export function getDuplicateItems( .sort((foo, bar) => keySelector(foo).localeCompare(keySelector(bar))); } -export function concat(...args: string[]) { +function concat(...args: string[]) { return args.join(":"); } -export function removeUndefinedKeys(itemFromArgs: T) { +function removeUndefinedKeys(itemFromArgs: T) { const item = {...itemFromArgs}; Object.keys(item).forEach(key => { if (item[key as keyof T] === undefined) { @@ -277,27 +282,31 @@ export async function getMigrationFilesAttrs(basePath: string, pathName: string) const files = await glob(fullPath, { ignore: ['node_modules'], absolute: true }); interface MigrationFileAttrs { + migrationFileName: string; migrationName: string; - fileName: string; + filePath: string; num: string; timestamp: string; } const migrationFiles = files .map((file): MigrationFileAttrs | undefined => { - const migrationName = basename(file); - const attrs = migrationName.match(/(?[0-9]+)-(?[0-9]+)/)?.groups as (Omit | undefined) + const migrationFileName = basename(file); + const attrs = migrationFileName.match(/(?[0-9]+)-(?[0-9]+)/)?.groups as ( + Pick | undefined + ); if (attrs) { return { ...attrs, - migrationName, - fileName: file, + migrationName: `${attrs.num}-${attrs.timestamp}`, + migrationFileName, + filePath: file, } } return undefined; }) .filter(isDefined) - .sort((a, b) => a.migrationName.localeCompare(b.migrationName)); + .sort((a, b) => a.migrationFileName.localeCompare(b.migrationFileName)); return migrationFiles; } @@ -380,7 +389,7 @@ export async function removeFiles(files: string[]) { await Promise.all(removePromises); } -export const languages: Language[] = ['en', 'fr', 'es', 'ar']; +const languages: Language[] = ['en', 'fr', 'es', 'ar']; export async function fetchServerState(apiUrl: string, authToken?: string) { const responsePromises = languages.map( @@ -413,3 +422,193 @@ export async function fetchServerState(apiUrl: string, authToken?: string) { return serverStrings; } +/* +export function getValueFromCellValue(cellValue: CellValue) { + if (isNotDefined(cellValue)) { + return undefined; + } + + if ( + typeof cellValue === 'number' + || typeof cellValue === 'string' + || typeof cellValue === 'boolean' + ) { + return cellValue; + } + + if (cellValue instanceof Date) { + return encodeDate(cellValue); + } + + if ('error' in cellValue) { + return undefined; + } + + if ('richText' in cellValue) { + return cellValue.richText.map(({ text }) => text).join(''); + } + + if ('hyperlink' in cellValue) { + const MAIL_IDENTIFIER = 'mailto:'; + if (cellValue.hyperlink.startsWith(MAIL_IDENTIFIER)) { + return cellValue.hyperlink.substring(MAIL_IDENTIFIER.length); + } + + return cellValue.hyperlink; + } + + if (isNotDefined(cellValue.result)) { + return undefined; + } + + if (typeof cellValue.result === 'object' && 'error' in cellValue.result) { + return undefined; + } + + // Formula result + return getValueFromCellValue(cellValue.result); +} +*/ + +function getCanonicalKey( + item: MigrationActionItem, + opts: { useNewKey: boolean }, +) { + if (opts.useNewKey && item.action === 'update') { + return concat( + item.newNamespace ?? item.namespace, + item.newKey ?? item.key, + ); + } + return concat( + item.namespace, + item.key, + ); +} + +export function mergeMigrationActionItems( + prevMigrationActionItems: MigrationActionItem[], + nextMigrationActionItems: MigrationActionItem[], +) { + interface PrevMappings { + [key: string]: MigrationActionItem, + } + + const prevCanonicalKeyMappings: PrevMappings = listToMap( + prevMigrationActionItems, + (item) => getCanonicalKey(item, { useNewKey: true }), + (item) => item, + ); + + interface NextMappings { + [key: string]: MigrationActionItem | null, + } + + const nextMappings = nextMigrationActionItems.reduce( + (acc, nextMigrationActionItem) => { + const canonicalKey = getCanonicalKey(nextMigrationActionItem, { useNewKey: false }) + + const prevItemWithCanonicalKey = prevCanonicalKeyMappings[canonicalKey]; + // const prevItemWithKey = prevKeyMappings[nextMigrationActionItem.key]; + + if (!prevItemWithCanonicalKey) { + return { + ...acc, + [canonicalKey]: nextMigrationActionItem, + }; + } + + if (prevItemWithCanonicalKey.action === 'add' && nextMigrationActionItem.action === 'add') { + throw `Action 'add' already exists for '${canonicalKey}'`; + } + if (prevItemWithCanonicalKey.action === 'add' && nextMigrationActionItem.action === 'remove') { + return { + ...acc, + [canonicalKey]: null, + }; + } + if (prevItemWithCanonicalKey.action === 'add' && nextMigrationActionItem.action === 'update') { + const newKey = nextMigrationActionItem.newKey + ?? prevItemWithCanonicalKey.key; + const newNamespace = nextMigrationActionItem.newNamespace + ?? prevItemWithCanonicalKey.namespace; + + const newMigrationItem = removeUndefinedKeys({ + action: 'add', + namespace: newNamespace, + key: newKey, + value: nextMigrationActionItem.newValue + ?? prevItemWithCanonicalKey.value, + }); + + const newCanonicalKey = getCanonicalKey(newMigrationItem, { useNewKey: true }); + if (acc[newCanonicalKey] !== undefined && acc[newCanonicalKey] !== null) { + throw `Action 'update' cannot be applied to '${newCanonicalKey}' as the key already exists`; + } + + return { + ...acc, + // Setting null so that we remove them on the mappings. + // No need to set null, if we have already overridden with other value + [canonicalKey]: acc[canonicalKey] === undefined || acc[canonicalKey] === null + ? null + : acc[canonicalKey], + [newCanonicalKey]: newMigrationItem, + } + } + if (prevItemWithCanonicalKey.action === 'remove' && nextMigrationActionItem.action === 'add') { + return { + ...acc, + [canonicalKey]: removeUndefinedKeys({ + action: 'update', + namespace: prevItemWithCanonicalKey.namespace, + key: prevItemWithCanonicalKey.key, + newValue: nextMigrationActionItem.value, + }) + }; + } + if (prevItemWithCanonicalKey.action === 'remove' && nextMigrationActionItem.action === 'remove') { + // pass + return acc; + } + if (prevItemWithCanonicalKey.action === 'remove' && nextMigrationActionItem.action === 'update') { + throw `Action 'update' cannot be applied to '${canonicalKey}' after action 'remove'`; + } + if (prevItemWithCanonicalKey.action === 'update' && nextMigrationActionItem.action === 'add') { + throw `Action 'add' cannot be applied to '${canonicalKey}' after action 'update'`; + } + if (prevItemWithCanonicalKey.action === 'update' && nextMigrationActionItem.action === 'update') { + return { + ...acc, + [canonicalKey]: removeUndefinedKeys({ + action: 'update', + namespace: prevItemWithCanonicalKey.namespace, + key: prevItemWithCanonicalKey.key, + newNamespace: nextMigrationActionItem.newNamespace ?? prevItemWithCanonicalKey.newNamespace, + newKey: nextMigrationActionItem.newKey ?? prevItemWithCanonicalKey.newKey, + newValue: nextMigrationActionItem.newValue ?? prevItemWithCanonicalKey.newValue, + }), + }; + } + if (prevItemWithCanonicalKey.action === 'update' && nextMigrationActionItem.action === 'remove') { + return { + ...acc, + [canonicalKey]: removeUndefinedKeys({ + action: 'remove', + namespace: prevItemWithCanonicalKey.namespace, + key: prevItemWithCanonicalKey.key, + }), + }; + } + return acc; + }, + {}, + ); + + const finalMappings = { + ...prevCanonicalKeyMappings, + ...nextMappings, + }; + + return Object.values(finalMappings).filter(isDefined); +} diff --git a/app/src/config.ts b/app/src/config.ts index ddc85d2e47..8475164dca 100644 --- a/app/src/config.ts +++ b/app/src/config.ts @@ -6,6 +6,7 @@ const { APP_MAPBOX_ACCESS_TOKEN, APP_TINY_API_KEY, APP_RISK_API_ENDPOINT, + APP_TRANSLATION_API_ENDPOINT, APP_SDT_URL, APP_POWER_BI_REPORT_ID_1, APP_SENTRY_DSN, @@ -30,6 +31,7 @@ export const api = APP_API_ENDPOINT; export const adminUrl = APP_ADMIN_URL ?? `${api}admin/`; export const mbtoken = APP_MAPBOX_ACCESS_TOKEN; export const riskApi = APP_RISK_API_ENDPOINT; +export const translationApi = APP_TRANSLATION_API_ENDPOINT; export const sdtUrl = APP_SDT_URL; export const powerBiReportId1 = APP_POWER_BI_REPORT_ID_1; diff --git a/app/src/utils/resolveUrl.ts b/app/src/utils/resolveUrl.ts index 8c6880a4af..8283994fb0 100644 --- a/app/src/utils/resolveUrl.ts +++ b/app/src/utils/resolveUrl.ts @@ -1,9 +1,17 @@ // eslint-disable-next-line import/prefer-default-export -export function resolveUrl(from: string, to: string) { - const resolvedUrl = new URL(to, new URL(from, 'resolve://')); - if (resolvedUrl.protocol === 'resolve:') { - const { pathname, search, hash } = resolvedUrl; - return pathname + search + hash; +export function resolveUrl(base: string, endpoint: string) { + // If endpoint is already an absolute URL, return it as-is + if (/^https?:\/\//i.test(endpoint)) { + return endpoint; } - return resolvedUrl.toString(); + + // Ensure base ends with a slash + const normalizedBase = base.endsWith('/') ? base : `${base}/`; + + // Remove leading slash from endpoint to avoid URL() resetting the path + const normalizedEndpoint = endpoint.startsWith('/') + ? endpoint.slice(1) + : endpoint; + + return new URL(normalizedEndpoint, normalizedBase).toString(); } diff --git a/app/src/utils/restRequest/go.ts b/app/src/utils/restRequest/go.ts index b9153939de..68c8885bc3 100644 --- a/app/src/utils/restRequest/go.ts +++ b/app/src/utils/restRequest/go.ts @@ -9,6 +9,7 @@ import { type ContextInterface } from '@togglecorp/toggle-request'; import { api, riskApi, + translationApi, } from '#config'; import { type UserAuth } from '#contexts/user'; import { @@ -39,8 +40,10 @@ export interface TransformedError { debugMessage: string; } +type ApiType = 'go' | 'risk' | 'translation'; + export interface AdditionalOptions { - apiType?: 'go' | 'risk'; + apiType?: ApiType; formData?: boolean; isCsvRequest?: boolean; enforceEnglishForQuery?: boolean; @@ -111,6 +114,18 @@ type GoContextInterface = ContextInterface< AdditionalOptions >; +function getEndPoint(apiType: ApiType | undefined) { + if (apiType === 'risk') { + return riskApi; + } + + if (apiType === 'translation') { + return translationApi; + } + + return api; +} + export const processGoUrls: GoContextInterface['transformUrl'] = (url, _, additionalOptions) => { if (isFalsyString(url)) { return ''; @@ -123,10 +138,12 @@ export const processGoUrls: GoContextInterface['transformUrl'] = (url, _, additi const { apiType } = additionalOptions; - return resolveUrl( - apiType === 'risk' ? riskApi : api, + const resolvedUrl = resolveUrl( + getEndPoint(apiType), url, ); + + return resolvedUrl; }; type Literal = string | number | boolean | File; @@ -164,6 +181,7 @@ export const processGoOptions: GoContextInterface['transformOptions'] = ( } = requestOptions; const { + apiType, formData, isCsvRequest, isExcelRequest, @@ -176,9 +194,12 @@ export const processGoOptions: GoContextInterface['transformOptions'] = ( const user = getFromStorage(KEY_USER_STORAGE); const token = user?.token; - const defaultHeaders: HeadersInit = { - Authorization: token ? `Token ${token}` : '', - }; + // FIXME: only inject on go apis + const defaultHeaders: HeadersInit = {}; + + if (apiType === 'go' && isDefined(token)) { + defaultHeaders.Authorization = `Token ${token}`; + } if (method === 'GET') { // Query diff --git a/app/src/utils/restRequest/index.ts b/app/src/utils/restRequest/index.ts index f8ebd92b38..64a9fd3c43 100644 --- a/app/src/utils/restRequest/index.ts +++ b/app/src/utils/restRequest/index.ts @@ -5,8 +5,10 @@ import { } from '@togglecorp/toggle-request'; import type { paths as riskApiPaths } from '#generated/riskTypes'; +import type { paths as translationApiPaths } from '#generated/translationTypes'; import type { paths as goApiPaths } from '#generated/types'; +// import type { paths as translationApiPaths } from '#translationTypes'; import type { ApiBody, ApiResponse, @@ -25,19 +27,31 @@ export type GoApiUrlQuery = ApiBody export type RiskApiResponse = ApiResponse; -// type RiskApiUrlQuery< -// URL extends keyof riskApiPaths, -// METHOD extends 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' = 'GET' -// > = ApiUrlQuery -// type RiskApiBody< -// URL extends keyof riskApiPaths, -// METHOD extends 'POST' | 'PUT' | 'PATCH' -// > = ApiBody export type ListResponseItem } | undefined> = NonNullable['results']>[number]; +/* +const useTranslationRequest = useRequest as < + PATH extends keyof translationApiPaths, + METHOD extends VALID_METHOD | undefined = 'GET', +>( + requestOptions: CustomRequestOptions & { + apiType: 'translation' + } +) => CustomRequestReturn; +*/ + +// FIXME: identify a way to do this without a cast +const useTranslationLazyRequest = useLazyRequest as < + PATH extends keyof translationApiPaths, + CONTEXT = unknown, + METHOD extends VALID_METHOD | undefined = 'GET', +>( + requestOptions: CustomLazyRequestOptions & { apiType: 'translation' } +) => CustomLazyRequestReturn; + // FIXME: identify a way to do this without a cast const useGoRequest = useRequest as < PATH extends keyof goApiPaths, @@ -83,4 +97,6 @@ export { useGoRequest as useRequest, useRiskLazyRequest, useRiskRequest, + useTranslationLazyRequest, + // useTranslationRequest, }; diff --git a/app/src/views/RootLayout/index.tsx b/app/src/views/RootLayout/index.tsx index 2a2ee6b52a..04f598d1b0 100644 --- a/app/src/views/RootLayout/index.tsx +++ b/app/src/views/RootLayout/index.tsx @@ -26,12 +26,9 @@ import { } from '@ifrc-go/ui/hooks'; import { _cs, - isDefined, - isFalsyString, - listToGroupList, + isNotDefined, listToMap, mapToList, - mapToMap, } from '@togglecorp/fujs'; import GlobalFooter from '#components/GlobalFooter'; @@ -46,8 +43,8 @@ import UserContext from '#contexts/user'; import useAuth from '#hooks/domain/useAuth'; import useDebouncedValue from '#hooks/useDebouncedValue'; import { - useLazyRequest, useRequest, + useTranslationLazyRequest, } from '#utils/restRequest'; import i18n from './i18n.json'; @@ -99,53 +96,44 @@ export function Component() { const { trigger: fetchLanguage, - } = useLazyRequest<'/api/v2/language/{id}/', { pages: Array }>({ - url: '/api/v2/language/{id}/', + } = useTranslationLazyRequest<'/strings', { pages: string[] }>({ + apiType: 'translation', + url: '/strings', // FIXME: fix typing in server (medium priority) - query: ({ pages }) => ({ page_name: pages }) as never, - pathVariables: () => ({ id: currentLanguage }), + query: ({ pages }) => ({ + pages: pages.join(','), + lang: currentLanguage, + }), onSuccess: (response, { pages }) => { - const stringMap = mapToMap( - listToGroupList( - response.strings?.map(({ value, page_name, ...otherArgs }) => { - // NOTE: removing empty translations or translations without pages - if (isFalsyString(value) || isFalsyString(page_name)) { - return undefined; - } - return { - value, - page_name, - ...otherArgs, - }; - }).filter(isDefined), - ({ page_name }) => page_name ?? 'common', - ), - (key) => key, - (values) => ( - listToMap( - values, - ({ key }) => key, - ({ value }) => value, - ) - ), - ); + if (!response?.ok || isNotDefined(response.data?.strings)) { + return; + } - setStrings( - (prevValue) => { - const namespaces = Object.keys(prevValue); + const translationsByPage = response.data.strings; - return { + setStrings( + (prevStrings) => { + const newStrings = { + ...prevStrings, ...listToMap( - namespaces, - (namespace) => namespace, - (namespace) => ({ - ...prevValue[namespace], - ...stringMap?.[namespace], - }), + pages, + (page) => page, + (page) => { + const existingTranslations = prevStrings[page]; + const newTranslations = translationsByPage[page] ?? {}; + + return { + ...existingTranslations, + ...newTranslations, + }; + }, ), }; + + return newStrings; }, ); + setLanguageNamespaceStatus( (prevValue) => ({ ...prevValue, @@ -156,6 +144,7 @@ export function Component() { ), }), ); + setLanguagePending(false); }, onFailure: (err, { pages }) => { @@ -177,18 +166,13 @@ export function Component() { }, }); - const queuedLanguages = useMemo( - () => { - const languages = mapToList( - languageNamespaceStatus, - (item, key) => ({ key, status: item }), - ); - return languages - .filter((item) => item.status === 'queued') - .map((item) => item.key) - .sort() - .join(','); - }, + const queuedNamespaces = useMemo( + () => mapToList( + languageNamespaceStatus, + (item, key) => ({ key, status: item }), + ).filter((item) => item.status === 'queued') + .map((item) => item.key) + .sort(), [languageNamespaceStatus], ); @@ -197,22 +181,21 @@ export function Component() { if ( languagePending || currentLanguage === 'en' - || isFalsyString(queuedLanguages) + || isNotDefined(queuedNamespaces) + || queuedNamespaces.length === 0 ) { return undefined; } languageRequestTimeoutRef.current = window.setTimeout( () => { - const keys = queuedLanguages.split(','); - unstable_batchedUpdates(() => { // FIXME: check if the component is still mounted setLanguageNamespaceStatus( (prevState) => ({ ...prevState, ...listToMap( - keys, + queuedNamespaces, (key) => key, () => 'pending', ), @@ -221,7 +204,7 @@ export function Component() { setLanguagePending(true); }); - fetchLanguage({ pages: keys }); + fetchLanguage({ pages: queuedNamespaces }); }, // FIXME: use constant 200, @@ -232,7 +215,7 @@ export function Component() { }; }, [ - queuedLanguages, + queuedNamespaces, languagePending, currentLanguage, fetchLanguage, diff --git a/nginx-serve/Dockerfile b/nginx-serve/Dockerfile index 8c095eaaf7..7b9ff2eb7b 100644 --- a/nginx-serve/Dockerfile +++ b/nginx-serve/Dockerfile @@ -27,6 +27,7 @@ ENV APP_ENVIRONMENT=APP_ENVIRONMENT_PLACEHOLDER ENV APP_MAPBOX_ACCESS_TOKEN=APP_MAPBOX_ACCESS_TOKEN_PLACEHOLDER ENV APP_TINY_API_KEY=APP_TINY_API_KEY_PLACEHOLDER ENV APP_API_ENDPOINT=https://APP-API-ENDPOINT-PLACEHOLDER.COM/ +ENV APP_TRANSLATION_API_ENDPOINT=https://APP-TRANSLATION-API-ENDPOINT-PLACEHOLDER.COM/ ENV APP_RISK_API_ENDPOINT=https://APP-RISK-API-ENDPOINT-PLACEHOLDER.COM/ ENV APP_SDT_URL=https://APP-SDT-URL-PLACEHOLDER.COM/ ENV APP_SENTRY_DSN=https://APP-SENTRY-DSN-PLACEHOLDER.COM/ diff --git a/nginx-serve/apply-config.sh b/nginx-serve/apply-config.sh index 1361d3390e..1e020def6e 100755 --- a/nginx-serve/apply-config.sh +++ b/nginx-serve/apply-config.sh @@ -30,6 +30,7 @@ find "$DESTINATION_DIRECTORY" -type f -exec sed -i "s|\|$APP_TINY_API_KEY|g" {} + # NOTE: We don't need a word boundary at end as we already have a trailing slash find "$DESTINATION_DIRECTORY" -type f -exec sed -i "s|\=0.4.0'} hasBin: true + adler-32@1.3.1: + resolution: {integrity: sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==} + engines: {node: '>=0.8'} + aggregate-error@3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} engines: {node: '>=8'} @@ -2982,6 +2989,10 @@ packages: caseless@0.12.0: resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} + cfb@1.2.2: + resolution: {integrity: sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==} + engines: {node: '>=0.8'} + chai@4.5.0: resolution: {integrity: sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==} engines: {node: '>=4'} @@ -3085,6 +3096,10 @@ packages: resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} engines: {node: '>=0.8'} + codepage@1.15.0: + resolution: {integrity: sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==} + engines: {node: '>=0.8'} + color-convert@1.9.3: resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} @@ -3841,6 +3856,10 @@ packages: resolution: {integrity: sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==} engines: {node: '>= 6'} + frac@1.1.2: + resolution: {integrity: sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==} + engines: {node: '>=0.8'} + fraction.js@4.3.7: resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} @@ -5788,6 +5807,10 @@ packages: sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + ssf@0.11.2: + resolution: {integrity: sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==} + engines: {node: '>=0.8'} + sshpk@1.18.0: resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==} engines: {node: '>=0.10.0'} @@ -6540,10 +6563,18 @@ packages: window-post-message-proxy@0.2.9: resolution: {integrity: sha512-hHmF5dvY27wy4EKN9c5qukPtzlbrdUzkMiCHud4gYKXCFAiGOBhCfi/dVBvwbUf0qrEGwFNnqkvk6DE54sdlcw==} + wmf@1.0.2: + resolution: {integrity: sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==} + engines: {node: '>=0.8'} + word-wrap@1.2.5: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + word@0.3.0: + resolution: {integrity: sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==} + engines: {node: '>=0.8'} + wordwrap@1.0.0: resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} @@ -6590,6 +6621,11 @@ packages: utf-8-validate: optional: true + xlsx@0.18.5: + resolution: {integrity: sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==} + engines: {node: '>=0.8'} + hasBin: true + xml-name-validator@3.0.0: resolution: {integrity: sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==} @@ -8709,6 +8745,8 @@ snapshots: acorn@8.14.0: {} + adler-32@1.3.1: {} + aggregate-error@3.1.0: dependencies: clean-stack: 2.2.0 @@ -9589,6 +9627,11 @@ snapshots: caseless@0.12.0: {} + cfb@1.2.2: + dependencies: + adler-32: 1.3.1 + crc-32: 1.2.2 + chai@4.5.0: dependencies: assertion-error: 1.1.0 @@ -9706,6 +9749,8 @@ snapshots: clone@1.0.4: optional: true + codepage@1.15.0: {} + color-convert@1.9.3: dependencies: color-name: 1.1.3 @@ -10242,7 +10287,7 @@ snapshots: dependencies: confusing-browser-globals: 1.0.11 eslint: 9.20.1(jiti@2.5.1) - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1(jiti@2.5.1))(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.2)(eslint@9.20.1(jiti@2.5.1)) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.26.0(eslint@9.20.1(jiti@2.5.1))(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.2)(eslint@9.20.1(jiti@2.5.1)) object.assign: 4.1.7 object.entries: 1.1.8 semver: 6.3.1 @@ -10251,7 +10296,7 @@ snapshots: dependencies: eslint: 9.20.1(jiti@2.5.1) eslint-config-airbnb-base: 15.0.0(eslint-plugin-import@2.31.0)(eslint@9.20.1(jiti@2.5.1)) - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1(jiti@2.5.1))(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.2)(eslint@9.20.1(jiti@2.5.1)) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.26.0(eslint@9.20.1(jiti@2.5.1))(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.2)(eslint@9.20.1(jiti@2.5.1)) eslint-plugin-jsx-a11y: 6.10.2(eslint@9.20.1(jiti@2.5.1)) eslint-plugin-react: 7.37.4(eslint@9.20.1(jiti@2.5.1)) eslint-plugin-react-hooks: 5.1.0(eslint@9.20.1(jiti@2.5.1)) @@ -10277,7 +10322,7 @@ snapshots: stable-hash: 0.0.4 tinyglobby: 0.2.12 optionalDependencies: - eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.1(eslint@9.20.1(jiti@2.5.1))(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.2)(eslint@9.20.1(jiti@2.5.1)) + eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.26.0(eslint@9.20.1(jiti@2.5.1))(typescript@5.7.3))(eslint-import-resolver-typescript@3.8.2)(eslint@9.20.1(jiti@2.5.1)) transitivePeerDependencies: - supports-color @@ -10662,6 +10707,8 @@ snapshots: es-set-tostringtag: 2.1.0 mime-types: 2.1.35 + frac@1.1.2: {} + fraction.js@4.3.7: {} fs-constants@1.0.0: {} @@ -12747,6 +12794,10 @@ snapshots: sprintf-js@1.0.3: {} + ssf@0.11.2: + dependencies: + frac: 1.1.2 + sshpk@1.18.0: dependencies: asn1: 0.2.6 @@ -13623,8 +13674,12 @@ snapshots: dependencies: es6-promise: 3.3.1 + wmf@1.0.2: {} + word-wrap@1.2.5: {} + word@0.3.0: {} + wordwrap@1.0.0: {} wrap-ansi@7.0.0: @@ -13657,6 +13712,16 @@ snapshots: ws@8.18.0: {} + xlsx@0.18.5: + dependencies: + adler-32: 1.3.1 + cfb: 1.2.2 + codepage: 1.15.0 + crc-32: 1.2.2 + ssf: 0.11.2 + wmf: 1.0.2 + word: 0.3.0 + xml-name-validator@3.0.0: optional: true