diff --git a/api/Dockerfile b/api/Dockerfile index 0bbc2c6..982d9fa 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -20,17 +20,17 @@ RUN apt-get update \ && apt-get -y install libpq-dev gcc \ && pip install psycopg2 +# Copy and install the transfer library wheel +COPY --from=transfer-builder /transfer-build/dist/*.whl /api/dist/ +RUN pip install --no-cache-dir /api/dist/pontoon-*.whl + # Build the API app COPY ./api/pyproject.toml /api/pyproject.toml COPY ./api/app /api/app COPY ./api/db /api/db RUN pip install build && python -m build --wheel -# Copy the transfer library wheel into dist -COPY --from=transfer-builder /transfer-build/dist/*.whl /api/dist/ - RUN pip install --no-cache-dir /api/dist/app-*.whl -RUN pip install --no-cache-dir /api/dist/pontoon-*.whl RUN pip install SQLAlchemy==2.0.41 EXPOSE 8000 diff --git a/api/app/models/transfer_run.py b/api/app/models/transfer_run.py index c6ba2d1..4aadd87 100644 --- a/api/app/models/transfer_run.py +++ b/api/app/models/transfer_run.py @@ -70,15 +70,34 @@ def get_latest_transfer_run(session, transfer_id:uuid.UUID, status:str = None) - @staticmethod - def list(session, destination_id:uuid.UUID, offset:int, limit:int) -> List[Model]: + def list(session, offset:int, limit:int, destination_id:uuid.UUID, execution_id:uuid.UUID = None) -> List[Model]: + """ + List TransferRuns with optional filtering by destination_id and/or execution_id. + + Args: + session: Database session + offset: Number of records to skip for pagination + limit: Maximum number of records to return + destination_id: Filter by destination ID + execution_id: Optional filter by execution_id from meta field + + Returns: + List[Model]: List of TransferRun models matching the filters + """ stmt = ( select(TransferRun.Model) .join(Transfer.Model, TransferRun.Model.transfer_id == Transfer.Model.transfer_id) .where(Transfer.Model.destination_id == destination_id) - .order_by(TransferRun.Model.created_at.desc()) - .offset(offset) - .limit(limit) ) + + # Apply execution_id filter if provided + if execution_id is not None: + stmt = stmt.where( + (TransferRun.Model.meta.is_not(None)) & + (TransferRun.Model.meta['execution_id'].as_string() == str(execution_id)) + ) + + stmt = stmt.order_by(TransferRun.Model.created_at.desc()).offset(offset).limit(limit) return session.exec(stmt).all() diff --git a/api/app/routers/destinations.py b/api/app/routers/destinations.py index 7e02ecf..816fc9f 100644 --- a/api/app/routers/destinations.py +++ b/api/app/routers/destinations.py @@ -1,8 +1,9 @@ import uuid import time -from typing import Annotated, List +from typing import Annotated, List, Optional, Literal from datetime import datetime, timezone from fastapi import HTTPException, Depends, Query, APIRouter, Security, status +from pydantic import BaseModel from app.models import Auth, Task, Destination, ScheduleModel, TransferRun, Transfer as TransferModel from app.routers.common import create_transfer_task, transfer_task_status @@ -29,6 +30,11 @@ def get_destination_by_id(session, destination_id:uuid.UUID): return Destination.get(session, destination_id) +class TransferScheduleOverride(BaseModel): + type: Literal["FULL_REFRESH", "INCREMENTAL"] + start: Optional[datetime] = None + end: Optional[datetime] = None + # # Transfer() management helpers # @@ -229,7 +235,12 @@ def create_destination_check(destination_id:uuid.UUID, session=Depends(get_sessi @router.post("/{destination_id}/run") -def run_destination_transfer(destination_id:uuid.UUID, session=Depends(get_session), auth:Auth = Security(get_auth)): +def run_destination_transfer( + destination_id: uuid.UUID, + schedule_override: Optional[TransferScheduleOverride] = None, + session=Depends(get_session), + auth:Auth = Security(get_auth) +): try: if settings.skip_transfers: return {"ok": True} @@ -248,17 +259,32 @@ def run_destination_transfer(destination_id:uuid.UUID, session=Depends(get_sessi run.set_organization(auth.org_uuid()) run.set_destination(str(destination_id)) - if schedule_model.type != 'FULL_REFRESH': - run.set_mode(Mode({ - 'type': schedule_model.type, - 'period': schedule_model.frequency, - 'start': now - Mode.delta(schedule_model.frequency), - 'end': now - })) + # Determine the mode based on request body or fall back to schedule defaults + if schedule_override: + if schedule_override.type == 'FULL_REFRESH': + run.set_mode(Mode({'type': 'FULL_REFRESH'})) + else: + end_time = schedule_override.end if schedule_override.end else now + start_time = schedule_override.start if schedule_override.start else (end_time - Mode.delta(schedule_model.frequency)) + run.set_mode(Mode({ + 'type': 'INCREMENTAL', + 'period': schedule_model.frequency, + 'start': start_time, + 'end': end_time + })) + else: + # Use schedule defaults + if schedule_model.type != 'FULL_REFRESH': + run.set_mode(Mode({ + 'type': schedule_model.type, + 'period': schedule_model.frequency, + 'start': now - Mode.delta(schedule_model.frequency), + 'end': now + })) run.run(expedited=False) - except (TransferException): + except TransferException as e: raise HTTPException(status_code=400, detail=str(e)) diff --git a/api/app/routers/transfers.py b/api/app/routers/transfers.py index 3432ce7..7de64dc 100644 --- a/api/app/routers/transfers.py +++ b/api/app/routers/transfers.py @@ -27,11 +27,12 @@ @router.get("", response_model=list[TransferRun.Model]) def search_transfers( destination_id: uuid.UUID, + execution_id: uuid.UUID = None, session = Depends(get_session), offset: int = 0, limit: Annotated[int, Query(le=100)] = 100, ): - return TransferRun.list(session, destination_id, offset, limit) + return TransferRun.list(session, offset=offset, limit=limit, destination_id=destination_id, execution_id=execution_id) @router.get("/{transfer_run_id}", response_model=TransferRun.Model) @@ -44,6 +45,16 @@ def get_transfer_run( raise HTTPException(status_code=404, detail="Transfer run not found") return transfer_run +@router.get("/transfer/{transfer_id}", response_model=TransferModel.Model) +def get_transfer( + transfer_id: uuid.UUID, + session = Depends(get_session), +): + transfer = TransferModel.get(session, transfer_id) + if not transfer: + raise HTTPException(status_code=404, detail="Transfer not found") + return transfer + @router.post("/{transfer_run_id}/rerun") def rerun_transfer( @@ -79,6 +90,9 @@ def rerun_transfer( raise TransferRun.Exception("Transfer run does not have arguments; can't re-run") args = meta.get('arguments', {}) + execution_id = meta.get('execution_id', None) + if execution_id is None: + raise TransferRun.Exception("Transfer run does not have an execution ID; can't re-run") # kick off a new transfer if settings.skip_transfers != True: @@ -88,6 +102,7 @@ def rerun_transfer( new_run.set_destination(str(destination.destination_id)) new_run.set_mode(Mode(args.get('mode', {}))) new_run.set_models(args.get('models', [])) + new_run.set_execution_id(execution_id) new_run.run(expedited=False) return {"ok": True} diff --git a/data-transfer/pontoon/pontoon/celery/tasks.py b/data-transfer/pontoon/pontoon/celery/tasks.py index 48a5a61..8612ccd 100644 --- a/data-transfer/pontoon/pontoon/celery/tasks.py +++ b/data-transfer/pontoon/pontoon/celery/tasks.py @@ -15,7 +15,9 @@ def transfer_task(self, args_json: str): print("Running transfer job as celery task...") try: args = json.loads(args_json).get('commandArgs', []) - args += ['--execution-id', str(self.request.id)] + if '--execution-id' not in args: + # Generate a new execution ID if one is not provided + args += ['--execution-id', str(self.request.id)] args += ['--retry-count', str(self.request.retries)] args += ['--retry-limit', str(TASK_MAX_RETRIES)] diff --git a/data-transfer/pontoon/pontoon/orchestration/client.py b/data-transfer/pontoon/pontoon/orchestration/client.py index 3f074a1..0a8e0f1 100644 --- a/data-transfer/pontoon/pontoon/orchestration/client.py +++ b/data-transfer/pontoon/pontoon/orchestration/client.py @@ -369,6 +369,10 @@ def set_models(self, model_ids:List[str]) -> 'Transfer': def set_organization(self, organization_id:str) -> 'Transfer': # set the org id for this transfer return self.set_argument('--organization-id', organization_id) + + def set_execution_id(self, execution_id:str) -> 'Transfer': + # set the execution id for this transfer + return self.set_argument('--execution-id', execution_id) def set_argument(self, arg_name:str, arg_value) -> 'Transfer': diff --git a/data-transfer/pontoon/pontoon/orchestration/transfer.py b/data-transfer/pontoon/pontoon/orchestration/transfer.py index 75a624f..190d1ef 100644 --- a/data-transfer/pontoon/pontoon/orchestration/transfer.py +++ b/data-transfer/pontoon/pontoon/orchestration/transfer.py @@ -219,7 +219,9 @@ def _schedule_to_replication_mode(self, schedule): schedule_hour = schedule.get('hour', 0) schedule_min = schedule.get('minute', 0) - end_time = self._now.replace(hour=schedule_hour, minute=schedule_min, microsecond=0) + end_time = self._now.replace(minute=schedule_min, second=0, microsecond=0) + if period != Mode.HOURLY: + end_time = end_time.replace(hour=schedule_hour) if period == Mode.WEEKLY: begin_time = end_time - timedelta(days=7, hours=12) diff --git a/docker-compose.yml b/docker-compose.yml index d72c045..eb5c520 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -94,6 +94,7 @@ services: - app-network depends_on: - redis + restart: unless-stopped volumes: postgres-data: diff --git a/test-env/mock_data_generator.py b/test-env/mock_data_generator.py index 9a9fb2c..7e8d402 100644 --- a/test-env/mock_data_generator.py +++ b/test-env/mock_data_generator.py @@ -191,11 +191,12 @@ def generate_and_insert_mock_data(): conn = get_db_connection() try: logger.info("Starting mock data generation...") + num_records_to_generate = 50 # Generate mock data - leads = generate_mock_leads(100) - campaigns = generate_mock_campaigns(100) - attributions = generate_mock_attribution(100) + leads = generate_mock_leads(num_records_to_generate) + campaigns = generate_mock_campaigns(num_records_to_generate) + attributions = generate_mock_attribution(num_records_to_generate) # Insert data insert_leads(conn, leads) @@ -217,8 +218,8 @@ def main(): while True: try: generate_and_insert_mock_data() - logger.info("Waiting 5 minutes before next generation...") - time.sleep(300) # 5 minutes + logger.info("Waiting 1 minute before next generation...") + time.sleep(60) # 1 minute except KeyboardInterrupt: logger.info("Mock data generator stopped by user") break diff --git a/web-app/pontoon/package-lock.json b/web-app/pontoon/package-lock.json index 805aca3..85ab2ac 100644 --- a/web-app/pontoon/package-lock.json +++ b/web-app/pontoon/package-lock.json @@ -15,6 +15,7 @@ "@mui/lab": "^7.0.0-beta.14", "@mui/material": "^7.2.0", "@mui/material-nextjs": "^7.2.0", + "@mui/x-date-pickers": "^8.10.2", "@tabler/icons-react": "^3.34.0", "dayjs": "^1.11.13", "formik": "^2.4.6", @@ -119,9 +120,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.27.6", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.6.tgz", - "integrity": "sha512-vbavdySgbTTrmFE+EsiqUTzlOr5bzlnJtUv9PynGCAKvfQqjIXbvFdumPM/GxMDfyuGMJaJAU6TO4zc1Jf1i8Q==", + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.3.tgz", + "integrity": "sha512-9uIQ10o0WGdpP6GDhXcdOJPJuDgFtIDtN/9+ArJQ2NAfAmiuhTQdzkaTGR33v43GYS2UrSA0eX2pPPHoFVvpxA==", "license": "MIT", "engines": { "node": ">=6.9.0" @@ -1283,12 +1284,12 @@ } }, "node_modules/@mui/types": { - "version": "7.4.4", - "resolved": "https://registry.npmjs.org/@mui/types/-/types-7.4.4.tgz", - "integrity": "sha512-p63yhbX52MO/ajXC7hDHJA5yjzJekvWD3q4YDLl1rSg+OXLczMYPvTuSuviPRCgRX8+E42RXz1D/dz9SxPSlWg==", + "version": "7.4.5", + "resolved": "https://registry.npmjs.org/@mui/types/-/types-7.4.5.tgz", + "integrity": "sha512-ZPwlAOE3e8C0piCKbaabwrqZbW4QvWz0uapVPWya7fYj6PeDkl5sSJmomT7wjOcZGPB48G/a6Ubidqreptxz4g==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.27.6" + "@babel/runtime": "^7.28.2" }, "peerDependencies": { "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0" @@ -1300,17 +1301,17 @@ } }, "node_modules/@mui/utils": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-7.2.0.tgz", - "integrity": "sha512-O0i1GQL6MDzhKdy9iAu5Yr0Sz1wZjROH1o3aoztuivdCXqEeQYnEjTDiRLGuFxI9zrUbTHBwobMyQH5sNtyacw==", + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-7.3.1.tgz", + "integrity": "sha512-/31y4wZqVWa0jzMnzo6JPjxwP6xXy4P3+iLbosFg/mJQowL1KIou0LC+lquWW60FKVbKz5ZUWBg2H3jausa0pw==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.27.6", - "@mui/types": "^7.4.4", + "@babel/runtime": "^7.28.2", + "@mui/types": "^7.4.5", "@types/prop-types": "^15.7.15", "clsx": "^2.1.1", "prop-types": "^15.8.1", - "react-is": "^19.1.0" + "react-is": "^19.1.1" }, "engines": { "node": ">=14.0.0" @@ -1330,11 +1331,99 @@ } }, "node_modules/@mui/utils/node_modules/react-is": { - "version": "19.1.0", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.1.0.tgz", - "integrity": "sha512-Oe56aUPnkHyyDxxkvqtd7KkdQP5uIUfHxd5XTb3wE9d/kRnZLmKbDB0GWk919tdQ+mxxPtG6EAs6RMT6i1qtHg==", + "version": "19.1.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.1.1.tgz", + "integrity": "sha512-tr41fA15Vn8p4X9ntI+yCyeGSf1TlYaY5vlTZfQmeLBrFo3psOPX6HhTDnFNL9uj3EhP0KAQ80cugCl4b4BERA==", "license": "MIT" }, + "node_modules/@mui/x-date-pickers": { + "version": "8.10.2", + "resolved": "https://registry.npmjs.org/@mui/x-date-pickers/-/x-date-pickers-8.10.2.tgz", + "integrity": "sha512-eS5t1jUojN/jL2FeJ8gtpCBxIEswUp9kLjM64aJ5LUKrNgM7X9dwsEHyplS+x07kWLiEAhO3nX3mepnS3Z43qg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.28.2", + "@mui/utils": "^7.3.1", + "@mui/x-internals": "8.10.2", + "@types/react-transition-group": "^4.4.12", + "clsx": "^2.1.1", + "prop-types": "^15.8.1", + "react-transition-group": "^4.4.5" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.9.0", + "@emotion/styled": "^11.8.1", + "@mui/material": "^5.15.14 || ^6.0.0 || ^7.0.0", + "@mui/system": "^5.15.14 || ^6.0.0 || ^7.0.0", + "date-fns": "^2.25.0 || ^3.2.0 || ^4.0.0", + "date-fns-jalali": "^2.13.0-0 || ^3.2.0-0 || ^4.0.0-0", + "dayjs": "^1.10.7", + "luxon": "^3.0.2", + "moment": "^2.29.4", + "moment-hijri": "^2.1.2 || ^3.0.0", + "moment-jalaali": "^0.7.4 || ^0.8.0 || ^0.9.0 || ^0.10.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + }, + "date-fns": { + "optional": true + }, + "date-fns-jalali": { + "optional": true + }, + "dayjs": { + "optional": true + }, + "luxon": { + "optional": true + }, + "moment": { + "optional": true + }, + "moment-hijri": { + "optional": true + }, + "moment-jalaali": { + "optional": true + } + } + }, + "node_modules/@mui/x-internals": { + "version": "8.10.2", + "resolved": "https://registry.npmjs.org/@mui/x-internals/-/x-internals-8.10.2.tgz", + "integrity": "sha512-dlC0BQRRBdiWtqn1yDppaHYRUjU3OuPWTxy0UtqxDaJjJf4pfR8ALr243nbxgJAFqvQyWPWyO4A6p9x9eJMJEQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.28.2", + "@mui/utils": "^7.3.1", + "reselect": "^5.1.1", + "use-sync-external-store": "^1.5.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "0.2.12", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", @@ -5364,6 +5453,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/reselect": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/reselect/-/reselect-5.1.1.tgz", + "integrity": "sha512-K/BG6eIky/SBpzfHZv/dd+9JBFiS4SWV7FIujVyJRux6e45+73RaUHXLmIR1f7WOMaQ0U1km6qwklRQxpJJY0w==", + "license": "MIT" + }, "node_modules/resolve": { "version": "1.22.10", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", diff --git a/web-app/pontoon/package.json b/web-app/pontoon/package.json index f25c93c..0450922 100644 --- a/web-app/pontoon/package.json +++ b/web-app/pontoon/package.json @@ -16,6 +16,7 @@ "@mui/lab": "^7.0.0-beta.14", "@mui/material": "^7.2.0", "@mui/material-nextjs": "^7.2.0", + "@mui/x-date-pickers": "^8.10.2", "@tabler/icons-react": "^3.34.0", "dayjs": "^1.11.13", "formik": "^2.4.6", diff --git a/web-app/pontoon/src/app/api/requests.js b/web-app/pontoon/src/app/api/requests.js index 2c0fd2b..99a215b 100644 --- a/web-app/pontoon/src/app/api/requests.js +++ b/web-app/pontoon/src/app/api/requests.js @@ -129,5 +129,25 @@ export async function rerunTransferRequest(key, { arg }) { } export async function runDestinationRequest(key, { arg }) { - return postRequest(`/destinations/${arg}/run`, { arg: {} }); + const { scheduleOverride } = arg; + + // Prepare the request body based on the schedule override + let requestBody = {}; + if (scheduleOverride && scheduleOverride.backfillType) { + requestBody.type = scheduleOverride.backfillType; + + if ( + scheduleOverride.backfillType === "INCREMENTAL" && + scheduleOverride.startTime && + scheduleOverride.endTime + ) { + // startTime and endTime are already ISO strings from the frontend + requestBody.start = scheduleOverride.startTime; + requestBody.end = scheduleOverride.endTime; + } + } + + return postRequest(key, { + arg: requestBody, + }); } diff --git a/web-app/pontoon/src/app/components/forms/FormRadioGroup.js b/web-app/pontoon/src/app/components/forms/FormRadioGroup.js new file mode 100644 index 0000000..7ee10c1 --- /dev/null +++ b/web-app/pontoon/src/app/components/forms/FormRadioGroup.js @@ -0,0 +1,43 @@ +import React from "react"; +import { + Typography, + FormControl, + FormControlLabel, + Radio, + RadioGroup, + FormHelperText, + Stack, +} from "@mui/material"; +import { useField } from "formik"; + +const FormRadioGroup = ({ titleText, helperText, options = [], ...props }) => { + const [field, meta] = useField(props); + return ( + + + {titleText} + + + + {options.map((option) => ( + } + label={option.label} + /> + ))} + + + {meta.touched && meta.error ? ( + {meta.error} + ) : null} + {helperText} + + ); +}; + +export default FormRadioGroup; diff --git a/web-app/pontoon/src/app/destinations/[id]/page.js b/web-app/pontoon/src/app/destinations/[id]/page.js index 87543c9..b6e3a23 100644 --- a/web-app/pontoon/src/app/destinations/[id]/page.js +++ b/web-app/pontoon/src/app/destinations/[id]/page.js @@ -14,10 +14,13 @@ import { IconButton, CircularProgress, LinearProgress, + RadioGroup, + FormControlLabel, + Radio, } from "@mui/material"; import Snackbar from "@mui/material/Snackbar"; import { IconDotsVertical } from "@tabler/icons-react"; -import useSWR from "swr"; +import useSWR, { mutate } from "swr"; import DashboardCard from "@/app//components/shared/DashboardCard"; import { ChevronLeft } from "@mui/icons-material"; import DeleteIcon from "@mui/icons-material/Delete"; @@ -44,6 +47,7 @@ import Duration from "dayjs/plugin/duration"; import RelativeTime from "dayjs/plugin/relativeTime"; import timezone from "dayjs/plugin/timezone"; import advancedFormat from "dayjs/plugin/advancedFormat"; +import utc from "dayjs/plugin/utc"; import TableBodyWrapper from "@/app/components/shared/TableBodyWrapper"; import { getScheduleText, getNextRunTime } from "@/utils/common"; import { @@ -52,12 +56,19 @@ import { rerunTransferRequest, runDestinationRequest, } from "@/app/api/requests"; +import { Form, Formik } from "formik"; +import * as Yup from "yup"; +import FormRadioGroup from "@/app/components/forms/FormRadioGroup"; +import { DateTimePicker } from "@mui/x-date-pickers/DateTimePicker"; +import { LocalizationProvider } from "@mui/x-date-pickers/LocalizationProvider"; +import { AdapterDayjs } from "@mui/x-date-pickers/AdapterDayjs"; dayjs.extend(LocalizedFormat); dayjs.extend(RelativeTime); dayjs.extend(Duration); dayjs.extend(timezone); dayjs.extend(advancedFormat); +dayjs.extend(utc); const getDataForTable = (destinationData, recipientData, modelsData) => { const destination = destinationData; @@ -176,6 +187,8 @@ const DestinationDetails = () => { ); const router = useRouter(); const [tab, setTab] = useState("1"); + const [openSuccess, setOpenSuccess] = useState(false); + const handleTabChange = (event, newValue) => { setTab(newValue); }; @@ -209,20 +222,41 @@ const DestinationDetails = () => { } > + setOpenSuccess(false)} + anchorOrigin={{ vertical: "top", horizontal: "center" }} + autoHideDuration={6000} + severity="success" + > + setOpenSuccess(false)} + severity="success" + variant="filled" + > + Transfer started + + + - + + + + + + + )} + + + ); +}; + +const TransferTable = ({ schedule, id, setOpenSuccess }) => { const theme = useTheme(); const router = useRouter(); - const [openSuccess, setOpenSuccess] = useState(false); - const { data: transfers, error: transfersError, isLoading: transfersLoading, isValidating: transfersValidating, mutate: mutateTransfers, - } = useSWR(`/transfers?destination_id=${id}`, getRequest); + } = useSWR(`/transfers?destination_id=${id}`, getRequest, { + refreshInterval: 3000, + }); const { trigger: triggerRerunTransfer } = useSWRMutation( `/transfers/:id/rerun`, rerunTransferRequest ); - const { trigger: triggerRunDestination } = useSWRMutation( - `/destinations/:id/run`, - runDestinationRequest - ); - - const refreshTransfers = () => { - mutateTransfers(); - }; - - const autoRefresh = () => { - const intervalId = setInterval(refreshTransfers, 3000); - // Stop after 5 min - setTimeout(() => { - clearInterval(intervalId); - }, 60000 * 5); - }; - const rerunTransfer = async (transferRunId) => { triggerRerunTransfer(transferRunId); setOpenSuccess(true); - autoRefresh(); - }; - - const runDestination = async (destinationId) => { - triggerRunDestination(destinationId); - setOpenSuccess(true); - autoRefresh(); + mutateTransfers(); }; const flattenTransferRuns = (transfers) => { @@ -311,52 +459,36 @@ const TransferTable = ({ schedule, id }) => { }); // Process each execution group - for (const transfers of executionsMap.values()) { - const retryMaxAttempts = transfers[0].meta.retry_max_attempts; - const statuses = transfers.map((t) => t.status); - - // Prioritize running transfers - const runningTransfer = transfers.find((t) => t.status === "RUNNING"); - if (runningTransfer) { - flatTransfers.push(runningTransfer); - continue; - } - - const successTransfer = transfers.find((t) => t.status === "SUCCESS"); - const latestTransfer = transfers[transfers.length - 1]; - - if (transfers.length === retryMaxAttempts) { - // All attempts used – show success if available, otherwise show latest failure - flatTransfers.push(successTransfer || latestTransfer); - } else if (!successTransfer) { - // Still retrying – mark latest attempt as retrying - flatTransfers.push({ ...latestTransfer, status: "RETRYING" }); - } else { - // Successfully completed - flatTransfers.push(successTransfer); - } + for (const transferFromExecutionGroup of executionsMap.values()) { + // Get the most recent transfer from the execution group + const mostRecentTransfer = transferFromExecutionGroup.reduce( + (latest, item) => { + if (!latest) return item; + return new Date(item.modified_at) > new Date(latest.modified_at) + ? item + : latest; + }, + null + ); + + flatTransfers.push(mostRecentTransfer); } return flatTransfers; }; + const flatTransfers = transfers ? flattenTransferRuns(transfers) : []; + + if (transfersError) { + return Error with API; + } + + if (transfersLoading) { + return ; + } + return ( - setOpenSuccess(false)} - anchorOrigin={{ vertical: "top", horizontal: "center" }} - autoHideDuration={6000} - severity="success" - > - setOpenSuccess(false)} - severity="success" - variant="filled" - > - Transfer started - - { Duration + + + Data Transfer Interval + + { numRows={4} numColumns={3} > - {(() => { - if (!transfers) return; - - const flatTransfers = flattenTransferRuns(transfers); - - return flatTransfers.map((transfer, idx) => ( + {flatTransfers && + flatTransfers.map((transfer, idx) => ( { {transfer.created_at - ? dayjs(transfer.created_at).format("LLL z").toString() + ? dayjs(transfer.created_at) + .format("MMM D, h:mm A z") + .toString() : ""} {transfer.scheduled_at ? dayjs(transfer.scheduled_at) - .format("LLLL z") + .format("MMM D, h:mm A z") .toString() : ""} @@ -481,7 +616,9 @@ const TransferTable = ({ schedule, id }) => { {transfer.status == "SUCCESS" || transfer.status == "FAILURE" - ? dayjs(transfer.modified_at).format("LTS").toString() + ? dayjs(transfer.modified_at) + .format("MMM D, h:mm A z") + .toString() : ""} @@ -512,6 +649,25 @@ const TransferTable = ({ schedule, id }) => { + + + {transfer?.meta?.arguments?.mode?.type === "FULL_REFRESH" + ? `Full Refresh at ${dayjs(transfer.created_at) + .format("MMM D, h:mm A z") + .toString()}` + : ""} + {transfer?.meta?.arguments?.mode?.type === "INCREMENTAL" + ? `${dayjs(transfer.meta.arguments.mode.start) + .format("MMM D, h:mm A z") + .toString()} - ${dayjs( + transfer.meta.arguments.mode.end + ) + .format("MMM D, h:mm A z") + .toString()}` + : ""} + + + {(transfer.status == "SUCCESS" || transfer.status == "FAILURE") && ( @@ -527,29 +683,9 @@ const TransferTable = ({ schedule, id }) => { Re-run )} - - {transfer.status == "SCHEDULED" && ( - - )} - - - - - - )); - })()} + ))}
diff --git a/web-app/pontoon/src/app/transfers/[id]/page.js b/web-app/pontoon/src/app/transfers/[id]/page.js index 9d50a29..00430da 100644 --- a/web-app/pontoon/src/app/transfers/[id]/page.js +++ b/web-app/pontoon/src/app/transfers/[id]/page.js @@ -7,6 +7,12 @@ import { Box, CircularProgress, LinearProgress, + Table, + TableBody, + TableCell, + TableHead, + TableRow, + useTheme, } from "@mui/material"; import { ChevronLeft } from "@mui/icons-material"; import useSWR from "swr"; @@ -19,23 +25,61 @@ import dayjs from "dayjs"; import LocalizedFormat from "dayjs/plugin/localizedFormat"; import timezone from "dayjs/plugin/timezone"; import advancedFormat from "dayjs/plugin/advancedFormat"; +import Duration from "dayjs/plugin/duration"; +import RelativeTime from "dayjs/plugin/relativeTime"; dayjs.extend(LocalizedFormat); dayjs.extend(timezone); dayjs.extend(advancedFormat); +dayjs.extend(Duration); +dayjs.extend(RelativeTime); const TransferDetails = () => { const params = useParams(); const { id } = params; const router = useRouter(); - + const theme = useTheme(); const { data: transferRun, error: transferRunError, isLoading: transferRunLoading, } = useSWR(`/transfers/${id}`, getRequest); + const transfer_id = transferRun?.transfer_id; + const execution_id = transferRun?.meta?.execution_id; + + const { + data: transfer, + error: transferError, + isLoading: transferLoading, + } = useSWR( + transfer_id ? ["/transfers/transfer", transfer_id] : null, + ([url, transfer_id]) => getRequest(`${url}/${transfer_id}`) + ); + const destination_id = transfer?.destination_id; - if (transferRunError) { + const { + data: transferRunsWithSameExecutionId, + error: transferRunsWithSameExecutionIdError, + isLoading: transferRunsWithSameExecutionIdLoading, + } = useSWR( + destination_id && execution_id + ? ["/transfers", destination_id, execution_id] + : null, + ([url, destination_id, execution_id]) => + getRequest( + `${url}?destination_id=${destination_id}&execution_id=${execution_id}` + ) + ); + const otherTransferRunsWithSameExecutionId = transferRunsWithSameExecutionId + ?.filter((run) => run.transfer_run_id !== transferRun.transfer_run_id) + .sort((a, b) => dayjs(b.created_at).diff(dayjs(a.created_at))); + const hasOtherRuns = otherTransferRunsWithSameExecutionId?.length > 0; + + if ( + transferRunError || + transferError || + transferRunsWithSameExecutionIdError + ) { return ( @@ -45,7 +89,11 @@ const TransferDetails = () => { ); } - if (transferRunLoading) { + if ( + transferRunLoading || + transferLoading || + transferRunsWithSameExecutionIdLoading + ) { return ( @@ -53,7 +101,7 @@ const TransferDetails = () => { ); } - if (!transferRun) { + if (!transferRun || !transfer) { return ( Transfer not found @@ -61,18 +109,58 @@ const TransferDetails = () => { ); } + const getStatus = (status) => { + if (status.toLowerCase().includes("success")) { + return "Success ✅"; + } + if (status.toLowerCase().includes("failed")) { + return "Failed ❌"; + } + if (status.toLowerCase().includes("running")) { + return "Running ⏳"; + } + return status; + }; + + const getDataTransferInterval = (transferMode) => { + if (transferMode?.type === "FULL_REFRESH") { + return `Full Refresh at ${dayjs(transferRun.created_at) + .format("MMM D, h:mm:ss A z") + .toString()}`; + } else if (transferMode?.type === "INCREMENTAL") { + return `${dayjs(transferMode?.start).format( + "MMM D, h:mm:ss A z" + )} - ${dayjs(transferMode?.end).format("MMM D, h:mm:ss A z")}`; + } + return "N/A"; + }; + const dataTransferInterval = getDataTransferInterval( + transferRun?.meta?.arguments?.mode + ); + const dataForTable = [ + ["Status", getStatus(transferRun.status)], + [ + "Duration", + dayjs + .duration(dayjs(transferRun.modified_at).diff(transferRun.created_at)) + .humanize(), + ], + [ + "Transfer Run Start Time", + dayjs(transferRun.created_at).format("MMM D, h:mm:ss A z").toString(), + ], + [ + "Transfer Run End Time", + dayjs(transferRun.modified_at).format("MMM D, h:mm:ss A z").toString(), + ], + ["Data Transfer Interval", dataTransferInterval], ["Transfer Run ID", transferRun.transfer_run_id], - ["Transfer ID", transferRun.transfer_id], - ["Status", transferRun.status], - ["Created At", dayjs(transferRun.created_at).format("LLL z").toString()], - ["Modified At", dayjs(transferRun.modified_at).format("LLL z").toString()], ]; return ( { )} + + {hasOtherRuns && otherTransferRunsWithSameExecutionId && ( + + + Other Runs from this Execution Group + + + + + + + Transfer Run ID + + + + + Status + + + + + Transfer Started At + + + + + + + {otherTransferRunsWithSameExecutionId.map((run, idx) => ( + { + router.push(`/transfers/${run.transfer_run_id}`); + }} + > + + + {run.transfer_run_id} + + + + + {getStatus(run.status)} + + + + + {dayjs(run.created_at).format("LLL z").toString()} + + + + ))} + +
+ + )} ); diff --git a/web-app/pontoon/src/utils/common.js b/web-app/pontoon/src/utils/common.js index 9efc1a4..0fa27b4 100644 --- a/web-app/pontoon/src/utils/common.js +++ b/web-app/pontoon/src/utils/common.js @@ -100,6 +100,18 @@ export const getScheduleText = (schedule) => { }; export const getNextRunTime = (schedule) => { + if (schedule.frequency === "HOURLY") { + const now = new Date(); + const nextHour = new Date(now); + + // Increment the hour by 1 + nextHour.setHours(now.getHours() + 1); + + // Zero out minutes, seconds, milliseconds + nextHour.setMinutes(0, 0, 0); + return nextHour.toISOString(); + } + // Set default values for day, hour, and minute const day = schedule.day ?? 0; // Default to Sunday if not provided const hour = schedule.hour ?? 0; // Default to 0 if not provided