Skip to content

Feature/check differences additional columns #67

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
deeonwuli wants to merge 3 commits into
base: development
Choose a base branch
from
Open

Feature/check differences additional columns #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bf22dc3
last submitted, last updated and categoryOption columns
deeonwuli Oct 26, 2022
9551c46
update to names in sql view
deeonwuli Oct 31, 2022
0bb5188
catoptioncombo column
deeonwuli Nov 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 65 additions & 54 deletions src/data/reports/mal-data-approval/MalDataApprovalDefaultRepository.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,7 @@ import {
MalDataApprovalOptions,
MalDataApprovalRepository,
} from "../../../domain/reports/mal-data-approval/repositories/MalDataApprovalRepository";
import {
DataDiffItem,
DataDiffItemIdentifier
} from "../../../domain/reports/mal-data-approval/entities/DataDiffItem";
import { DataDiffItem, DataDiffItemIdentifier } from "../../../domain/reports/mal-data-approval/entities/DataDiffItem";
import { Namespaces } from "../../common/clients/storage/Namespaces";

export interface Pagination {
Expand Down Expand Up @@ -120,7 +117,12 @@ type SqlFieldDiff =
| "dataelement"
| "apvddataelement"
| "comment"
| "apvdcomment";
| "apvdcomment"
| "lastupdated"
| "apvdlastupdated"
| "apvdstoredby"
| "storedby"
| "categoryoptioncomboname";

type SqlField =
| "datasetuid"
Expand Down Expand Up @@ -196,6 +198,11 @@ export class MalDataApprovalDefaultRepository implements MalDataApprovalReposito
apvdDataElement: item.apvddataelement,
comment: item.comment,
apvdComment: item.apvdcomment,
lastUpdated: item.lastupdated,
lastUpdatedBy: item.storedby,
lastSubmissionDate: item.apvdlastupdated,
lastSubmittedBy: item.apvdstoredby,
categoryOption: item.categoryoptioncomboname,
})
);

Expand Down Expand Up @@ -466,44 +473,44 @@ export class MalDataApprovalDefaultRepository implements MalDataApprovalReposito
async duplicateDataValues(dataValues: DataDiffItemIdentifier[]): Promise<boolean> {
try {
const approvalDataSetId = process.env.REACT_APP_APPROVE_DATASET_ID ?? "fRrt4V8ImqD";
const uniqueDataSets = _.uniqBy(dataValues, 'dataSet');
const uniqueDataElementsNames = _.uniq(_.map(dataValues, 'dataElement'));
const uniqueDataSets = _.uniqBy(dataValues, "dataSet");
const uniqueDataElementsNames = _.uniq(_.map(dataValues, "dataElement"));

const DSDataElements: { dataSetElements: dataSetElementsType[] }[] = await promiseMap(uniqueDataSets, async item =>
this.api
.get<any>(`/dataSets/${item.dataSet}`, { fields: "dataSetElements[dataElement[id,name]]" })
.getData()
const DSDataElements: { dataSetElements: dataSetElementsType[] }[] = await promiseMap(
uniqueDataSets,
async item =>
this.api
.get<any>(`/dataSets/${item.dataSet}`, { fields: "dataSetElements[dataElement[id,name]]" })
.getData()
);

const dataValueSets: dataSetsValueType[] = await promiseMap(uniqueDataSets, async item =>
this.api.get<any>("/dataValueSets", {
dataSet: item.dataSet,
period: item.period,
orgUnit: item.orgUnit,
}).getData()
this.api
.get<any>("/dataValueSets", {
dataSet: item.dataSet,
period: item.period,
orgUnit: item.orgUnit,
})
.getData()
);

const ADSDataElementsRaw: { dataSetElements: dataSetElementsType[] } = await this.api
.get<any>(`/dataSets/${approvalDataSetId}`, { fields: "dataSetElements[dataElement[id,name]]" })
.getData();

const ADSDataElements: dataElementsType[] = ADSDataElementsRaw.dataSetElements.map(
(element) => {
return {
id: element.dataElement.id,
name: element.dataElement.name,
};
}
);
const ADSDataElements: dataElementsType[] = ADSDataElementsRaw.dataSetElements.map(element => {
return {
id: element.dataElement.id,
name: element.dataElement.name,
};
});

const dataElementsMatchedArray: { [key: string]: any }[] = DSDataElements.flatMap(DSDataElement => {
return DSDataElement.dataSetElements.flatMap((element) => {
return DSDataElement.dataSetElements.flatMap(element => {
const dataElement = element.dataElement;
if (uniqueDataElementsNames.includes(dataElement.name)) {
const othername = dataElement.name + "-APVD";
const ADSDataElement = ADSDataElements.find(
(DataElement) => DataElement.name === othername
);
const ADSDataElement = ADSDataElements.find(DataElement => DataElement.name === othername);
return {
origId: dataElement?.id,
destId: ADSDataElement?.id,
Expand All @@ -513,24 +520,22 @@ export class MalDataApprovalDefaultRepository implements MalDataApprovalReposito
return [];
}
});
})
});

const apvdDataValues = dataValueSets.flatMap(dataValueSet => {
if (dataValueSet.dataValues) {
const data = dataValueSet.dataValues.flatMap(
(dataValue) => {
const data = { ...dataValue };
const destId = dataElementsMatchedArray.find(
dataElementsMatchedObj => dataElementsMatchedObj.origId === dataValue.dataElement
)?.destId;
data.dataElement = destId;
data.dataSet = approvalDataSetId;
delete data.lastUpdated;
delete data.comment;

return data.dataElement ? data : [];
}
);
const data = dataValueSet.dataValues.flatMap(dataValue => {
const data = { ...dataValue };
const destId = dataElementsMatchedArray.find(
dataElementsMatchedObj => dataElementsMatchedObj.origId === dataValue.dataElement
)?.destId;
data.dataElement = destId;
data.dataSet = approvalDataSetId;
delete data.lastUpdated;
delete data.comment;

return data.dataElement ? data : [];
});
return data;
} else {
return [];
Expand Down Expand Up @@ -558,7 +563,7 @@ export class MalDataApprovalDefaultRepository implements MalDataApprovalReposito
.post<any>("/dataValueSets.json", {}, { dataValues: _.reject(chunk, _.isEmpty) })
.getData();

copyResponse.push(response)
copyResponse.push(response);
}
return _.every(copyResponse, item => item.status === "SUCCESS");
} else {
Expand Down Expand Up @@ -586,7 +591,11 @@ export class MalDataApprovalDefaultRepository implements MalDataApprovalReposito
};

const revokeResponse = await this.api
.delete<any>("/dataApprovals", { ds: revokeData.dataSet, pe: revokeData.period, ou: revokeData.orgUnit })
.delete<any>("/dataApprovals", {
ds: revokeData.dataSet,
pe: revokeData.period,
ou: revokeData.orgUnit,
})
.getData();

return duplicateResponse && revokeResponse === "";
Expand Down Expand Up @@ -631,17 +640,19 @@ export class MalDataApprovalDefaultRepository implements MalDataApprovalReposito
try {
const response: any[] = [];
dataSets.forEach(async dataSet => {
const isApproved = await this.api.get<any>(
"/dataApprovals",
{ wf: dataSet.workflow, pe: dataSet.period, ou: dataSet.orgUnit }
).getData();
const isApproved = await this.api
.get<any>("/dataApprovals", { wf: dataSet.workflow, pe: dataSet.period, ou: dataSet.orgUnit })
.getData();

if (isApproved.state === "APPROVED_HERE") {
response.push(
await this.api.delete<any>(
"/dataApprovals",
{ wf: dataSet.workflow, pe: dataSet.period, ou: dataSet.orgUnit }
).getData()
await this.api
.delete<any>("/dataApprovals", {
wf: dataSet.workflow,
pe: dataSet.period,
ou: dataSet.orgUnit,
})
.getData()
);
}
});
Expand Down Expand Up @@ -727,7 +738,7 @@ type DataSetRow = Record<CsvField, string>;

function getISODate() {
const date = new Date().toISOString();
return date.slice(0, date.lastIndexOf(":"))
return date.slice(0, date.lastIndexOf(":"));
}

/* From the docs: "The variables must contain alphanumeric, dash, underscore and
Expand Down
32 changes: 27 additions & 5 deletions src/data/reports/mal-data-approval/sql-views/mal-data-approval-diff.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,41 @@ SELECT
'${periods}' as period,
sourceid,
periodid,
categoryoptioncomboid,
(select name from dataelement where dataelementid = datavalue.dataelementid) as dataelement,
datavalue.value as value,
datavalue.storedby as storedby,
datavalue.lastupdated as lastupdated,
datavalue.comment as comment,
(select name from categoryoptioncombo where categoryoptioncomboid = datavalue.categoryoptioncomboid) as categoryoptioncomboname,

(select dva.value from datavalue dva
where dva.dataelementid=(select dataelementid from dataelement where name = CONCAT((select name from dataelement where dataelementid = datavalue.dataelementid), '-APVD'))
where dva.dataelementid=(select dataelementid from dataelement where uid = UPPER((select uid from dataelement where dataelementid = datavalue.dataelementid)))
and dva.sourceid=datavalue.sourceid
and dva.attributeoptioncomboid= datavalue.attributeoptioncomboid
and dva.periodid=datavalue.periodid
and dva.deleted = false
and dva.categoryoptioncomboid = datavalue.categoryoptioncomboid
) as apvdvalue,

(select dva.lastupdated from datavalue dva
where dva.dataelementid=(select dataelementid from dataelement where uid = UPPER((select uid from dataelement where dataelementid = datavalue.dataelementid)))
and dva.sourceid=datavalue.sourceid
and dva.attributeoptioncomboid= datavalue.attributeoptioncomboid
and dva.periodid=datavalue.periodid
and dva.deleted = false
and dva.categoryoptioncomboid = datavalue.categoryoptioncomboid
) as apvdlastupdated,
(select dva.storedby from datavalue dva
where dva.dataelementid=(select dataelementid from dataelement where uid = UPPER((select uid from dataelement where dataelementid = datavalue.dataelementid)))
and dva.sourceid=datavalue.sourceid
and dva.attributeoptioncomboid= datavalue.attributeoptioncomboid
and dva.periodid=datavalue.periodid
and dva.deleted = false
and dva.categoryoptioncomboid = datavalue.categoryoptioncomboid
) as apvdstoredby,
(select dva.comment from datavalue dva
where dva.dataelementid=(select dataelementid from dataelement where name = CONCAT((select name from dataelement where dataelementid = datavalue.dataelementid), '-APVD'))
where dva.dataelementid=(select dataelementid from dataelement where uid = UPPER((select uid from dataelement where dataelementid = datavalue.dataelementid)))
and dva.sourceid=datavalue.sourceid
and dva.attributeoptioncomboid= datavalue.attributeoptioncomboid
and dva.periodid=datavalue.periodid
Expand All @@ -28,23 +49,24 @@ max(lastupdated) as lastupdated

FROM datavalue
WHERE
datavalue.dataelementid not in (select dataelementid from dataelement where valuetype = 'FILE_RESOURCE') and
datavalue.sourceid in (select organisationunitid from organisationunit where uid ~ ('^' || replace('${orgUnits}', '-', '|') || '$'))
and datavalue.periodid in (select periodid from _periodstructure where iso ~ ('^' || replace('${periods}', '-', '|') || '$'))
and datavalue.dataelementid in ( select dse.dataelementid from datasetelement dse where dse.datasetid = (select datasetid from dataset where uid ~ ('^' || replace('${dataSets}', '-', '|') || '$')) )
and datavalue.deleted = false
and datavalue.dataelementid not in (select dataelementid from dataelement where name='MAL - Submission date')
and (datavalue.value != (select dva.value from datavalue dva
where dva.dataelementid=(select dataelementid from dataelement where name = CONCAT((select name from dataelement where dataelementid = datavalue.dataelementid), '-APVD'))
where dva.dataelementid=(select dataelementid from dataelement where uid= UPPER((select uid from dataelement where dataelementid = datavalue.dataelementid)))
and dva.sourceid=datavalue.sourceid
and dva.attributeoptioncomboid= datavalue.attributeoptioncomboid
and dva.periodid=datavalue.periodid
and dva.deleted = false
and dva.categoryoptioncomboid = datavalue.categoryoptioncomboid)
or (datavalue.value is not null and ((select dva.value from datavalue dva
where dva.dataelementid=(select dataelementid from dataelement where name = CONCAT((select name from dataelement where dataelementid = datavalue.dataelementid), '-APVD'))
where dva.dataelementid=(select dataelementid from dataelement where uid = UPPER((select uid from dataelement where dataelementid = datavalue.dataelementid)))
and dva.sourceid=datavalue.sourceid
and dva.attributeoptioncomboid= datavalue.attributeoptioncomboid
and dva.periodid=datavalue.periodid
and dva.deleted = false
and dva.categoryoptioncomboid = datavalue.categoryoptioncomboid) is null)))
GROUP BY datavalue.periodid, datavalue.sourceid, datavalue.attributeoptioncomboid, datavalue.dataelementid, datavalue.categoryoptioncomboid
GROUP BY datavalue.periodid, datavalue.sourceid, datavalue.attributeoptioncomboid, datavalue.dataelementid, datavalue.categoryoptioncomboid
23 changes: 7 additions & 16 deletions src/domain/reports/mal-data-approval/entities/DataDiffItem.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ export interface DataDiffItem {
apvdDataElement: string | undefined;
comment: string | undefined;
apvdComment: string | undefined;
lastUpdated: string | undefined;
lastUpdatedBy: string | undefined;
lastSubmissionDate: string | undefined;
lastSubmittedBy: string | undefined;
categoryOption: string | undefined;
}

export interface DataDiffItemIdentifier {
Expand All @@ -20,25 +25,11 @@ export interface DataDiffItemIdentifier {
}

export function getDataDiffItemId(item: DataDiffItem): string {
return [
item.dataSetUid,
item.period,
item.orgUnitUid,
item.dataElement,
item.value,
item.comment,
].join("|||");
return [item.dataSetUid, item.period, item.orgUnitUid, item.dataElement, item.value, item.comment].join("|||");
}

export function parseDataDiffItemId(string: string): DataDiffItemIdentifier | undefined {
const [
dataSet,
period,
orgUnit,
dataElement,
value,
comment,
] = string.split("|||");
const [dataSet, period, orgUnit, dataElement, value, comment] = string.split("|||");

if (!dataSet || !period || !orgUnit || !dataElement || !value) return undefined;

Expand Down
10 changes: 10 additions & 0 deletions src/webapp/reports/mal-data-approval/DataDiffViewModel.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ export interface DataDiffViewModel {
apvdDataElement: string | undefined;
comment: string | undefined;
apvdComment: string | undefined;
lastUpdated: string | undefined;
lastUpdatedBy: string | undefined;
lastSubmissionDate: string | undefined;
lastSubmittedBy: string | undefined;
categoryOption: string | undefined;
}

export function getDataDiffViews(_config: Config, items: DataDiffItem[]): DataDiffViewModel[] {
Expand All @@ -27,6 +32,11 @@ export function getDataDiffViews(_config: Config, items: DataDiffItem[]): DataDi
apvdDataElement: item.apvdDataElement,
comment: item.comment,
apvdComment: item.apvdComment,
lastUpdated: item.lastUpdated,
lastUpdatedBy: item.lastUpdatedBy,
lastSubmissionDate: item.lastSubmissionDate,
lastSubmittedBy: item.lastSubmittedBy,
categoryOption: item.categoryOption,
};
});
}
12 changes: 11 additions & 1 deletion src/webapp/reports/mal-data-approval/DataDifferencesList.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,12 @@ interface DataDifferencesListProps {
isUpdated: () => void;
}

export const DataDifferencesList: React.FC<DataDifferencesListProps> = ({ selectedIds, revoke, isMalAdmin, isUpdated }) => {
export const DataDifferencesList: React.FC<DataDifferencesListProps> = ({
selectedIds,
revoke,
isMalAdmin,
isUpdated,
}) => {
const { compositionRoot, config } = useAppContext();
const [visibleColumns, setVisibleColumns] = useState<string[]>();
const snackbar = useSnackbar();
Expand All @@ -40,6 +45,11 @@ export const DataDifferencesList: React.FC<DataDifferencesListProps> = ({ select
{ name: "comment", text: i18n.t("Comment"), sortable: false },
{ name: "apvdValue", text: i18n.t("Approved value"), sortable: false },
{ name: "apvdComment", text: i18n.t("Approved value comment"), sortable: false },
{ name: "lastUpdated", text: i18n.t("Last updated"), sortable: false },
{ name: "lastUpdatedBy", text: i18n.t("Last updated by"), sortable: false },
{ name: "lastSubmissionDate", text: i18n.t("Last submission date"), sortable: false },
{ name: "lastSubmittedBy", text: i18n.t("Last submitted by"), sortable: false },
{ name: "categoryOption", text: i18n.t("Category Option"), sortable: false },
],
actions: [
{
Expand Down