[MLOPS-320] Optimization of generation grid column schemas process, change logic on front-end and back-end

.github/workflows/build-app-stack.yml

@@ -14,7 +14,7 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Build Docker containers using compose
-      run: docker-compose up -d
+      run: docker compose up -d
 
     - name: Wait for application to initialize
       run: sleep 10
@@ -23,5 +23,5 @@ jobs:
       run: curl -I http://localhost:3000/projects
 
     - name: Stop the containers
-      run: docker-compose down
+      run: docker compose down
 

client/src/components/experiments/iterations/iterations-container.tsx

@@ -48,6 +48,7 @@ import { Iteration } from "@/types/iteration";
 import NoIterationsInfo from "./no-iterations-info";
 import { useNavigate, useSearchParams } from "react-router-dom";
 import { useCookies } from "react-cookie";
+import { ColumnsMetadata } from "@/types/columns-metadata";
 
 interface IterationsContainerProps {
     projectData: Project;
@@ -105,12 +106,18 @@ const IterationsContainer = ({
             JSON.stringify(TreeSelectBaseColumnsChecked)
         );
 
+        const columns_metadata: ColumnsMetadata[] = activeExperiments.map(
+            (experiment) => experiment.columns_metadata
+        );
+
+        console.log(columns_metadata);
+
         let [
             gridColumnsParameters,
             TreeSelectBaseColumnsOptionsWithParams,
             TreeSelectBaseColumnsCheckedWithParams,
         ] = extractColumnsData(
-            rowData,
+            columns_metadata,
             "parameters",
             TreeSelectBaseColumnsOptionsAll,
             TreeSelectBaseColumnsCheckedAll
@@ -121,7 +128,7 @@ const IterationsContainer = ({
             TreeSelectBaseColumnsOptionsWithParamsAndMetrics,
             TreeSelectBaseColumnsCheckedWithParamsAndMetrics,
         ] = extractColumnsData(
-            rowData,
+            columns_metadata,
             "metrics",
             TreeSelectBaseColumnsOptionsWithParams,
             TreeSelectBaseColumnsCheckedWithParams

client/src/hooks/use-data-hook.ts

@@ -231,16 +231,56 @@ export const useData = create<DataStore>((set) => ({
 
                     if (experiment_index === -1) return state;
 
+                    let experiment: Experiment =
+                        state.projects[index].experiments[experiment_index];
+
                     state.projects[index].experiments[
                         experiment_index
                     ].iterations = state.projects[index].experiments[
                         experiment_index
-                    ].iterations.filter(
-                        (iteration) =>
-                            !iterationsToDelete[experiment_id].includes(
+                    ].iterations.filter((iteration) => {
+                        if (
+                            iterationsToDelete[experiment_id].includes(
                                 iteration.id
                             )
-                    );
+                        ) {
+                            const metrics = Object.keys(iteration.metrics);
+                            const parameters = Object.keys(
+                                iteration.parameters
+                            );
+
+                            for (const metric of metrics) {
+                                experiment.columns_metadata[metric][
+                                    "count"
+                                ]! -= 1;
+                                if (
+                                    experiment.columns_metadata[metric][
+                                        "count"
+                                    ] === 0
+                                ) {
+                                    delete experiment.columns_metadata[metric];
+                                }
+                            }
+
+                            for (const parameter of parameters) {
+                                experiment.columns_metadata[parameter][
+                                    "count"
+                                ]! -= 1;
+                                if (
+                                    experiment.columns_metadata[parameter][
+                                        "count"
+                                    ] === 0
+                                ) {
+                                    delete experiment.columns_metadata[
+                                        parameter
+                                    ];
+                                }
+                            }
+
+                            return false;
+                        }
+                        return true;
+                    });
                 }
 
                 return { projects: [...state.projects] };

client/src/lib/utils.ts

@@ -1,6 +1,5 @@
 import { Chart } from "@/types/chart";
 import { Dataset } from "@/types/dataset";
-import { Iteration } from "@/types/iteration";
 import { Model } from "@/types/model";
 import {
     BinMethod,
@@ -178,60 +177,57 @@ export const dateToHumanize = (date: string) => {
 };
 
 export const extractColumnsData = (
-    rowData: Iteration[],
+    columns_metadata: any[],
     type: "parameters" | "metrics",
     TreeSelectBaseColumnsOptionsAll: any,
     TreeSelectBaseColumnsCheckedAll: any
 ) => {
-    let columnsPerIterations = rowData.map(
-        (iteration) =>
-            new Set(
-                iteration[type]
-                    ? Object.getOwnPropertyNames(iteration[type])
-                    : ""
-            )
-    );
+    const columns: Set<string> = new Set();
+
+    const mapping = {
+        parameters: "parameter",
+        metrics: "metric",
+    };
 
-    let columnsUniqueSet = new Set<string>();
+    const metadataType = mapping[type];
 
-    columnsPerIterations.forEach((set) => {
-        set.forEach((value) => {
-            columnsUniqueSet.add(value);
+    columns_metadata.forEach((metadata) => {
+        Object.keys(metadata).forEach((key) => {
+            if (metadata[key].type === metadataType) {
+                columns.add(key);
+            }
         });
     });
 
-    let columnsUniqueArray = Array.from(columnsUniqueSet);
+    const columnsArray = Array.from(columns);
 
-    let gridColumns = [];
-    let treeselectColumns = [];
+    const gridColumns = [];
+    const treeselectColumns = [];
 
-    if (columnsUniqueArray.length > 0) {
+    if (columnsArray.length > 0) {
         Object.assign(TreeSelectBaseColumnsCheckedAll, {
             [type]: {
                 checked: true,
                 partialChecked: false,
             },
         });
 
-        for (let i = 0; i < columnsUniqueArray.length; i++) {
-            const key = `${type}.` + columnsUniqueArray[i];
+        for (let i = 0; i < columnsArray.length; i++) {
+            const key = `${type}.` + columnsArray[i];
             gridColumns.push({
                 field: key,
-                headerName: columnsUniqueArray[i],
+                headerName: columnsArray[i],
                 filter: "agNumberColumnFilter",
                 cellRenderer: (val: any) => {
-                    if (
-                        val.data[type] &&
-                        val.data[type][columnsUniqueArray[i]]
-                    ) {
-                        return val.data[type][columnsUniqueArray[i]];
+                    if (val.data[type] && val.data[type][columnsArray[i]]) {
+                        return val.data[type][columnsArray[i]];
                     }
                     return "-";
                 },
             });
             treeselectColumns.push({
                 key: key,
-                label: columnsUniqueArray[i],
+                label: columnsArray[i],
             });
             Object.assign(TreeSelectBaseColumnsCheckedAll, {
                 [key]: {