diff --git a/.github/workflows/build-app-stack.yml b/.github/workflows/build-app-stack.yml index 6153d6c..10c5734 100644 --- a/.github/workflows/build-app-stack.yml +++ b/.github/workflows/build-app-stack.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v3 - name: Build Docker containers using compose - run: docker-compose up -d + run: docker compose up -d - name: Wait for application to initialize run: sleep 10 @@ -23,5 +23,5 @@ jobs: run: curl -I http://localhost:3000/projects - name: Stop the containers - run: docker-compose down + run: docker compose down diff --git a/client/package-lock.json b/client/package-lock.json index c49e41d..c48e196 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -94,9 +94,9 @@ } }, "node_modules/@adobe/css-tools": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.3.1.tgz", - "integrity": "sha512-/62yikz7NLScCGAAST5SHdnjaDJQBDq0M2muyRTpf2VQhw6StBg2ALiu73zSJQ4fMVLA+0uBhBHAle7Wg+2kSg==" + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.4.0.tgz", + "integrity": "sha512-Ff9+ksdQQB3rMncgqDK78uLznstjyfIf2Arnh22pW8kBpLs6rpKDwgnZT46hin5Hl1WzazzK64DOrhSwYpS7bQ==" }, "node_modules/@alloc/quick-lru": { "version": "5.2.0", @@ -5397,19 +5397,19 @@ } }, "node_modules/ag-grid-community": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/ag-grid-community/-/ag-grid-community-30.2.0.tgz", - "integrity": "sha512-Gd6GXmtzEQSCDloBdRxxCDqnjTBRAOf/zzlaxxyyVBJgc+cePuNgGdplRUhT/rwIiDwvyuoynvxelVE/iYdXsA==" + "version": "30.2.1", + "resolved": "https://registry.npmjs.org/ag-grid-community/-/ag-grid-community-30.2.1.tgz", + "integrity": "sha512-1slonXskJbbI9ybhTx//4YKfJpZVAEnHL8dui1rQJRSXKByUi+/f7XtvkLsbgBkawoWbqvRAySjYtvz80+kBfA==" }, "node_modules/ag-grid-react": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/ag-grid-react/-/ag-grid-react-30.2.0.tgz", - "integrity": "sha512-y4esND0ADJMw/aCyfiT1GA885sy2XvlnXUNQdiXijoxcGY6OSk3jE2DPtZfE+RsdmNdXimMpoRZHwJW/aPCegA==", + "version": "30.2.1", + "resolved": "https://registry.npmjs.org/ag-grid-react/-/ag-grid-react-30.2.1.tgz", + "integrity": "sha512-WYt5ZstSoPEGAcTqXBdaonihXtapZdjTHZ3dc3xTK1xIdbF0/Vw4zDWCQSsG5H4M5CeUKjvbeHx7kKM1Yiah3g==", "dependencies": { "prop-types": "^15.8.1" }, "peerDependencies": { - "ag-grid-community": "~30.2.0", + "ag-grid-community": "~30.2.1", "react": "^16.3.0 || ^17.0.0 || ^18.0.0", "react-dom": "^16.3.0 || ^17.0.0 || ^18.0.0" } @@ -5601,11 +5601,11 @@ } }, "node_modules/axios": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz", - "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==", + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.4.tgz", + "integrity": "sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==", "dependencies": { - "follow-redirects": "^1.15.0", + "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } @@ -5633,11 +5633,11 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -6800,9 +6800,9 @@ "integrity": "sha512-P9bmyZ3h/PRG+Nzga+rbdI4OEpNDzAVyy74uVO9ATgzLK6VtAsYybF/+TOCvrc0MO793d6+42lLyZTw7/ArVzA==" }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -6866,9 +6866,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", @@ -8143,9 +8143,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", - "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", "funding": [ { "type": "github", @@ -8389,9 +8389,9 @@ } }, "node_modules/picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", + "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==" }, "node_modules/picomatch": { "version": "2.3.1", @@ -8431,9 +8431,9 @@ } }, "node_modules/postcss": { - "version": "8.4.29", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.29.tgz", - "integrity": "sha512-cbI+jaqIeu/VGqXEarWkRCCffhjgXc0qjBtXpqJhTBohMUjUQnbBr0xqX3vEKudc4iviTewcJo5ajcec5+wdJw==", + "version": "8.4.41", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.41.tgz", + "integrity": "sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ==", "funding": [ { "type": "opencollective", @@ -8449,9 +8449,9 @@ } ], "dependencies": { - "nanoid": "^3.3.6", - "picocolors": "^1.0.0", - "source-map-js": "^1.0.2" + "nanoid": "^3.3.7", + "picocolors": "^1.0.1", + "source-map-js": "^1.2.0" }, "engines": { "node": "^10 || ^12 || >=14" @@ -9313,9 +9313,9 @@ } }, "node_modules/source-map-js": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", - "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", + "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", "engines": { "node": ">=0.10.0" } @@ -9863,9 +9863,9 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, "node_modules/vite": { - "version": "4.4.9", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz", - "integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==", + "version": "4.5.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.3.tgz", + "integrity": "sha512-kQL23kMeX92v3ph7IauVkXkikdDRsYMGTVl5KY2E9OY4ONLvkHf04MDTbnfo6NKxZiDLWzVpP5oTa8hQD8U3dg==", "dependencies": { "esbuild": "^0.18.10", "postcss": "^8.4.27", @@ -10147,9 +10147,9 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/ws": { - "version": "8.14.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz", - "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "engines": { "node": ">=10.0.0" }, diff --git a/client/src/components/experiments/iterations/iterations-container.tsx b/client/src/components/experiments/iterations/iterations-container.tsx index 17e4e5c..995551c 100644 --- a/client/src/components/experiments/iterations/iterations-container.tsx +++ b/client/src/components/experiments/iterations/iterations-container.tsx @@ -48,6 +48,7 @@ import { Iteration } from "@/types/iteration"; import NoIterationsInfo from "./no-iterations-info"; import { useNavigate, useSearchParams } from "react-router-dom"; import { useCookies } from "react-cookie"; +import { ColumnsMetadata } from "@/types/columns-metadata"; interface IterationsContainerProps { projectData: Project; @@ -105,12 +106,18 @@ const IterationsContainer = ({ JSON.stringify(TreeSelectBaseColumnsChecked) ); + const columns_metadata: ColumnsMetadata[] = activeExperiments.map( + (experiment) => experiment.columns_metadata + ); + + console.log(columns_metadata); + let [ gridColumnsParameters, TreeSelectBaseColumnsOptionsWithParams, TreeSelectBaseColumnsCheckedWithParams, ] = extractColumnsData( - rowData, + columns_metadata, "parameters", TreeSelectBaseColumnsOptionsAll, TreeSelectBaseColumnsCheckedAll @@ -121,7 +128,7 @@ const IterationsContainer = ({ TreeSelectBaseColumnsOptionsWithParamsAndMetrics, TreeSelectBaseColumnsCheckedWithParamsAndMetrics, ] = extractColumnsData( - rowData, + columns_metadata, "metrics", TreeSelectBaseColumnsOptionsWithParams, TreeSelectBaseColumnsCheckedWithParams diff --git a/client/src/hooks/use-data-hook.ts b/client/src/hooks/use-data-hook.ts index 3ebc0e7..326e1c1 100644 --- a/client/src/hooks/use-data-hook.ts +++ b/client/src/hooks/use-data-hook.ts @@ -231,16 +231,56 @@ export const useData = create((set) => ({ if (experiment_index === -1) return state; + let experiment: Experiment = + state.projects[index].experiments[experiment_index]; + state.projects[index].experiments[ experiment_index ].iterations = state.projects[index].experiments[ experiment_index - ].iterations.filter( - (iteration) => - !iterationsToDelete[experiment_id].includes( + ].iterations.filter((iteration) => { + if ( + iterationsToDelete[experiment_id].includes( iteration.id ) - ); + ) { + const metrics = Object.keys(iteration.metrics); + const parameters = Object.keys( + iteration.parameters + ); + + for (const metric of metrics) { + experiment.columns_metadata[metric][ + "count" + ]! -= 1; + if ( + experiment.columns_metadata[metric][ + "count" + ] === 0 + ) { + delete experiment.columns_metadata[metric]; + } + } + + for (const parameter of parameters) { + experiment.columns_metadata[parameter][ + "count" + ]! -= 1; + if ( + experiment.columns_metadata[parameter][ + "count" + ] === 0 + ) { + delete experiment.columns_metadata[ + parameter + ]; + } + } + + return false; + } + return true; + }); } return { projects: [...state.projects] }; diff --git a/client/src/lib/utils.ts b/client/src/lib/utils.ts index 9aace8d..6a3deb0 100644 --- a/client/src/lib/utils.ts +++ b/client/src/lib/utils.ts @@ -1,6 +1,5 @@ import { Chart } from "@/types/chart"; import { Dataset } from "@/types/dataset"; -import { Iteration } from "@/types/iteration"; import { Model } from "@/types/model"; import { BinMethod, @@ -178,34 +177,34 @@ export const dateToHumanize = (date: string) => { }; export const extractColumnsData = ( - rowData: Iteration[], + columns_metadata: any[], type: "parameters" | "metrics", TreeSelectBaseColumnsOptionsAll: any, TreeSelectBaseColumnsCheckedAll: any ) => { - let columnsPerIterations = rowData.map( - (iteration) => - new Set( - iteration[type] - ? Object.getOwnPropertyNames(iteration[type]) - : "" - ) - ); + const columns: Set = new Set(); + + const mapping = { + parameters: "parameter", + metrics: "metric", + }; - let columnsUniqueSet = new Set(); + const metadataType = mapping[type]; - columnsPerIterations.forEach((set) => { - set.forEach((value) => { - columnsUniqueSet.add(value); + columns_metadata.forEach((metadata) => { + Object.keys(metadata).forEach((key) => { + if (metadata[key].type === metadataType) { + columns.add(key); + } }); }); - let columnsUniqueArray = Array.from(columnsUniqueSet); + const columnsArray = Array.from(columns); - let gridColumns = []; - let treeselectColumns = []; + const gridColumns = []; + const treeselectColumns = []; - if (columnsUniqueArray.length > 0) { + if (columnsArray.length > 0) { Object.assign(TreeSelectBaseColumnsCheckedAll, { [type]: { checked: true, @@ -213,25 +212,22 @@ export const extractColumnsData = ( }, }); - for (let i = 0; i < columnsUniqueArray.length; i++) { - const key = `${type}.` + columnsUniqueArray[i]; + for (let i = 0; i < columnsArray.length; i++) { + const key = `${type}.` + columnsArray[i]; gridColumns.push({ field: key, - headerName: columnsUniqueArray[i], + headerName: columnsArray[i], filter: "agNumberColumnFilter", cellRenderer: (val: any) => { - if ( - val.data[type] && - val.data[type][columnsUniqueArray[i]] - ) { - return val.data[type][columnsUniqueArray[i]]; + if (val.data[type] && val.data[type][columnsArray[i]]) { + return val.data[type][columnsArray[i]]; } return "-"; }, }); treeselectColumns.push({ key: key, - label: columnsUniqueArray[i], + label: columnsArray[i], }); Object.assign(TreeSelectBaseColumnsCheckedAll, { [key]: { diff --git a/client/src/types/experiment.ts b/client/src/types/experiment.ts index 4f39d43..6b24ae6 100644 --- a/client/src/types/experiment.ts +++ b/client/src/types/experiment.ts @@ -1,4 +1,5 @@ import { Iteration } from "@/types/iteration"; +import { ColumnsMetadata } from "@/types/columns-metadata.ts"; export interface Experiment { id: string; @@ -8,4 +9,5 @@ export interface Experiment { updated_at: Date; iterations: Iteration[]; checked?: boolean; + columns_metadata: ColumnsMetadata; } diff --git a/server/app/models/experiment.py b/server/app/models/experiment.py index a5ce1bf..05d80e3 100644 --- a/server/app/models/experiment.py +++ b/server/app/models/experiment.py @@ -17,6 +17,7 @@ class Experiment(BaseModel): - **created_at (datetime)**: Experiment creation date. - **updated_at (Optional[datetime])**: Experiment last update date. - **iterations (List[Iteration])**: Experiment iterations. + - **columns_metadata (dict)**: Experiment's iterations columns metadata. """ id: PydanticObjectId = Field(default_factory=PydanticObjectId, alias="id") @@ -26,6 +27,7 @@ class Experiment(BaseModel): created_at: datetime = Field(default_factory=datetime.now) updated_at: Optional[datetime] = Field(default_factory=datetime.now) iterations: List[Iteration] = [] + columns_metadata: dict = {} def __repr__(self) -> str: return f"" diff --git a/server/app/routers/experiment.py b/server/app/routers/experiment.py index 005a126..42be1e6 100644 --- a/server/app/routers/experiment.py +++ b/server/app/routers/experiment.py @@ -13,6 +13,7 @@ from app.routers.exceptions.iteration import iteration_not_found_exception, \ iteration_in_experiment_assigned_to_monitored_model_exception, iteration_assigned_to_monitored_model_exception from app.routers.exceptions.project import project_not_found_exception +from app.services.experiment_service import ExperimentService experiment_router = APIRouter() @@ -218,6 +219,7 @@ async def delete_iterations(project_id: PydanticObjectId, experiment_dict: Dict[ await delete_iteration_from_dataset_deleting_iterations(iteration) experiment.iterations.remove(iteration) + ExperimentService.update_experiment_columns_metadata(experiment, iteration, "deleted") await project.save() diff --git a/server/app/routers/iteration.py b/server/app/routers/iteration.py index 1eee5cd..08cd1a8 100644 --- a/server/app/routers/iteration.py +++ b/server/app/routers/iteration.py @@ -2,7 +2,7 @@ from fastapi import APIRouter, status from beanie import PydanticObjectId -from typing import List, Dict +from typing import List from app.models.dataset import Dataset from app.models.iteration import Iteration, UpdateIteration @@ -13,6 +13,7 @@ from app.routers.exceptions.project import project_not_found_exception from app.routers.exceptions.iteration import iteration_not_found_exception, \ iteration_assigned_to_monitored_model_exception, iteration_no_path_to_model_exception +from app.services.experiment_service import ExperimentService iteration_router = APIRouter() @@ -147,6 +148,7 @@ async def add_iteration(project_id: PydanticObjectId, experiment_id: PydanticObj await add_iteration_to_dataset_linked_iterations(iteration) experiment.iterations.append(iteration) + ExperimentService.update_experiment_columns_metadata(experiment, iteration, "created") await project.save() return iteration @@ -218,6 +220,7 @@ async def delete_iteration(project_id: PydanticObjectId, experiment_id: Pydantic await delete_iteration_from_dataset_deleting_iteration(iteration) experiment.iterations.remove(iteration) + ExperimentService.update_experiment_columns_metadata(experiment, iteration, "deleted") await project.save() return None diff --git a/server/app/routers/project.py b/server/app/routers/project.py index 6b1c84e..85c7a84 100644 --- a/server/app/routers/project.py +++ b/server/app/routers/project.py @@ -2,11 +2,10 @@ from fastapi import APIRouter, status from beanie import PydanticObjectId -from typing import List, Dict +from typing import List from app.models.dataset import Dataset from app.models.experiment import Experiment -from app.models.iteration import Iteration from app.models.project import Project, UpdateProject, DisplayProject from app.routers.exceptions.dataset import dataset_not_found_exception from app.routers.exceptions.iteration import iteration_in_experiment_in_project_assigned_to_monitored_model_exception @@ -14,6 +13,7 @@ project_not_found_exception, project_title_not_unique_exception, ) +from app.services.experiment_service import ExperimentService router = APIRouter() @@ -30,6 +30,14 @@ async def get_all_projects() -> List[Project]: - **List[Project]**: List of all projects. """ projects = await Project.find_all().to_list() + + # For backward compatibility, update experiment columns metadata if not present + for project in projects: + for experiment in project.experiments: + ExperimentService.update_experiment_columns_metadata(experiment) + + await project.save() + return projects diff --git a/server/app/services/__init__.py b/server/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/app/services/experiment_service.py b/server/app/services/experiment_service.py new file mode 100644 index 0000000..738bfe5 --- /dev/null +++ b/server/app/services/experiment_service.py @@ -0,0 +1,104 @@ +from app.models.experiment import Experiment +from app.models.iteration import Iteration +from app.services.iteration_service import IterationService + +from typing import Literal, List + + +class ExperimentService: + + @staticmethod + def update_experiment_columns_metadata(experiment: Experiment, iteration: Iteration = None, + method=Literal['created', 'deleted']) -> None: + """ + Update experiment columns metadata. + + Args: + - **experiment (Experiment)**: Experiment + - **iteration (Iteration)**: Iteration + - **method (Literal['created', 'deleted'])**: Update method + + Returns: + - **None** + """ + # For backwards compatibility, check if columns_metadata is not present in experiment + if not iteration and not experiment.columns_metadata: + experiment.columns_metadata = {} + + for iteration in experiment.iterations: + metrics = IterationService.get_metrics(iteration) + parameters = IterationService.get_parameters(iteration) + + ExperimentService.__increment_columns_metadata(experiment, metrics, parameters) + + # If iteration is provided + if iteration: + # If method is created, increment count of columns + if method == "created": + metrics = IterationService.get_metrics(iteration) + parameters = IterationService.get_parameters(iteration) + + ExperimentService.__increment_columns_metadata(experiment, metrics, parameters) + + # If method is deleted, decrement count of columns + elif method == "deleted": + metrics = IterationService.get_metrics(iteration) + parameters = IterationService.get_parameters(iteration) + + ExperimentService.__decrement_columns_metadata(experiment, metrics, parameters) + + @staticmethod + def __increment_columns_metadata(experiment: Experiment, + metrics: List[str] = None, + parameters: List[str] = None) -> None: + """ + Increment experiment columns metadata. + + Args: + - **experiment (Experiment)**: Experiment + - **metrics (List[str])**: List of metrics + - **parameters (List[str])**: List of parameters + + Returns: + - **None** + """ + if metrics: + for metric in metrics: + if metric not in experiment.columns_metadata: + experiment.columns_metadata[metric] = {"type": "metric", "count": 1} + else: + experiment.columns_metadata[metric]["count"] += 1 + + if parameters: + for parameter in parameters: + if parameter not in experiment.columns_metadata: + experiment.columns_metadata[parameter] = {"type": "parameter", "count": 1} + else: + experiment.columns_metadata[parameter]["count"] += 1 + + @staticmethod + def __decrement_columns_metadata(experiment: Experiment, + metrics: List[str] = None, + parameters: List[str] = None) -> None: + """ + Decrement experiment columns metadata. + + Args: + - **experiment (Experiment)**: Experiment + - **metrics (List[str])**: List of metrics + - **parameters (List[str])**: List of parameters + + Returns: + - **None** + """ + if metrics: + for metric in metrics: + experiment.columns_metadata[metric]["count"] -= 1 + if experiment.columns_metadata[metric]["count"] == 0: + del experiment.columns_metadata[metric] + + if parameters: + for parameter in parameters: + experiment.columns_metadata[parameter]["count"] -= 1 + if experiment.columns_metadata[parameter]["count"] == 0: + del experiment.columns_metadata[parameter] diff --git a/server/app/services/iteration_service.py b/server/app/services/iteration_service.py new file mode 100644 index 0000000..af2b154 --- /dev/null +++ b/server/app/services/iteration_service.py @@ -0,0 +1,32 @@ +from app.models.iteration import Iteration + +from typing import List + + +class IterationService: + + @staticmethod + def get_metrics(iteration: Iteration) -> List[str]: + """ + Get metrics from iteration. + + Args: + - **iteration (Iteration)**: Iteration + + Returns: + - **List[str]**: List of metrics + """ + return list(iteration.metrics.keys()) if iteration.metrics else [] + + @staticmethod + def get_parameters(iteration: Iteration) -> List[str]: + """ + Get parameters from iteration. + + Args: + - **iteration (Iteration)**: Iteration + + Returns: + - **List[str]**: List of parameters + """ + return list(iteration.parameters.keys()) if iteration.parameters else [] diff --git a/server/app/tests/routers/test_iteration.py b/server/app/tests/routers/test_iteration.py index 78a1cf2..f26f826 100644 --- a/server/app/tests/routers/test_iteration.py +++ b/server/app/tests/routers/test_iteration.py @@ -76,6 +76,13 @@ async def test_add_iteration(client: AsyncClient): assert response.status_code == 201 assert response.json()['iteration_name'] == iteration['iteration_name'] + response = await client.get(f"/projects/{project_id}/experiments/{experiment_id}") + experiment = response.json() + + assert "columns_metadata" in experiment and isinstance(experiment["columns_metadata"], dict) + assert "accuracy" in experiment["columns_metadata"] and isinstance(experiment["columns_metadata"]["accuracy"], dict) + assert experiment["columns_metadata"]["accuracy"]["type"] == "metric" + assert experiment["columns_metadata"]["accuracy"]["count"] == 1 @pytest.mark.asyncio async def test_add_iteration2(client: AsyncClient): @@ -111,6 +118,14 @@ async def test_add_iteration2(client: AsyncClient): assert response.status_code == 201 assert response.json()['iteration_name'] == iteration['iteration_name'] + response = await client.get(f"/projects/{project_id}/experiments/{experiment_id}") + experiment = response.json() + + assert "columns_metadata" in experiment and isinstance(experiment["columns_metadata"], dict) + assert "learning_rate" in experiment["columns_metadata"] and isinstance(experiment["columns_metadata"]["learning_rate"], dict) + assert experiment["columns_metadata"]["learning_rate"]["type"] == "parameter" + assert experiment["columns_metadata"]["learning_rate"]["count"] == 2 + @pytest.mark.asyncio async def test_get_iterations(client: AsyncClient): @@ -233,6 +248,14 @@ async def test_delete_iteration_by_id(client: AsyncClient): response = await client.delete(f"/projects/{project_id}/experiments/{experiment_id}/iterations/{iteration_id}") assert response.status_code == 204 + response = await client.get(f"/projects/{project_id}/experiments/{experiment_id}") + experiment = response.json() + + assert "columns_metadata" in experiment and isinstance(experiment["columns_metadata"], dict) + assert "learning_rate" in experiment["columns_metadata"] and isinstance(experiment["columns_metadata"]["learning_rate"], dict) + assert experiment["columns_metadata"]["learning_rate"]["type"] == "parameter" + assert experiment["columns_metadata"]["learning_rate"]["count"] == 2 + @pytest.mark.asyncio async def test_delete_iteration_with_dataset(client: AsyncClient):