fix(backend.storage): import dataset

backend/api/models/data/spectrogram_analysis.py

@@ -4,7 +4,7 @@
 
 from dateutil import parser
 from django.conf import settings
-from django.db import models
+from django.db import models, transaction
 from django.db.models import CheckConstraint, Q, Min, Max
 from typing_extensions import deprecated
 
@@ -191,30 +191,47 @@ def get_osekit_spectro_dataset_serialized_path(self) -> Path:
             )
         )
 
+    @transaction.atomic
     def add_spectrograms(self, spectrograms: list["Spectrogram"]):
         """Add spectrogram objects to current analysis"""
-        self.spectrograms.bulk_create(
-            spectrograms, ignore_conflicts=True, batch_size=100
+        existing_spectrograms = []
+        new_spectrograms = []
+        dataset_spectrograms = self.spectrograms.model.objects.filter(
+            analysis__dataset=self.dataset
         )
-        self.spectrograms.through.objects.bulk_create(
-            [
-                self.spectrograms.through(
-                    spectrogram=self.spectrograms.model.objects.get(
-                        filename=spectrogram.filename,
-                        format=spectrogram.format,
-                        start=spectrogram.start,
-                        end=spectrogram.end,
-                    ),
-                    spectrogramanalysis=self,
+
+        for s in spectrograms:
+            params = {
+                "filename": s.filename,
+                "format": s.format,
+                "start": s.start,
+                "end": s.end,
+            }
+            if dataset_spectrograms.filter(**params).exists():
+                existing_spectrograms.append(
+                    dataset_spectrograms.filter(**params).first()
                 )
-                for spectrogram in spectrograms
-            ]
+            else:
+                new_spectrograms.append(s)
+
+        new_spectrograms: list[
+            "Spectrogram"
+        ] = self.spectrograms.model.objects.bulk_create(
+            new_spectrograms, ignore_conflicts=True
         )
+        spectrogram_analysis_rel = []
+        spectrograms = existing_spectrograms + new_spectrograms
+        for spectrogram in spectrograms:
+            spectrogram.save()
+            spectrogram_analysis_rel.append(
+                self.spectrograms.through(
+                    spectrogram=spectrogram, spectrogramanalysis=self
+                )
+            )
 
-        info = self.spectrograms.aggregate(start=Min("start"), end=Max("end"))
-        self.start = info["start"]
-        self.end = info["end"]
-        self.save()
+        self.spectrograms.through.objects.bulk_create(spectrogram_analysis_rel)
+
+        self.update_dates()
 
     def update_dates(self):
         """Update start and end dates based on spectrogram data"""

backend/api/tests/schema/annotation_phase/all_annotation_phases.py

@@ -102,7 +102,7 @@ def test_connected_admin_filter_owner(self):
 
         content = json.loads(response.content)["data"]["allAnnotationPhases"]["results"]
         self.assertEqual(len(content), AnnotationPhase.objects.count())
-        self.assertEqual(content[1]["phase"], "Annotation")
+        self.assertEqual(content[0]["phase"], "Annotation")
 
     def test_connected_admin_filter_annotator(self):
         response = self.gql_query(

backend/storage/resolvers/_abstract.py

@@ -1,15 +1,17 @@
 from pathlib import PureWindowsPath
 
 from backend.api.models import Dataset, SpectrogramAnalysis, Spectrogram
-from backend.storage.exceptions import CannotGetChildrenException
+from backend.storage.exceptions import (
+    CannotGetChildrenException,
+)
+from backend.storage.models import ImportStatus
 from backend.storage.types import (
     StorageItem,
     StorageDataset,
     StorageAnalysis,
     FailedItem,
     StorageFolder,
 )
-from backend.storage.models import ImportStatus
 from backend.storage.utils import make_path_relative, is_local_root, listdir, isfile
 
 
@@ -43,7 +45,6 @@ def analysis(self) -> SpectrogramAnalysis | FailedItem | None:
         return self.__analysis
 
     def __init__(self, path: str):
-        # pylint: disable=broad-exception-caught
         self.__path = path
         self.__dataset = self.get_dataset(path=self.path)
         self.__all_analysis = self.get_all_analysis(path=self.path)
@@ -56,7 +57,7 @@ def _get_dataset_for_path(
 
     def get_dataset(self, path: str | None = None) -> Dataset | FailedItem | None:
         """Returns dataset from storage"""
-        # pylint: disable=broad-exception-caught, assignment-from-none
+        # pylint: disable=assignment-from-none
         if path is None:
             return self.dataset
 
@@ -70,30 +71,29 @@ def get_dataset(self, path: str | None = None) -> Dataset | FailedItem | None:
         return self.get_dataset(path=PureWindowsPath(path).parent.as_posix())
 
     def _get_all_analysis_for_dataset(
-        self, dataset: Dataset
+        self, dataset: Dataset, detailed: bool = False
     ) -> list[SpectrogramAnalysis | FailedItem]:
         return []
 
-    def _get_all_detailed_analysis_for_dataset(
-        self, dataset: Dataset
-    ) -> list[SpectrogramAnalysis | FailedItem]:
-        return self._get_all_analysis_for_dataset(dataset=dataset)
-
     def get_all_analysis(
         self, path: str | None = None, detailed: bool = False
     ) -> list[SpectrogramAnalysis | FailedItem]:
         """Returns analysis list from storage"""
         if path is None and not detailed:
             return self.all_analysis
+        path = self.path
         dataset = self.get_dataset(path=path)
         if not dataset or isinstance(dataset, FailedItem):
             return []
-        if detailed:
-            return self._get_all_detailed_analysis_for_dataset(dataset=dataset)
-        return self._get_all_analysis_for_dataset(dataset=dataset)
+        return self._get_all_analysis_for_dataset(dataset=dataset, detailed=detailed)
+
+    def _get_analysis(
+        self, dataset: Dataset, relative_path: str, detailed: bool = False
+    ) -> SpectrogramAnalysis | FailedItem | None:
+        return None
 
     def get_analysis(
-        self, path: str | None = None
+        self, path: str | None = None, detailed: bool = False
     ) -> SpectrogramAnalysis | FailedItem | None:
         """Returns analysis from storage"""
         if path is None:
@@ -103,10 +103,9 @@ def get_analysis(
             return None
         path = path or self.path
         relative_path = make_path_relative(path, to=dataset.path)
-        for analysis in self.get_all_analysis(path=path):
-            if analysis.path == relative_path:
-                return analysis
-        return None
+        return self._get_analysis(
+            dataset=dataset, relative_path=relative_path, detailed=detailed
+        )
 
     def get_all_spectrograms_for_analysis(
         self, analysis: SpectrogramAnalysis
@@ -130,19 +129,22 @@ def _get_storage_dataset_from_dataset(self, dataset: Dataset) -> StorageDataset:
             import_status=ImportStatus.AVAILABLE,
         )
 
-    def get_item(self, path: str | None = None) -> StorageItem | None:
+    def get_item(
+        self, path: str | None = None, discover_analysis: bool = True
+    ) -> StorageItem | None:
         """Get item from storage"""
         dataset = self.get_dataset(path)
         if not dataset:
             return StorageFolder(path=path)
 
-        analysis = self.get_analysis(path=path)
-        if analysis:
-            if isinstance(analysis, FailedItem):
-                return analysis.to_storage_analysis()
-            return self._get_storage_analysis_from_spectrogram_analysis(
-                analysis=analysis
-            )
+        if discover_analysis:
+            analysis = self.get_analysis(path=path)
+            if analysis:
+                if isinstance(analysis, FailedItem):
+                    return analysis.to_storage_analysis()
+                return self._get_storage_analysis_from_spectrogram_analysis(
+                    analysis=analysis
+                )
 
         if isinstance(dataset, FailedItem):
             return dataset.to_storage_dataset()
@@ -152,7 +154,7 @@ def get_children_items(self) -> list[StorageItem]:
         """Get children items from storage"""
         dataset = self.get_dataset()
         analysis = self.get_analysis()
-        if analysis:
+        if analysis and not isinstance(analysis, FailedItem):
             raise CannotGetChildrenException(self.path)
 
         if dataset:
@@ -164,7 +166,7 @@ def get_children_items(self) -> list[StorageItem]:
             ]
 
         return [
-            self.get_item(path=folder)
+            self.get_item(path=folder, discover_analysis=False)
             for folder in listdir(self.path)
             if not isfile(folder)
         ]