docs(cache): add docstrings to cache (#7)

Author: SeregaCodit

docs/api/cache_io.md

@@ -0,0 +1 @@
+::: tools.cache.CacheIO

docs/api/video_slicer.md

@@ -0,0 +1 @@
+::: tools.video_slicer.VideoSlicer

mkdocs.yml

@@ -31,6 +31,7 @@ nav:
   - API Reference:
       - Main: api/data_forge.md
       - Image Comparer: api/img_comparer.md
+      - Video slicer: api/video_slicer.md
       - Hasher:
           - Base Hasher: api/base_hasher.md
           - DHash: api/dhash.md
@@ -48,6 +49,7 @@ nav:
           - TXT writer: api/yolo_writer.md
       - Mixins:
           - FileRemoverMixin: api/file_remover.md
+      - CacheIO: api/cache_io.md
   - CLI:
       - CLI launching: cli/data_forge.md
       - Default Settings: cli/default_values.md

requirements.txt

@@ -1,16 +1,51 @@
-mkdocs>=1.6.0
-mkdocs-material>=9.5.0
-mkdocstrings[python]>=0.26.0
-mkdocs-autorefs>=1.2.0
-pydantic>=2.10.0
-pydantic-settings>=2.8.0
-python-dotenv>=1.0.0
-PyYAML>=6.0.0
-numpy>=1.24.0,<2.0.0
-opencv-python>=4.8.0
-requests>=2.31.0
-pytest>=8.0.0
-python-dateutil>=2.8.0
-watchdog>=4.0.0
-xmltodict
-pillow
+annotated-types==0.7.0
+babel==2.18.0
+backrefs==6.1
+certifi==2026.1.4
+charset-normalizer==3.4.4
+click==8.3.1
+colorama==0.4.6
+exceptiongroup==1.3.1
+ghp-import==2.1.0
+griffe==1.15.0
+idna==3.11
+iniconfig==2.3.0
+Jinja2==3.1.6
+Markdown==3.10.1
+MarkupSafe==3.0.3
+mergedeep==1.3.4
+mkdocs==1.6.1
+mkdocs-autorefs==1.4.3
+mkdocs-get-deps==0.2.0
+mkdocs-material==9.7.1
+mkdocs-material-extensions==1.3.1
+mkdocstrings==1.0.2
+mkdocstrings-python==2.0.1
+numpy==1.26.4
+opencv-python==4.11.0.86
+packaging==26.0
+paginate==0.5.7
+pandas==3.0.0
+pathspec==1.0.4
+pillow==12.1.0
+platformdirs==4.5.1
+pluggy==1.6.0
+pyarrow==23.0.0
+pydantic==2.12.5
+pydantic-settings==2.12.0
+pydantic_core==2.41.5
+Pygments==2.19.2
+pymdown-extensions==10.20.1
+pytest==9.0.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.2.1
+PyYAML==6.0.3
+pyyaml_env_tag==1.1
+requests==2.32.5
+six==1.17.0
+tomli==2.4.0
+typing-inspection==0.4.2
+typing_extensions==4.15.0
+urllib3==2.6.3
+watchdog==6.0.0
+xmltodict==1.0.2

tests/test_cache.py

@@ -0,0 +1,115 @@
+import pytest
+import numpy as np
+import pandas as pd
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+from tools.cache import CacheIO
+from const_utils.default_values import AppSettings
+
+
+@pytest.fixture
+def mock_settings():
+    """Provides mock AppSettings for testing CacheIO."""
+    settings = AppSettings(
+        log_path=Path("./test_log"),
+        cache_file_path=Path("./test_cache")
+    )
+    return settings
+
+
+@pytest.fixture
+def cache_io_instance(mock_settings):
+    """Provides a CacheIO instance with mock settings."""
+    return CacheIO(settings=mock_settings)
+
+
+@pytest.fixture
+def temp_cache_file(tmp_path):
+    """Provides a temporary cache file path for testing."""
+    return tmp_path / "test_cache.parquet"
+
+
+@pytest.fixture
+def sample_hash_map():
+    """Provides a sample hash map for testing."""
+    return {
+        Path("/path/to/file1.jpg"): np.array([True, False, True], dtype=bool),
+        Path("/path/to/file2.png"): np.array([False, True, False], dtype=bool),
+    }
+
+
+def test_cache_io_init(cache_io_instance):
+    """Tests if CacheIO initializes correctly."""
+    assert isinstance(cache_io_instance, CacheIO)
+    assert cache_io_instance.settings is not None
+    assert cache_io_instance.logger is not None
+
+
+def test_save_empty_hash_map(cache_io_instance, temp_cache_file):
+    """Tests saving an empty hash map, which should not create a file."""
+    hash_map = {}
+    cache_io_instance.save(hash_map, temp_cache_file)
+    assert not temp_cache_file.exists()
+
+
+def test_save_valid_hash_map(cache_io_instance, temp_cache_file, sample_hash_map):
+    """Tests saving a valid hash map and checks if the file is created."""
+    cache_io_instance.save(sample_hash_map, temp_cache_file)
+    assert temp_cache_file.exists()
+
+
+def test_load_non_existent_file(cache_io_instance, temp_cache_file):
+    """Tests loading from a cache file that does not exist."""
+    loaded_data = cache_io_instance.load(temp_cache_file)
+    assert loaded_data == {}
+
+
+def test_load_damaged_file(cache_io_instance, temp_cache_file):
+    """Tests loading from a damaged cache file (simulated by EOFError)."""
+    temp_cache_file.write_text("corrupted data")
+    with patch('pandas.read_parquet', side_effect=EOFError):
+        loaded_data = cache_io_instance.load(temp_cache_file)
+        assert loaded_data == {}
+
+
+def test_load_valid_file(cache_io_instance, temp_cache_file, sample_hash_map):
+    """Tests loading from a valid cache file."""
+    cache_io_instance.save(sample_hash_map, temp_cache_file)
+    loaded_data = cache_io_instance.load(temp_cache_file)
+
+    assert len(loaded_data) == len(sample_hash_map)
+    for path, hash_array in sample_hash_map.items():
+        assert path in loaded_data
+        assert np.array_equal(loaded_data[path], hash_array)
+
+
+def test_generate_cache_filename_no_custom_name():
+    """Tests generating a cache filename without a custom name."""
+    source_path = Path("/home/user/images")
+    hash_type = "dhash"
+    core_size = 16
+    filename = CacheIO.generate_cache_filename(source_path, hash_type, core_size, None)
+    assert filename.startswith("cache_")
+    assert "_dimages" in filename
+    assert "dhash_s16.parquet" in filename
+
+
+def test_generate_cache_filename_with_custom_name():
+    """Tests generating a cache filename with a custom name."""
+    source_path = Path("/home/user/videos")
+    hash_type = "phash"
+    core_size = 32
+    custom_name = Path("my_video_cache")
+    filename = CacheIO.generate_cache_filename(source_path, hash_type, core_size, custom_name)
+    assert filename == "my_video_cache_phash_s32.parquet"
+
+
+def test_generate_cache_filename_with_custom_name_with_suffix():
+    """Tests generating a cache filename with a custom name that already includes the suffix."""
+    source_path = Path("/home/user/documents")
+    hash_type = "ahash"
+    core_size = 8
+    custom_name = Path("doc_cache.parquet")
+    filename = CacheIO.generate_cache_filename(source_path, hash_type, core_size, custom_name)
+    assert filename == "doc_cache_ahash_s8.parquet"

tools/cache.py

@@ -1,19 +1,23 @@
 import hashlib
-import pickle
 from pathlib import Path
 from typing import Dict, Optional
 import numpy as np
+import pandas as pd
 
 from const_utils.default_values import AppSettings
 from logger.logger import LoggerConfigurator
 from logger.logger_protocol import LoggerProtocol
 
 
 class CacheIO:
-    SUFFIX = ".pkl"
+    SUFFIX = ".parquet"
     def __init__(self, settings: AppSettings):
-        """
-        saving and reading cache files for faster loading data
+        """Initializes the CacheIO class.
+
+        This class helps to save and load cache files. It makes data loading faster.
+
+        Args:
+            settings (AppSettings): The application settings for logging and paths.
         """
         self.settings = settings
         self.logger = LoggerConfigurator.setup(
@@ -23,33 +27,78 @@ def __init__(self, settings: AppSettings):
         )
 
 
-    def load(self: LoggerProtocol, cache_file) -> Dict[Path, np.ndarray]:
-        """loading cache file"""
-        try:
-            self.logger.info(f"Loading cache file {cache_file}")
-            with open(cache_file, "rb") as file:
-                return pickle.load(file)
+    def load(self: LoggerProtocol, cache_file: Path) -> Dict[Path, np.ndarray]:
+        """Loads data from a cache file.
+
+        It reads a parquet file and converts it into a dictionary of hashes.
 
-        except FileNotFoundError:
+        Args:
+            cache_file (Path): The path to the cache file.
+
+        Returns:
+            Dict[Path, np.ndarray]: A dictionary where keys are file paths and values are hashes.
+                Returns an empty dictionary if the file does not exist or is broken.
+        """
+        if not cache_file.exists():
             self.logger.warning(f"Cache file {cache_file} does not exist")
             return {}
+
+        try:
+            self.logger.info(f"Loading cache file {cache_file}")
+            df = pd.read_parquet(cache_file)
+            data = {
+                Path(row['path']): np.array(row['hash'], dtype=bool)
+                for _, row in df.iterrows()
+            }
+            return data
         except EOFError:
             self.logger.warning(f"Cache file {cache_file} is damaged. Deleting cache file")
             return {}
 
+
     def save(self: LoggerProtocol, hash_map: Dict[Path, np.ndarray], cache_file: Path) -> None:
-        """save hashmap to cache file"""
+        """Saves a hash map to a cache file.
+
+        It converts the dictionary into a parquet file for later use.
+
+        Args:
+            hash_map (Dict[Path, np.ndarray]): The dictionary of hashes to save.
+            cache_file (Path): The path where the cache file will be saved.
+        """
+        if not hash_map:
+            self.logger.warning("Hash map is empty, skipping saving cache file")
+            return
+
         cache_file.parent.mkdir(parents=True, exist_ok=True)
-        self.logger.info(f"Saving cache file {cache_file}")
+        self.logger.info(f"Saving {len(hash_map)} hashes to {cache_file.name}")
 
-        with open(cache_file, 'wb') as file:
-            pickle.dump(hash_map, file)
+        try:
+            data = [
+                {'path': str(p), 'hash': h.tolist()}
+                for p, h in hash_map.items()
+            ]
+            df = pd.DataFrame(data)
+            df.to_parquet(cache_file, engine="pyarrow", compression="snappy", index=False)
+            self.logger.info(f"Cache saved successfully.")
+        except Exception as e:
+            self.logger.error(f"Critical error saving cache: {e}")
 
-        self.logger.info(f"Cache file {cache_file} saved")
 
     @classmethod
     def generate_cache_filename(cls, source_path: Path, hash_type: str, core_size: int, cache_name: Optional[Path]) -> str:
-        """generate cache filename from source_path """
+        """Creates a unique name for the cache file.
+
+        The name is based on the folder path, hash type, and size.
+
+        Args:
+            source_path (Path): The folder path being processed.
+            hash_type (str): The type of hash used (e.g., 'dhash').
+            core_size (int): The size of the hash.
+            cache_name (Optional[Path]): A custom name for the cache file, if provided.
+
+        Returns:
+            str: The generated filename for the cache.
+        """
         suffix = f"{hash_type}_s{core_size}{cls.SUFFIX}"
         if cache_name is None:
             abs_path = str(source_path.resolve())

tools/video_slicer.py

@@ -2,19 +2,31 @@
 import cv2
 
 class VideoSlicer:
-    """slicing video file and saving it to target directory"""
+    """A class to cut a video into many images.
+
+    This class takes a video file and saves its frames as separate image files in a folder.
+    """
     def __init__(self):
+        """Initializes the VideoSlicer.
+
+        It sets the initial state of the slicer.
+        """
         self.__sliced: bool = False
 
 
     def slice(self, source_file: Path, target_dir: Path, suffix: str = ".jpg", step: float = 1) -> tuple:
-        """
-        slicing video file and saving it to target directory
-        :param source_file: path to video file
-        :param target_dir: target dir where sliced images from video file will be saved
-        :param suffix: a suffix to add to image filename
-        :param step: time step for saving image in seconds
-        :return: count of images saved in target_dir
+        """Cuts the video into images and saves them to a folder.
+
+        Args:
+            source_file (Path): The path to the video file you want to cut.
+            target_dir (Path): The folder where you want to save the images.
+            suffix (str): The file extension for the images (for example, '.jpg'). Defaults to '.jpg'.
+            step (float): How many seconds to wait between saving images. Defaults to 1.
+
+        Returns:
+            tuple: A tuple containing:
+                - bool: True if the video was sliced successfully, False otherwise.
+                - int: The total number of images saved.
         """
         cap = cv2.VideoCapture(str(source_file))
 
@@ -44,6 +56,12 @@ def slice(self, source_file: Path, target_dir: Path, suffix: str = ".jpg", step:
         self.__sliced = True
         return self.sliced, img_counter
 
+
     @property
     def sliced(self) -> bool:
+        """Checks if the video has been sliced.
+
+        Returns:
+            bool: True if the slice method was called and finished, False otherwise.
+        """
         return self.__sliced

tst_commands.py

@@ -60,6 +60,6 @@
     MAPPING[Commands.dedup].append("16")
 
 
-    sys.argv = MAPPING[Commands.convert_annotations]
+    sys.argv = MAPPING[Commands.dedup]
     app = DataForge()
     app.execute()