lockutils, minio path util

Author: JATAYU000

openml/datasets/dataset.py

@@ -15,10 +15,10 @@
 import numpy as np
 import pandas as pd
 import scipy.sparse
-from filelock import FileLock
 
 import openml
 from openml.base import OpenMLBase
+from openml.utils._openml import file_lock
 
 from .data_feature import OpenMLDataFeature
 
@@ -467,7 +467,7 @@ def _parse_data_from_arff(  # noqa: C901, PLR0912, PLR0915
             List[str]: List of column names.
         """
         lock_path = str(arff_file_path) + ".lock"
-        with FileLock(lock_path):
+        with file_lock(lock_path):
             try:
                 data = self._get_arff(self.format)
             except OSError as e:
@@ -618,7 +618,7 @@ def _parse_data_from_file(
 
     def _parse_data_from_pq(self, data_file: Path) -> tuple[list[str], list[bool], pd.DataFrame]:
         lock_path = str(data_file) + ".lock"
-        with FileLock(lock_path):
+        with file_lock(lock_path):
             try:
                 data = pd.read_parquet(data_file)
             except Exception as e:
@@ -635,7 +635,7 @@ def _load_data(self) -> tuple[pd.DataFrame, list[bool], list[str]]:  # noqa: PLR
         if need_to_create_pickle or need_to_create_feather:
             cache_file = self.data_pickle_file if need_to_create_pickle else self.data_feather_file
             lock_path = str(cache_file) + ".lock"
-            with FileLock(lock_path):
+            with file_lock(lock_path):
                 if self.data_file is None:
                     self._download_data()
 

openml/utils/_openml.py

@@ -2,11 +2,12 @@
 from __future__ import annotations
 
 import contextlib
+import hashlib
 import re
 import shutil
 import warnings
 from abc import ABC, abstractmethod
-from collections.abc import Callable, Iterable, Mapping, Sequence, Sized
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Sized
 from functools import wraps
 from pathlib import Path
 from typing import (
@@ -454,6 +455,24 @@ def _create_lockfiles_dir() -> Path:
     return path
 
 
+@contextlib.contextmanager
+def file_lock(lock_path: str) -> Iterator[None]:
+    """Context manager that uses `oslo_concurrency.lockutils.external_lock`
+
+    The oslo-based locks are placed in the centralized cache `locks` folder
+    returned by ``_create_lockfiles_dir()``. A deterministic name derived from
+    the lock path is used to avoid collisions.
+    """
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        from oslo_concurrency import lockutils
+
+    lock_dir = _create_lockfiles_dir()
+    name = hashlib.sha256(str(lock_path).encode()).hexdigest()
+    with lockutils.external_lock(name=name, lock_path=lock_dir):
+        yield
+
+
 class ProgressBar(ProgressType):
     """Progressbar for MinIO function's `progress` parameter."""
 

tests/test_datasets/test_dataset_functions.py

@@ -1835,7 +1835,7 @@ def _dataset_features_is_downloaded(did: int):
 
 def _dataset_data_file_is_downloaded(dataset: OpenMLDataset):
     if dataset._parquet_url is not None:
-        pq_directory = Path(openml.config.get_cache_directory()) / Path(openml.config.get_minio_download_path(dataset._parquet_url)).parent
+        pq_directory = Path(openml.config.get_minio_download_path(dataset._parquet_url)).parent
         if pq_directory.exists():
             return any(f.suffix == ".pq" for f in pq_directory.iterdir())
     arff_directory = Path(openml.config.get_cache_directory()) / "data/v1/download" / str(dataset.id)

tests/test_tasks/test_task_functions.py

@@ -165,7 +165,7 @@ def test_get_task(self):
             os.path.join(openml.config.get_cache_directory(), "tasks", "1", "datasplits.arff")
         )
         assert os.path.exists(
-            os.path.join(openml.config.get_cache_directory(), openml.config.get_minio_download_path(dataset._parquet_url))
+            openml.config.get_minio_download_path(dataset._parquet_url)
         )
 
     @pytest.mark.test_server()