Implement SQL-based document search matching MATLAB's database behavior

.github/workflows/python-package.yml

@@ -1,30 +1,53 @@
-name: Python package
+name: CI
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [main]
   pull_request:
-    branches: [ "main" ]
+    branches: [main]
+  workflow_dispatch:
 
 jobs:
-  build:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install linting tools
+        run: |
+          python -m pip install --upgrade pip
+          pip install ruff black
+
+      - name: Check formatting with black
+        run: black --check src/ tests/
 
+      - name: Lint with ruff
+        run: ruff check src/ tests/
+
+  test:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e ".[dev]"
-    - name: Test with pytest
-      run: |
-        pytest
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Test with pytest
+        run: pytest

src/did/binarydoc.py

@@ -1,13 +1,14 @@
 import abc
 
+
 class BinaryDoc(abc.ABC):
     def __init__(self, *args, **kwargs):
         pass
 
     def __del__(self):
         try:
             self.fclose()
-        except:
+        except Exception:
             pass
 
     @abc.abstractmethod
@@ -36,4 +37,4 @@ def fread(self, count, precision, skip):
 
     @abc.abstractmethod
     def fclose(self):
-        pass
+        pass

src/did/common.py

@@ -3,12 +3,14 @@
 from pathlib import Path
 import uuid
 
+
 def toolboxdir():
     """
     Returns the path to the toolbox directory.
     """
     return os.path.dirname(os.path.abspath(__file__))
 
+
 def must_be_writable(folder_path):
     """
     Checks if a folder is writable, and creates it if it doesn't exist.
@@ -18,35 +20,41 @@ def must_be_writable(folder_path):
             os.makedirs(folder_path)
         except OSError:
             # Fallback for potential permission errors
-            folder_path = os.path.join(tempfile.gettempdir(), os.path.basename(folder_path))
+            folder_path = os.path.join(
+                tempfile.gettempdir(), os.path.basename(folder_path)
+            )
             os.makedirs(folder_path, exist_ok=True)
 
     test_file = os.path.join(folder_path, f"testfile_{uuid.uuid4()}.txt")
     try:
-        with open(test_file, 'w') as f:
-            f.write('test')
+        with open(test_file, "w") as f:
+            f.write("test")
     except IOError:
-        raise IOError(f'We do not have write access to the folder at {folder_path}')
+        raise IOError(f"We do not have write access to the folder at {folder_path}")
     finally:
         if os.path.exists(test_file):
             os.remove(test_file)
 
+
 class PathConstants:
     """
     Class that defines some global constants for the DID package.
     """
+
     PATH = toolboxdir()
-    DEFPATH = os.path.join(PATH, 'example_schema', 'demo_schema1')
+    DEFPATH = os.path.join(PATH, "example_schema", "demo_schema1")
 
     DEFINITIONS = {
-        '$DIDDOCUMENT_EX1': os.path.join(DEFPATH, 'database_documents'),
-        '$DIDSCHEMA_EX1': os.path.join(DEFPATH, 'database_schema'),
-        '$DIDCONTROLLEDVOCAB_EX1': os.path.join(DEFPATH, 'controlled_vocabulary')
+        "$DIDDOCUMENT_EX1": os.path.join(DEFPATH, "database_documents"),
+        "$DIDSCHEMA_EX1": os.path.join(DEFPATH, "database_schema"),
+        "$DIDCONTROLLEDVOCAB_EX1": os.path.join(DEFPATH, "controlled_vocabulary"),
     }
 
-    _temp_path = os.path.join(tempfile.gettempdir(), 'didtemp')
-    _file_cache_path = os.path.join(str(Path.home()), 'Documents', 'DID', 'fileCache')
-    _preferences_path = os.path.join(str(Path.home()), 'Documents', 'DID', 'Preferences')
+    _temp_path = os.path.join(tempfile.gettempdir(), "didtemp")
+    _file_cache_path = os.path.join(str(Path.home()), "Documents", "DID", "fileCache")
+    _preferences_path = os.path.join(
+        str(Path.home()), "Documents", "DID", "Preferences"
+    )
 
     @property
     def temppath(self):
@@ -63,14 +71,17 @@ def preferences(self):
         must_be_writable(self._preferences_path)
         return self._preferences_path
 
+
 # Placeholder for fileCache class
 class FileCache:
     def __init__(self, path, size):
         self.path = path
         self.size = size
 
+
 _cached_cache = None
 
+
 def get_cache():
     """
     Returns a persistent cache object.
@@ -79,4 +90,4 @@ def get_cache():
     if _cached_cache is None:
         path_constants = PathConstants()
         _cached_cache = FileCache(path_constants.filecachepath, 33)
-    return _cached_cache
+    return _cached_cache

src/did/database.py

@@ -1,19 +1,20 @@
 import abc
 
+
 class Database(abc.ABC):
-    def __init__(self, connection='', **kwargs):
+    def __init__(self, connection="", **kwargs):
         self.connection = connection
         self.version = None
-        self.current_branch_id = ''
+        self.current_branch_id = ""
         self.frozen_branch_ids = []
         self.dbid = None
         self.preferences = {}
-        self.debug = kwargs.get('debug', False)
+        self.debug = kwargs.get("debug", False)
 
     def __del__(self):
         try:
             self.close()
-        except:
+        except Exception:
             pass
 
     def open(self):
@@ -87,7 +88,7 @@ def _do_get_doc_ids(self, branch_id=None):
     def _do_add_doc(self, document_obj, branch_id, **kwargs):
         pass
 
-    def get_docs(self, document_ids, branch_id=None, OnMissing='error', **kwargs):
+    def get_docs(self, document_ids, branch_id=None, OnMissing="error", **kwargs):
         is_single = False
         if not isinstance(document_ids, list):
             document_ids = [document_ids]
@@ -100,36 +101,40 @@ def get_docs(self, document_ids, branch_id=None, OnMissing='error', **kwargs):
 
         # Checking logic here (inefficient but generic):
         if branch_id is not None:
-             branch_doc_ids = self.get_doc_ids(branch_id)
-             # If branch doesn't exist? get_doc_ids might return empty or raise?
-             # get_doc_ids calls _do_get_doc_ids.
+            branch_doc_ids = self.get_doc_ids(branch_id)
+            # If branch doesn't exist? get_doc_ids might return empty or raise?
+            # get_doc_ids calls _do_get_doc_ids.
 
         docs = []
         for doc_id in document_ids:
             if branch_id is not None:
                 if doc_id not in branch_doc_ids:
                     # Document not in branch
-                    if OnMissing == 'error':
-                        raise ValueError(f"Document {doc_id} not found in branch {branch_id}")
-                    elif OnMissing == 'warn':
-                         print(f"Warning: Document {doc_id} not found in branch {branch_id}")
-                         continue
+                    if OnMissing == "error":
+                        raise ValueError(
+                            f"Document {doc_id} not found in branch {branch_id}"
+                        )
+                    elif OnMissing == "warn":
+                        print(
+                            f"Warning: Document {doc_id} not found in branch {branch_id}"
+                        )
+                        continue
                     else:
-                         continue
+                        continue
 
             docs.append(self._do_get_doc(doc_id, OnMissing=OnMissing, **kwargs))
 
-        if not docs and OnMissing != 'ignore' and len(document_ids) > 0:
-             # If filtered out all?
-             pass
+        if not docs and OnMissing != "ignore" and len(document_ids) > 0:
+            # If filtered out all?
+            pass
 
         if is_single:
-             return docs[0] if docs else None
+            return docs[0] if docs else None
         else:
-             return docs
+            return docs
 
     @abc.abstractmethod
-    def _do_get_doc(self, document_id, OnMissing='error', **kwargs):
+    def _do_get_doc(self, document_id, OnMissing="error", **kwargs):
         pass
 
     def remove_docs(self, document_ids, branch_id=None, **kwargs):
@@ -179,9 +184,11 @@ def search(self, query_obj, branch_id=None):
             branch_id = self.current_branch_id
 
         doc_ids = self.get_doc_ids(branch_id)
-        docs = self.get_docs(doc_ids, OnMissing='ignore')
-        if docs is None: docs = []
-        if not isinstance(docs, list): docs = [docs]
+        docs = self.get_docs(doc_ids, OnMissing="ignore")
+        if docs is None:
+            docs = []
+        if not isinstance(docs, list):
+            docs = [docs]
 
         search_params = query_obj.to_search_structure()
 
@@ -196,4 +203,4 @@ def search(self, query_obj, branch_id=None):
 
     @abc.abstractmethod
     def do_run_sql_query(self, query_str, **kwargs):
-        pass
+        pass