Main ingester wrapper refactoring, introducing metadata file read/writing and Dataset Tracker integration.

.gitignore

@@ -1,134 +1,14 @@
-# Byte-compiled / optimized / DLL files
+# Python-generated files
 __pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
+*.py[oc]
 build/
-develop-eggs/
 dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
 wheels/
-pip-wheel-metadata/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
+*.egg-info
+.DS_Store
 
-# pyenv
-.python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
+# Virtual environments
 .venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
 
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-.vscode/
-
-*.env
-.netrc.gpg
+# Environment variables (contains secrets)
+.env

README.md

@@ -26,7 +26,7 @@ If you're developing locally, install dependencies and work in a virtual environ
 ```
 uv venv --python 3.11 # or greater
 source .venv/bin/activate
-uv pip install --all-extras -r pyproject.toml -e .
+uv sync --all-extras
 ```
 
 Then try running the tests:

pyproject.toml

@@ -17,6 +17,8 @@ dependencies = [
     "PyHyperScattering",
     "pymongo",
     "pyscicat @ git+https://github.com/SciCatProject/pyscicat.git@main",
+    "dataset_metadata_schemas @ git+https://github.com/als-computing/dataset_metadata_schemas.git@main",
+    "dataset_tracker_client @ git+https://github.com/als-computing/dataset_tracker_client.git@main",
     "python-dotenv>=1.2.1",
     "requests",
     "typer",
@@ -37,5 +39,5 @@ requires = [
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools.packages.find]
-where = [".", "src"]
+where = ["src", "."]
 include = ["scicat_beamline*"]

requirements.txt

@@ -10,6 +10,8 @@ pydantic>=2.11
 PyHyperScattering
 pymongo
 pyscicat @ git+https://github.com/SciCatProject/pyscicat.git@main
+dataset_tracker_client @ git+https://github.com/als-computing/dataset_tracker_client.git@main
+dataset_metadata_schemas @ git+https://github.com/als-computing/dataset_metadata_schemas.git@main
 python-dotenv>=1.2.1
 requests
 typer

scripts/733_batch.py

@@ -3,7 +3,6 @@
 from pathlib import Path
 from typing import List
 
-from common_ingester_utils import Issue
 from ingesters import als_733_SAXS
 from pyscicat.client import from_token
 
@@ -12,14 +11,11 @@
 user = sys.argv[3]
 scicat_url_base = sys.argv[4]
 try:
-    issues: List[Issue] = []
     client = from_token(scicat_url_base, token)
     txt_files = folder.glob("**/*.txt")
     with tempfile.TemporaryDirectory() as thumbs_dir:
         for txt_file in list(txt_files):
-            als_733_SAXS.ingest(client, user, txt_file, thumbs_dir, issues)
+            als_733_SAXS.ingest(client, user, txt_file, thumbs_dir)
             print(f"Ingesting {txt_file}")
-            if len(issues) > 0:
-                print(f"    Issues found {[str(issue) for issue in issues]}")
 except Exception as e:
     print(e)

scripts/manual_ingest.py

@@ -1,5 +1,7 @@
 import os
 import pathlib
+from pathlib import Path
+import glob
 
 from dotenv import load_dotenv
 
@@ -22,21 +24,30 @@
 SCICAT_INGEST_OWNER_USERNAME = os.getenv("SCICAT_INGEST_OWNER_USERNAME")
 SCICAT_INGEST_PASSWORD = os.getenv("SCICAT_INGEST_PASSWORD")
 SCICAT_INGEST_SPEC = os.getenv("SCICAT_INGEST_SPEC")
+DATASETTRACKER_URL = os.getenv("DATASETTRACKER_URL")
+DATASETTRACKER_USERNAME = os.getenv("DATASETTRACKER_USERNAME")
+DATASETTRACKER_PASSWORD = os.getenv("DATASETTRACKER_PASSWORD")
+DATASETTRACKER_SHARE_IDENTIFIER = os.getenv("DATASETTRACKER_SHARE_IDENTIFIER")
 
 assert type(SCICAT_INGEST_BASE_FOLDER) == str and len(SCICAT_INGEST_BASE_FOLDER) != 0
 assert type(SCICAT_INGEST_URL) == str and len(SCICAT_INGEST_URL) != 0
 assert type(SCICAT_INGEST_USERNAME) == str and len(SCICAT_INGEST_USERNAME) != 0
 assert type(SCICAT_INGEST_PASSWORD) == str and len(SCICAT_INGEST_PASSWORD) != 0
-assert type(SCICAT_INGEST_OWNER_USERNAME) == str and len(SCICAT_INGEST_OWNER_USERNAME) != 0
 assert type(SCICAT_INGEST_SPEC) == str and len(SCICAT_INGEST_SPEC) != 0
 
-dataset_path = pathlib.Path(SCICAT_INGEST_BASE_FOLDER, SCICAT_INGEST_SUBFOLDER).resolve()
+dataset_path = Path(SCICAT_INGEST_BASE_FOLDER, SCICAT_INGEST_SUBFOLDER).resolve()
+dataset_files = [p for p in map(Path, glob.iglob(str(dataset_path) + "/**", recursive=True))]
 
 ingest(
-    ingester_spec=SCICAT_INGEST_SPEC,
     dataset_path=dataset_path,
+    dataset_files=dataset_files,
+    ingester_spec=SCICAT_INGEST_SPEC,
     owner_username=SCICAT_INGEST_OWNER_USERNAME,
-    base_url=SCICAT_INGEST_URL,
-    username=SCICAT_INGEST_USERNAME,
-    password=SCICAT_INGEST_PASSWORD,
+    scicat_url=SCICAT_INGEST_URL,
+    scicat_username=SCICAT_INGEST_USERNAME,
+    scicat_password=SCICAT_INGEST_PASSWORD,
+    datasettracker_url=DATASETTRACKER_URL,
+    datasettracker_username=DATASETTRACKER_USERNAME,
+    datasettracker_password=DATASETTRACKER_PASSWORD,
+    datasettracker_share_identifier=DATASETTRACKER_SHARE_IDENTIFIER
 )

src/scicat_beamline/_tests/test_utils.py

@@ -1,7 +1,7 @@
 import json
 import numpy as np
 
-from scicat_beamline.utils import NPArrayEncoder, clean_email, UNKNOWN_EMAIL, build_search_terms, calculate_access_controls, Issue
+from scicat_beamline.utils import NPArrayEncoder, clean_email, UNKNOWN_EMAIL, search_terms_from_name, calculate_access_controls
 
 
 def test_clean_email_valid():
@@ -48,8 +48,8 @@ def test_np_encoder():
     assert json.dumps(encoded_np, allow_nan=False)
 
 
-def test_build_search_terms():
-    terms = build_search_terms("Time-is_an illusion. Lunchtime/2x\\so.")
+def test_search_terms_from_name():
+    terms = search_terms_from_name("Time-is_an illusion. Lunchtime/2x\\so.")
     assert "time" in terms
     assert "is" in terms
     assert "an" in terms