From 8b477ccd2477274f9955f77f7b0f65e45b27db47 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 17:44:48 -0400 Subject: [PATCH 01/21] Promote the seqcol app factory into refget --- refget/__init__.py | 4 + refget/app.py | 302 ++++++++++++++++++++++++++++++++ refget/const.py | 28 +++ refget/router.py | 163 ++++++++++------- seqcolapi/const.py | 14 +- seqcolapi/main.py | 111 +++--------- tests/local/test_app_factory.py | 181 +++++++++++++++++++ 7 files changed, 638 insertions(+), 165 deletions(-) create mode 100644 refget/app.py create mode 100644 tests/local/test_app_factory.py diff --git a/refget/__init__.py b/refget/__init__.py index 8fae16e..d3aef04 100644 --- a/refget/__init__.py +++ b/refget/__init__.py @@ -7,7 +7,11 @@ from refget.utils import compare_seqcols, validate_seqcol, seqcol_digest from refget.clients import SequenceCollectionClient, FastaDrsClient from refget.router import create_refget_router + from refget.app import create_seqcol_app, create_store_app, prepare_store from refget.agents import RefgetDBAgent + +`refget.app` and `refget.router` require the optional `fastapi` dependency, so +they are not imported eagerly here. """ from ._version import __version__ diff --git a/refget/app.py b/refget/app.py new file mode 100644 index 0000000..6f2fca5 --- /dev/null +++ b/refget/app.py @@ -0,0 +1,302 @@ +"""Store-backed sequence-collections application factory. + +This module owns the canonical wiring for serving the GA4GH sequence +collections API out of a :class:`~refget.store.RefgetStore`: opening and +loading the store, converting it to a thread-safe readonly snapshot, binding +the backend, mounting the refget router, optional freshness reloading, and a +complete GA4GH ``/service-info`` document. + +**The factory returns a mountable app; it never mutates a caller's app.** + +That is deliberate and load-bearing. :func:`refget.router.setup_backend` stores +the backend on ``app.state.backend`` and :func:`refget.router.get_backend` +reads it back off ``request.app.state`` -- an *app-global* binding, not a +router-scoped one. Including ``create_refget_router()`` twice at two prefixes +of the same app therefore does **not** give you two stores; both mounts resolve +through the same ``app.state.backend``. Returning a self-contained +sub-application instead means each store gets its own ``state.backend``, its +own service-info and its own freshness policy, so a host app can do:: + + host.mount("/jungle", create_seqcol_app(store_path=url_a, remote=True)) + host.mount("/other", create_seqcol_app(store_path=url_b, remote=True)) + +Serving several stores from one process is not a requirement today, and +nothing here implements it. The point is only that the shape does not preclude +it. + +Typical use:: + + from refget.app import create_seqcol_app + + app.mount("/seqcol", create_seqcol_app(store_path=store_url, remote=True)) +""" + +import json +import logging +import os + +from .const import ALL_VERSIONS, SEQCOL_SCHEMA_PATH, SEQCOL_SPEC_VERSION + +_LOGGER = logging.getLogger(__name__) + +DEFAULT_CACHE_DIR = "/tmp/seqcol_cache" +DEFAULT_ORGANIZATION = {"name": "Databio Lab", "url": "https://databio.org"} +DEFAULT_CONTACT_URL = "https://github.com/refgenie/refget/issues" + + +def prepare_store( + store_path: str, + remote: bool = False, + cache_dir: str = DEFAULT_CACHE_DIR, +): + """Open a RefgetStore and return a fully-loaded ReadonlyRefgetStore. + + Loads every collection and then converts to a readonly snapshot, because + concurrent HTTP reads must borrow the store immutably -- a mutable + ``RefgetStore`` lazy-loads through a mutable borrow (see + ``refget.backend.RefgetStoreBackend``). + + Note that ``into_readonly()`` **consumes** the receiver: the pyo3 binding + does ``std::mem::replace(&mut self.inner, RefgetStore::in_memory())``, so + the ``RefgetStore`` object handed in is left empty. Never pass a store that + someone else still holds a reference to -- open a second one instead. + + Args: + store_path: Path to a store on disk, or a URL for a remote store. + remote: If True, open as a remote store. + cache_dir: Local cache directory used for remote stores. + + Returns: + A ReadonlyRefgetStore suitable for concurrent serving. + """ + from .store import RefgetStore + + if remote: + store = RefgetStore.open_remote(cache_dir, store_path) + else: + store = RefgetStore.on_disk(store_path) + + store.load_all_collections() + return store.into_readonly() + + +def load_seqcol_schema() -> dict | None: + """Load the canonical seqcol JSON schema shipped with the package. + + Resolved through ``refget.const.SEQCOL_SCHEMA_PATH`` rather than any + repo-relative path, so it works from site-packages in a deployed image. + """ + try: + with open(SEQCOL_SCHEMA_PATH) as f: + return json.load(f) + except Exception as e: + _LOGGER.warning(f"Could not load seqcol schema from {SEQCOL_SCHEMA_PATH}: {e}") + return None + + +def store_service_info( + *, + service_info_id: str, + service_info_name: str, + store_url: str | None, + capabilities: dict | None = None, + description: str | None = None, + organization: dict | None = None, + contact_url: str | None = None, + documentation_url: str | None = None, + extra_seqcol: dict | None = None, +) -> dict: + """Build the GA4GH service-info body for a store-backed seqcol service. + + Args: + service_info_id: Reverse-domain service id, e.g. ``org.refgenie.seqcol``. + service_info_name: Human-readable service name. + store_url: The publicly reachable RefgetStore URL to advertise. The + ``REFGET_STORE_HTTP_URL`` environment variable overrides it, for + deployments whose internal store path is not publicly fetchable. + capabilities: Backend capabilities dict (``backend.capabilities()``). + extra_seqcol: Extra keys merged into the ``seqcol`` block (seqcolapi + passes its ``scom`` block through here). + + Returns: + A JSON-serializable service-info dict carrying ``seqcol.refget_store.url``. + """ + caps = capabilities or {} + advertised = os.environ.get("REFGET_STORE_HTTP_URL", store_url) + + seqcol: dict = {"schema": load_seqcol_schema()} + if advertised: + seqcol["refget_store"] = {"enabled": True, "url": advertised, **caps} + else: + seqcol["refget_store"] = {"enabled": False} + seqcol["aliases"] = { + "enabled": bool( + caps.get("collection_alias_namespaces") or caps.get("sequence_alias_namespaces") + ) + } + seqcol["fhr_metadata"] = {"enabled": bool(caps.get("fhr_metadata_collections"))} + if extra_seqcol: + seqcol.update(extra_seqcol) + + info = { + "id": service_info_id, + "name": service_info_name, + "type": { + "group": "org.ga4gh", + "artifact": "refget-seqcol", + "version": SEQCOL_SPEC_VERSION, + }, + "description": description + or "Store-backed API providing metadata for collections of reference sequences", + "organization": organization or DEFAULT_ORGANIZATION, + "contactUrl": contact_url or DEFAULT_CONTACT_URL, + "version": ALL_VERSIONS, + "seqcol": seqcol, + } + if documentation_url: + info["documentationUrl"] = documentation_url + return info + + +def create_seqcol_app( + store=None, + *, + store_path: str | None = None, + remote: bool = False, + cache_dir: str = DEFAULT_CACHE_DIR, + store_url: str | None = None, + service_info_id: str = "org.ga4gh.seqcol.store", + service_info_name: str = "Sequence collections (store-backed)", + description: str | None = None, + organization: dict | None = None, + contact_url: str | None = None, + documentation_url: str | None = None, + service_info_extra=None, + sequences: bool = False, + collections: bool = True, + pangenomes: bool = False, + fasta_drs: bool = False, + compliance: bool = True, + freshness: bool | None = None, + freshness_interval: int = 300, + cors: bool = True, + title: str = "Sequence Collections API (Store-backed)", +): + """Create a self-contained, mountable seqcol app served from a RefgetStore. + + The returned app owns its own ``state.backend``, so mounting several of + them at different prefixes yields several independent stores. Do not pass + the returned app to ``setup_backend`` again. + + Args: + store: An already-prepared ReadonlyRefgetStore. If omitted, + ``store_path`` is opened via :func:`prepare_store`. + store_path: Store path/URL to open when ``store`` is not given. + remote: Open ``store_path`` as a remote store. + cache_dir: Local cache directory for remote stores. + store_url: Store URL to advertise in service-info. Defaults to + ``store_path`` when the store is remote. + service_info_id: Reverse-domain id for this service instance. + service_info_name: Human-readable name for this service instance. + service_info_extra: Extra ``seqcol`` keys for service-info. Either a + dict or a zero-argument callable evaluated per request (seqcolapi + uses the callable form because its SCOM digests load lazily). + freshness: Attach ``StoreFreshnessMiddleware`` so the app picks up a + republished store without a restart. Defaults to ``remote``. + cors: Add a permissive CORS middleware. Set False when the host app + already installs one. + + Returns: + A FastAPI application ready to serve standalone or to ``app.mount()``. + """ + from fastapi import FastAPI + + from .router import create_refget_router, setup_backend + + if store is None: + if store_path is None: + raise ValueError("create_seqcol_app requires either `store` or `store_path`") + store = prepare_store(store_path, remote=remote, cache_dir=cache_dir) + + if store_url is None and remote: + store_url = store_path + + app = FastAPI(title=title, version=ALL_VERSIONS["refget_version"]) + + if cors: + from fastapi.middleware.cors import CORSMiddleware + + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + setup_backend(app, store=store) + app.include_router( + create_refget_router( + sequences=sequences, + collections=collections, + pangenomes=pangenomes, + fasta_drs=fasta_drs, + compliance=compliance, + refget_store_url=store_url, + ) + ) + + if freshness is None: + freshness = remote + if freshness: + if not store_path: + raise ValueError("freshness requires `store_path` to poll for rgstore.json") + from .middleware import StoreFreshnessMiddleware + + app.add_middleware( + StoreFreshnessMiddleware, + store_url=store_path, + cache_dir=cache_dir, + check_interval=freshness_interval, + ) + + @app.get("/service-info", summary="GA4GH service info", tags=["General endpoints"]) + async def service_info(): + backend = getattr(app.state, "backend", None) + caps = backend.capabilities() if backend and hasattr(backend, "capabilities") else {} + extra = service_info_extra() if callable(service_info_extra) else service_info_extra + return store_service_info( + service_info_id=service_info_id, + service_info_name=service_info_name, + store_url=store_url, + capabilities=caps, + description=description, + organization=organization, + contact_url=contact_url, + documentation_url=documentation_url, + extra_seqcol=extra, + ) + + return app + + +def create_store_app( + store_path: str, + remote: bool = False, + cache_dir: str = DEFAULT_CACHE_DIR, + **kwargs, +): + """Create a standalone store-backed seqcol app (no database). + + Thin convenience wrapper around :func:`create_seqcol_app` for serving a + single store at the root of its own process. + + Args: + store_path: Path to store on disk, or S3/HTTP URL for remote stores. + remote: If True, open as a remote store. + cache_dir: Local cache directory for remote stores. + + Returns: + FastAPI app with store-backed seqcol endpoints at the root. + """ + return create_seqcol_app(store_path=store_path, remote=remote, cache_dir=cache_dir, **kwargs) diff --git a/refget/const.py b/refget/const.py index 66f3175..a16f43d 100644 --- a/refget/const.py +++ b/refget/const.py @@ -1,8 +1,36 @@ import logging import os +from platform import python_version _LOGGER = logging.getLogger(__name__) +# Version of the GA4GH sequence collections specification this package implements. +SEQCOL_SPEC_VERSION = "1.0.0" + + +def _gtars_version() -> str: + try: + from gtars import __version__ as gtars_version + + return gtars_version + except Exception: # pragma: no cover - gtars is an optional dependency + return "not installed" + + +def _all_versions() -> dict: + """Version block advertised in GA4GH service-info documents.""" + from refget._version import __version__ as refget_version + + return { + "refget_version": refget_version, + "gtars_version": _gtars_version(), + "python_version": python_version(), + "seqcol_spec_version": SEQCOL_SPEC_VERSION, + } + + +ALL_VERSIONS = _all_versions() + def _schema_path(name): return os.path.join(SCHEMA_FILEPATH, name) diff --git a/refget/router.py b/refget/router.py index ce1ba1d..18bed77 100644 --- a/refget/router.py +++ b/refget/router.py @@ -17,6 +17,13 @@ For concurrent serving, ``store`` should be a fully loaded ReadonlyRefgetStore (see refget.store), obtained via ``RefgetStore.into_readonly()``. + +Note that ``setup_backend`` binds the backend to ``app.state``, which +``get_backend`` reads back off ``request.app.state`` -- an app-global binding. +Including this router twice at two prefixes of the same app therefore serves +the *same* store from both. To serve a store from a sub-path (or to leave room +for serving several stores), prefer ``refget.app.create_seqcol_app()``, which +returns a self-contained app to mount. """ import logging @@ -74,6 +81,88 @@ async def get_dbagent(request: Request): return dbagent +def _self_target_url(request: Request, mount_prefix: str = "") -> str: + """Best-effort public base URL of the seqcol service handling this request. + + The compliance runner needs the URL of the *seqcol* service, not of the + server root. Two things can put the seqcol API below the root: + + * an ASGI mount or a reverse-proxy root path, which shows up in + ``scope["root_path"]`` (so mounting this router's app at ``/seqcol`` + is handled automatically), and + * ``include_router(router, prefix="/seqcol")``, which does not, and so has + to be declared via ``create_refget_router(mount_prefix="/seqcol")``. + """ + scheme = request.headers.get("x-forwarded-proto", request.url.scheme) + host = request.headers.get("host", request.url.netloc) + root_path = request.scope.get("root_path", "") or "" + return f"{scheme}://{host}{root_path.rstrip('/')}{mount_prefix.rstrip('/')}" + + +def _create_compliance_router(mount_prefix: str = "") -> APIRouter: + """Build the compliance router, closing over the router's mount prefix.""" + router = APIRouter() + + @router.get( + "/compliance/run", + summary="Run compliance checks against a seqcol server", + tags=["Compliance"], + ) + def run_compliance_endpoint( + request: Request, + target_url: str | None = Query( + None, description="Target server URL to test (defaults to self)" + ), + ): + """ + Run GA4GH SeqCol compliance structure tests against a server. + + Only runs structure tests (service-info, list, pagination, collection structure). + Content tests that require specific test data are not included. + + If no target_url is provided, tests run against this server. + """ + from .compliance import run_compliance + + if target_url is None: + target_url = _self_target_url(request, mount_prefix) + + return run_compliance(target_url) + + @router.get( + "/compliance/stream", + summary="Stream compliance checks via Server-Sent Events", + tags=["Compliance"], + ) + def stream_compliance_endpoint( + request: Request, + target_url: str | None = Query( + None, description="Target server URL to test (defaults to self)" + ), + ): + """ + Stream compliance check results in real-time via Server-Sent Events. + + Each event contains a JSON object with type "start", "result", or "done". + """ + from .compliance import run_compliance_stream + + if target_url is None: + target_url = _self_target_url(request, mount_prefix) + + def event_stream(): + for data in run_compliance_stream(target_url): + yield f"data: {data}\n\n" + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + return router + + def create_refget_router( sequences: bool = False, collections: bool = True, @@ -81,6 +170,7 @@ def create_refget_router( fasta_drs: bool = False, compliance: bool = True, refget_store_url: str = None, + mount_prefix: str = "", ) -> APIRouter: """ Create a FastAPI router for the sequence collection API. @@ -94,6 +184,11 @@ def create_refget_router( pangenomes (bool): Include pangenome endpoints fasta_drs (bool): Include FASTA DRS endpoints refget_store_url (str): URL of backing RefgetStore (e.g., s3://bucket/store/) + mount_prefix (str): The path prefix this router will be included under, + when it is included with ``include_router(..., prefix=...)`` rather + than mounted as a sub-application. Only used so the compliance + endpoints self-target the seqcol service instead of the server root. + Leave empty when mounting an app (``scope["root_path"]`` covers it). Returns: (APIRouter): A FastAPI router with the specified endpoints @@ -122,7 +217,7 @@ def create_refget_router( refget_router.include_router(fasta_drs_router, prefix="/fasta") if compliance: _LOGGER.info("Adding compliance endpoints...") - refget_router.include_router(compliance_router) + refget_router.include_router(_create_compliance_router(mount_prefix)) return refget_router @@ -675,69 +770,3 @@ async def get_fasta_index( } except ValueError: raise HTTPException(status_code=404, detail="Object not found") - - -compliance_router = APIRouter() - - -@compliance_router.get( - "/compliance/run", - summary="Run compliance checks against a seqcol server", - tags=["Compliance"], -) -def run_compliance_endpoint( - request: Request, - target_url: str | None = Query( - None, description="Target server URL to test (defaults to self)" - ), -): - """ - Run GA4GH SeqCol compliance structure tests against a server. - - Only runs structure tests (service-info, list, pagination, collection structure). - Content tests that require specific test data are not included. - - If no target_url is provided, tests run against this server. - """ - from .compliance import run_compliance - - if target_url is None: - scheme = request.headers.get("x-forwarded-proto", request.url.scheme) - host = request.headers.get("host", request.url.netloc) - target_url = f"{scheme}://{host}" - - return run_compliance(target_url) - - -@compliance_router.get( - "/compliance/stream", - summary="Stream compliance checks via Server-Sent Events", - tags=["Compliance"], -) -def stream_compliance_endpoint( - request: Request, - target_url: str | None = Query( - None, description="Target server URL to test (defaults to self)" - ), -): - """ - Stream compliance check results in real-time via Server-Sent Events. - - Each event contains a JSON object with type "start", "result", or "done". - """ - from .compliance import run_compliance_stream - - if target_url is None: - scheme = request.headers.get("x-forwarded-proto", request.url.scheme) - host = request.headers.get("host", request.url.netloc) - target_url = f"{scheme}://{host}" - - def event_stream(): - for data in run_compliance_stream(target_url): - yield f"data: {data}\n\n" - - return StreamingResponse( - event_stream(), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, - ) diff --git a/seqcolapi/const.py b/seqcolapi/const.py index 34e6aac..6b03ef6 100644 --- a/seqcolapi/const.py +++ b/seqcolapi/const.py @@ -1,15 +1,9 @@ import os -from platform import python_version -from gtars import __version__ as gtars_version +# ALL_VERSIONS moved into the distributed `refget` package (refget/const.py) so +# the shared store-backed app factory can build service-info without seqcolapi, +# which is not shipped in the refget wheel. Re-exported here for compatibility. +from refget.const import ALL_VERSIONS # noqa: F401 -from refget._version import __version__ as refget_version - -ALL_VERSIONS = { - "refget_version": refget_version, - "gtars_version": gtars_version, - "python_version": python_version(), - "seqcol_spec_version": "1.0.0", -} STATIC_DIRNAME = "static" STATIC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), STATIC_DIRNAME) diff --git a/seqcolapi/main.py b/seqcolapi/main.py index e61c243..3b0287d 100644 --- a/seqcolapi/main.py +++ b/seqcolapi/main.py @@ -240,6 +240,11 @@ async def service_info(): def create_store_app(store_path: str, remote: bool = False, cache_dir: str = "/tmp/seqcol_cache"): """Create a seqcolapi FastAPI app backed by a RefgetStore (no database). + Thin wrapper over ``refget.app.create_seqcol_app``, which owns the shared + store-backed wiring (readonly store, backend, router, freshness, GA4GH + service-info). Everything seqcolapi adds on top is SCOM, which is not a + refget concern, so it is injected through ``service_info_extra``. + Args: store_path: Path to store on disk, or S3 URL for remote stores. remote: If True, open as a remote (S3) store. @@ -248,101 +253,31 @@ def create_store_app(store_path: str, remote: bool = False, cache_dir: str = "/t Returns: FastAPI app with store-backed seqcol endpoints. """ - from refget.store import RefgetStore - - if remote: - store = RefgetStore.open_remote(cache_dir, store_path) - else: - store = RefgetStore.on_disk(store_path) - - # Load all collections and convert to a thread-safe ReadonlyRefgetStore so - # concurrent HTTP reads borrow immutably (no mutable lazy-loading borrow). - store.load_all_collections() - store = store.into_readonly() - - store_app = FastAPI( - title="Sequence Collections API (Store-backed)", - version=ALL_VERSIONS["refget_version"], - ) - - store_app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ) - - setup_backend(store_app, store=store) - router = create_refget_router( - sequences=False, pangenomes=False, refget_store_url=store_path if remote else None - ) - store_app.include_router(router) + from refget.app import create_seqcol_app # Load SCOM config: check SCOM_CONFIG_URL env var, then fall back to store convention _load_scom_config(store_path, remote) - if remote: - from refget.middleware import StoreFreshnessMiddleware - - store_app.add_middleware( - StoreFreshnessMiddleware, - store_url=store_path, - cache_dir=cache_dir, - ) - - @store_app.get("/service-info", summary="GA4GH service info", tags=["General endpoints"]) - async def store_service_info(): - import json as _json - - backend = getattr(store_app.state, "backend", None) - caps = backend.capabilities() if backend and hasattr(backend, "capabilities") else {} - - # Load the seqcol schema (same schema used by the DB-backed app). - # Resolve via the installed refget package, not a dev-relative path — - # in the deployed image refget lives in site-packages, not ../refget. - from refget.const import SEQCOL_SCHEMA_PATH - - try: - with open(SEQCOL_SCHEMA_PATH) as _f: - schema = _json.load(_f) - except Exception: - schema = None - + def _scom_block(): + # Evaluated per request, because _SAMPLE_DIGESTS can be repopulated. return { - "id": "org.databio.seqcolapi.store", - "name": "Sequence collections (store-backed)", - "type": { - "group": "org.ga4gh", - "artifact": "refget-seqcol", - "version": ALL_VERSIONS["seqcol_spec_version"], - }, - "description": "Store-backed API providing metadata for collections of reference sequences", - "organization": {"name": "Databio Lab", "url": "https://databio.org"}, - "contactUrl": "https://github.com/refgenie/refget/issues", - "version": ALL_VERSIONS, - "seqcol": { - "schema": schema, - "refget_store": { - "enabled": True, - "url": os.environ.get("REFGET_STORE_HTTP_URL", store_path), - **caps, - }, - "aliases": { - "enabled": bool( - caps.get("collection_alias_namespaces") - or caps.get("sequence_alias_namespaces") - ) - }, - "fhr_metadata": {"enabled": bool(caps.get("fhr_metadata_collections"))}, - "scom": { - "enabled": bool(_SAMPLE_DIGESTS), - "species": list(_SAMPLE_DIGESTS.keys()), - }, - }, + "scom": { + "enabled": bool(_SAMPLE_DIGESTS), + "species": list(_SAMPLE_DIGESTS.keys()), + } } - return store_app + return create_seqcol_app( + store_path=store_path, + remote=remote, + cache_dir=cache_dir, + # Advertise the store path (REFGET_STORE_HTTP_URL overrides it inside + # store_service_info), matching the pre-extraction service-info exactly. + store_url=store_path, + service_info_id="org.databio.seqcolapi.store", + service_info_name="Sequence collections (store-backed)", + service_info_extra=_scom_block, + ) _STORE_URL_ENV = os.environ.get("REFGET_STORE_URL") diff --git a/tests/local/test_app_factory.py b/tests/local/test_app_factory.py new file mode 100644 index 0000000..7145e51 --- /dev/null +++ b/tests/local/test_app_factory.py @@ -0,0 +1,181 @@ +"""Tests for the shared store-backed seqcol app factory (refget.app). + +Covers the three properties the factory exists to guarantee: + +- it returns a *mountable* app whose backend is its own, so two mounts are two + independent stores (the multi-store door the ADR asks us not to close), +- its GA4GH service-info carries ``seqcol.refget_store.url`` plus the seqcol + JSON schema, and +- the compliance endpoints self-target the seqcol service, not the server root. +""" + +import json +from pathlib import Path + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +try: + from refget.store import RefgetStore + + _RUST_BINDINGS_AVAILABLE = True +except ImportError: + _RUST_BINDINGS_AVAILABLE = False + +from refget.app import create_seqcol_app, store_service_info +from refget.router import create_refget_router, setup_backend + +TEST_FASTA_DIR = Path("test_fasta") +BASE_FASTA = TEST_FASTA_DIR / "base.fa" + +with open(TEST_FASTA_DIR / "test_fasta_digests.json") as fp: + TEST_DIGESTS = json.load(fp) + +BASE_DIGEST = TEST_DIGESTS["base.fa"]["top_level_digest"] + +pytestmark = pytest.mark.skipif(not _RUST_BINDINGS_AVAILABLE, reason="gtars is not installed") + + +def _readonly_store(): + store = RefgetStore.in_memory() + store.add_sequence_collection_from_fasta(str(BASE_FASTA)) + store.load_all_collections() + return store.into_readonly() + + +def _app(**kwargs): + kwargs.setdefault("store", _readonly_store()) + kwargs.setdefault("cors", False) + return create_seqcol_app(**kwargs) + + +class TestServiceInfo: + def test_advertises_store_url(self): + client = TestClient(_app(store_url="https://example.com/store/")) + info = client.get("/service-info").json() + assert info["seqcol"]["refget_store"]["enabled"] is True + assert info["seqcol"]["refget_store"]["url"] == "https://example.com/store/" + + def test_carries_seqcol_schema_and_versions(self): + client = TestClient(_app(store_url="https://example.com/store/")) + info = client.get("/service-info").json() + assert isinstance(info["seqcol"]["schema"], dict) + assert info["version"]["refget_version"] + assert info["type"] == { + "group": "org.ga4gh", + "artifact": "refget-seqcol", + "version": "1.0.0", + } + + def test_identity_is_caller_supplied(self): + client = TestClient( + _app( + store_url="https://example.com/store/", + service_info_id="org.refgenie.seqcol", + service_info_name="Refgenie Sequence Collections", + ) + ) + info = client.get("/service-info").json() + assert info["id"] == "org.refgenie.seqcol" + assert info["name"] == "Refgenie Sequence Collections" + + def test_no_store_url_reports_disabled(self): + client = TestClient(_app()) + assert client.get("/service-info").json()["seqcol"]["refget_store"] == {"enabled": False} + + def test_http_url_env_override(self, monkeypatch): + monkeypatch.setenv("REFGET_STORE_HTTP_URL", "https://public.example.com/store/") + client = TestClient(_app(store_url="/local/on/disk")) + info = client.get("/service-info").json() + assert info["seqcol"]["refget_store"]["url"] == "https://public.example.com/store/" + + def test_extra_seqcol_block_can_be_a_callable(self): + client = TestClient( + _app( + store_url="https://example.com/store/", + service_info_extra=lambda: {"scom": {"enabled": True, "species": ["human"]}}, + ) + ) + assert client.get("/service-info").json()["seqcol"]["scom"]["species"] == ["human"] + + def test_store_service_info_builder_is_json_serializable(self): + body = store_service_info( + service_info_id="org.test", + service_info_name="Test", + store_url="https://example.com/store/", + ) + json.dumps(body) + + +class TestMountable: + """The factory must return an app, not mutate one -- see ADR decision C.""" + + def test_serves_collections_when_mounted_under_a_prefix(self): + host = FastAPI() + host.mount("/seqcol", _app(store_url="https://a/store/")) + client = TestClient(host) + assert client.get("/seqcol/list/collection").status_code == 200 + assert client.get(f"/seqcol/collection/{BASE_DIGEST}").status_code == 200 + + def test_two_mounts_are_two_independent_stores(self): + host = FastAPI() + host.mount("/a", _app(store_url="https://a/store/", service_info_id="org.test.a")) + host.mount("/b", _app(store_url="https://b/store/", service_info_id="org.test.b")) + client = TestClient(host) + + a = client.get("/a/service-info").json() + b = client.get("/b/service-info").json() + assert a["id"] == "org.test.a" + assert b["id"] == "org.test.b" + assert a["seqcol"]["refget_store"]["url"] == "https://a/store/" + assert b["seqcol"]["refget_store"]["url"] == "https://b/store/" + + def test_host_app_keeps_its_own_root_service_info(self): + host = FastAPI() + + @host.get("/service-info") + def root_service_info(): + return {"id": "org.test.host"} + + host.mount("/seqcol", _app(store_url="https://a/store/")) + client = TestClient(host) + assert client.get("/service-info").json() == {"id": "org.test.host"} + assert client.get("/seqcol/service-info").json()["seqcol"]["refget_store"]["url"] + + def test_factory_does_not_touch_the_host_app_state(self): + host = FastAPI() + host.mount("/seqcol", _app(store_url="https://a/store/")) + assert not hasattr(host.state, "backend") + + +class TestComplianceSelfTarget: + def test_mounted_app_targets_the_mount_path(self): + host = FastAPI() + host.mount("/seqcol", _app(store_url="https://a/store/")) + result = TestClient(host).get("/seqcol/compliance/run").json() + assert result["server_url"] == "http://testserver/seqcol" + + def test_included_router_targets_the_declared_mount_prefix(self): + app = FastAPI() + setup_backend(app, store=_readonly_store()) + app.include_router(create_refget_router(mount_prefix="/seqcol"), prefix="/seqcol") + result = TestClient(app).get("/seqcol/compliance/run").json() + assert result["server_url"] == "http://testserver/seqcol" + + def test_root_mounted_router_still_targets_the_root(self): + app = FastAPI() + setup_backend(app, store=_readonly_store()) + app.include_router(create_refget_router()) + result = TestClient(app).get("/compliance/run").json() + assert result["server_url"] == "http://testserver" + + def test_explicit_target_url_wins(self): + host = FastAPI() + host.mount("/seqcol", _app(store_url="https://a/store/")) + result = ( + TestClient(host) + .get("/seqcol/compliance/run", params={"target_url": "https://elsewhere.example.com"}) + .json() + ) + assert result["server_url"] == "https://elsewhere.example.com" From ad8a07b1986a2bc35c3587ff986e77fcb8bf87bf Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 17:44:48 -0400 Subject: [PATCH 02/21] Let a host mount routes, bind store later --- refget/app.py | 16 +++++++++++++--- tests/local/test_app_factory.py | 34 +++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/refget/app.py b/refget/app.py index 6f2fca5..63cf369 100644 --- a/refget/app.py +++ b/refget/app.py @@ -180,6 +180,7 @@ def create_seqcol_app( freshness: bool | None = None, freshness_interval: int = 300, cors: bool = True, + defer_backend: bool = False, title: str = "Sequence Collections API (Store-backed)", ): """Create a self-contained, mountable seqcol app served from a RefgetStore. @@ -205,6 +206,12 @@ def create_seqcol_app( republished store without a restart. Defaults to ``remote``. cors: Add a permissive CORS middleware. Set False when the host app already installs one. + defer_backend: Build the routes now but do not open or bind a store. + The caller must then call ``refget.router.setup_backend(seqcol_app, + store=...)`` before the first request. Mounted sub-applications do + not receive their own lifespan events, so a host app that wants to + open the store on startup rather than at import time has to mount + the routes early and bind the backend from *its* lifespan. Returns: A FastAPI application ready to serve standalone or to ``app.mount()``. @@ -213,9 +220,11 @@ def create_seqcol_app( from .router import create_refget_router, setup_backend - if store is None: + if store is None and not defer_backend: if store_path is None: - raise ValueError("create_seqcol_app requires either `store` or `store_path`") + raise ValueError( + "create_seqcol_app requires `store`, `store_path`, or defer_backend=True" + ) store = prepare_store(store_path, remote=remote, cache_dir=cache_dir) if store_url is None and remote: @@ -234,7 +243,8 @@ def create_seqcol_app( allow_headers=["*"], ) - setup_backend(app, store=store) + if store is not None: + setup_backend(app, store=store) app.include_router( create_refget_router( sequences=sequences, diff --git a/tests/local/test_app_factory.py b/tests/local/test_app_factory.py index 7145e51..eb8deab 100644 --- a/tests/local/test_app_factory.py +++ b/tests/local/test_app_factory.py @@ -10,6 +10,7 @@ """ import json +from contextlib import asynccontextmanager from pathlib import Path import pytest @@ -149,6 +150,39 @@ def test_factory_does_not_touch_the_host_app_state(self): assert not hasattr(host.state, "backend") +class TestDeferredBackend: + """A mounted sub-app gets no lifespan events, so the host binds its store.""" + + def test_routes_exist_before_a_store_is_bound(self): + seqcol = create_seqcol_app(defer_backend=True, store_url="https://a/store/", cors=False) + assert not hasattr(seqcol.state, "backend") + assert "/list/collection" in {r.path for r in seqcol.routes} + # service-info still answers; it just reports no backend capabilities. + info = TestClient(seqcol).get("/service-info").json() + assert info["seqcol"]["refget_store"]["url"] == "https://a/store/" + assert "n_collections" not in info["seqcol"]["refget_store"] + + def test_host_can_bind_the_store_later_from_its_own_lifespan(self): + seqcol = create_seqcol_app(defer_backend=True, store_url="https://a/store/", cors=False) + + @asynccontextmanager + async def lifespan(app): + setup_backend(seqcol, store=_readonly_store()) + yield + + host = FastAPI(lifespan=lifespan) + host.mount("/seqcol", seqcol) + + with TestClient(host) as client: + assert client.get("/seqcol/list/collection").status_code == 200 + caps = client.get("/seqcol/service-info").json()["seqcol"]["refget_store"] + assert caps["n_collections"] == 1 + + def test_no_store_and_no_defer_is_an_error(self): + with pytest.raises(ValueError, match="defer_backend"): + create_seqcol_app() + + class TestComplianceSelfTarget: def test_mounted_app_targets_the_mount_path(self): host = FastAPI() From e9be560b23c5684ed270caf4225a878b6611e31f Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 17:44:48 -0400 Subject: [PATCH 03/21] Move seqcolapi into the refget package --- README.md | 9 +- pyproject.toml | 12 +- refget/__init__.py | 9 +- refget/router.py | 4 +- refget/seqcolapi/__init__.py | 60 ++++ refget/seqcolapi/__main__.py | 10 + refget/{ => seqcolapi}/app.py | 24 +- refget/seqcolapi/const.py | 10 + refget/seqcolapi/examples.py | 143 +++++++++ refget/seqcolapi/main.py | 300 ++++++++++++++++++ .../seqcolapi}/static/css/databio.css | 0 .../seqcolapi}/static/css/style.css | 0 .../seqcolapi}/static/favicon.ico | Bin .../seqcolapi}/static/index.html | 0 .../seqcolapi}/static/logo_databio_long.svg | 0 .../seqcolapi}/static/seqcol_logo.svg | 0 seqcolapi/__init__.py | 13 + seqcolapi/__main__.py | 11 +- seqcolapi/const.py | 14 +- seqcolapi/examples.py | 144 +-------- seqcolapi/main.py | 299 +---------------- tests/integration/conftest.py | 6 +- tests/local/test_app_factory.py | 4 +- tests/local/test_import_gating.py | 75 +++++ 24 files changed, 677 insertions(+), 470 deletions(-) create mode 100644 refget/seqcolapi/__init__.py create mode 100644 refget/seqcolapi/__main__.py rename refget/{ => seqcolapi}/app.py (95%) create mode 100644 refget/seqcolapi/const.py create mode 100644 refget/seqcolapi/examples.py create mode 100644 refget/seqcolapi/main.py rename {seqcolapi => refget/seqcolapi}/static/css/databio.css (100%) rename {seqcolapi => refget/seqcolapi}/static/css/style.css (100%) rename {seqcolapi => refget/seqcolapi}/static/favicon.ico (100%) rename {seqcolapi => refget/seqcolapi}/static/index.html (100%) rename {seqcolapi => refget/seqcolapi}/static/logo_databio_long.svg (100%) rename {seqcolapi => refget/seqcolapi}/static/seqcol_logo.svg (100%) create mode 100644 tests/local/test_import_gating.py diff --git a/README.md b/README.md index bebe9b8..57b59f4 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,11 @@ User-facing documentation is hosted at [refgenie.org/refget](https://refgenie.or This repository includes: 1. `/refget`: The `refget` Python package, which provides a Python interface to both remote and local use of refget standards. It has clients and functions for both refget sequences and refget sequence collections (seqcol). -2. `/seqcolapi`: Sequence collections API software, a FastAPI wrapper built on top of the `refget` package. It provides a bare-bones Sequence Collections API service. -3. `/deployment`: Server configurations for demo instances and public deployed instances. There are also github workflows (in `.github/workflows`) that deploy the demo server instance from this repository. -4. `/test_fasta` and `/test_api`: Dummy data and a compliance test, to test external implementations of the Refget Sequence Collections API. -5. `/frontend`: a React seqcolapi front-end. +2. `/refget/seqcolapi`: Sequence collections API software, a FastAPI wrapper built on top of the `refget` package. It provides a bare-bones Sequence Collections API service. It ships in the `refget` wheel, but its dependencies do not: install them with `pip install 'refget[seqcolapi]'`. Nothing on the plain `import refget` path imports this subpackage, so a base install never pays for fastapi/uvicorn/sqlmodel/psycopg2. +3. `/seqcolapi`: A thin compatibility shim re-exporting `refget.seqcolapi`, kept so existing deployments that run `uvicorn seqcolapi.main:app` (or `:store_app`) keep working. New code should import `refget.seqcolapi`. +4. `/deployment`: Server configurations for demo instances and public deployed instances. There are also github workflows (in `.github/workflows`) that deploy the demo server instance from this repository. +5. `/test_fasta` and `/test_api`: Dummy data and a compliance test, to test external implementations of the Refget Sequence Collections API. +6. `/frontend`: a React seqcolapi front-end. ## Deploy to AWS ECS diff --git a/pyproject.toml b/pyproject.toml index a704e3c..06f9845 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,12 +23,16 @@ classifiers = [ "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", ] +# Base install must stay light: no fastapi, no uvicorn, no sqlmodel, no +# psycopg2. Those are service/database dependencies, gated behind the +# `seqcolapi` extra and behind the refget.seqcolapi / refget.agents / +# refget.models module boundaries -- nothing on the `import refget` path +# touches them. dependencies = [ "gtars>=0.9.0", "jsonschema", "pyyaml", "requests", - "sqlmodel", "tomli_w", "typer>=0.9.0", ] @@ -37,7 +41,11 @@ dependencies = [ refget = "refget.cli:main" [project.optional-dependencies] -test = ["pytest", "pytest-cov>=6.0.0", "fastapi", "httpx"] +test = ["pytest", "pytest-cov>=6.0.0", "fastapi", "httpx", "sqlmodel"] +# Everything needed to run the services in refget/seqcolapi/. psycopg2-binary is +# only used by the PostgreSQL-backed app (refget.seqcolapi.main:app); the +# store-backed app needs none of it. Splitting it into its own extra is a +# judgement call left to the maintainer. seqcolapi = [ "fastapi", "psycopg2-binary", diff --git a/refget/__init__.py b/refget/__init__.py index d3aef04..51a2aa4 100644 --- a/refget/__init__.py +++ b/refget/__init__.py @@ -7,11 +7,14 @@ from refget.utils import compare_seqcols, validate_seqcol, seqcol_digest from refget.clients import SequenceCollectionClient, FastaDrsClient from refget.router import create_refget_router - from refget.app import create_seqcol_app, create_store_app, prepare_store + from refget.seqcolapi import create_seqcol_app, create_store_app, prepare_store from refget.agents import RefgetDBAgent -`refget.app` and `refget.router` require the optional `fastapi` dependency, so -they are not imported eagerly here. +`refget.seqcolapi` (the API service) and `refget.router` require the optional +`fastapi` dependency; `refget.agents` and `refget.models` require `sqlmodel`. +None of them are imported here, and nothing on this base-install path imports +them, so `pip install refget` never pays for them. Install the service with +`pip install 'refget[seqcolapi]'`. """ from ._version import __version__ diff --git a/refget/router.py b/refget/router.py index 18bed77..cb86477 100644 --- a/refget/router.py +++ b/refget/router.py @@ -22,8 +22,8 @@ ``get_backend`` reads back off ``request.app.state`` -- an app-global binding. Including this router twice at two prefixes of the same app therefore serves the *same* store from both. To serve a store from a sub-path (or to leave room -for serving several stores), prefer ``refget.app.create_seqcol_app()``, which -returns a self-contained app to mount. +for serving several stores), prefer ``refget.seqcolapi.create_seqcol_app()``, +which returns a self-contained app to mount. """ import logging diff --git a/refget/seqcolapi/__init__.py b/refget/seqcolapi/__init__.py new file mode 100644 index 0000000..5d23cca --- /dev/null +++ b/refget/seqcolapi/__init__.py @@ -0,0 +1,60 @@ +"""Sequence Collections API service — the optional, server-side half of refget. + +Everything in this subpackage requires the optional service dependencies +(``pip install 'refget[seqcolapi]'``). They are imported here at module level, +in ordinary Python style, because **the package boundary is the gate**: nothing +in ``refget/__init__.py`` — or on any other base-install code path — imports +``refget.seqcolapi``. Either you asked for the service and the whole subpackage +loads, or it is never loaded at all. There is no middle state, and no +function-local imports scattered through the service code to maintain. + +Two applications ship here: + +* :mod:`refget.seqcolapi.main` — the PostgreSQL-backed ``app`` that runs + seqcolapi.databio.org, plus the ``store_app`` built from the environment. +* :func:`create_seqcol_app` — the store-backed factory, which returns a + self-contained, mountable app served out of a + :class:`~refget.store.RefgetStore`. + +Import the package, not its submodules:: + + from refget.seqcolapi import create_seqcol_app, prepare_store + +:mod:`refget.seqcolapi.main` is deliberately *not* imported here: it builds a +FastAPI app at module scope and, absent ``REFGET_STORE_URL`` / +``REFGET_STORE_PATH``, connects to PostgreSQL on import. Ask for it by name +(``uvicorn refget.seqcolapi.main:app``) when you want that. +""" + +try: # The gate. Fail here, once, with something actionable. + import fastapi # noqa: F401 +except ImportError as e: # pragma: no cover - exercised in a bare venv + raise ImportError( + "refget.seqcolapi requires the optional sequence-collections service " + "dependencies (fastapi, uvicorn, ...), which are not installed.\n" + "Install them with: pip install 'refget[seqcolapi]'" + ) from e + +from refget.const import ALL_VERSIONS + +from .app import ( + DEFAULT_CACHE_DIR, + create_seqcol_app, + create_store_app, + load_seqcol_schema, + prepare_store, + store_service_info, +) +from .const import STATIC_DIRNAME, STATIC_PATH + +__all__ = [ + "ALL_VERSIONS", + "DEFAULT_CACHE_DIR", + "STATIC_DIRNAME", + "STATIC_PATH", + "create_seqcol_app", + "create_store_app", + "load_seqcol_schema", + "prepare_store", + "store_service_info", +] diff --git a/refget/seqcolapi/__main__.py b/refget/seqcolapi/__main__.py new file mode 100644 index 0000000..3ea99ba --- /dev/null +++ b/refget/seqcolapi/__main__.py @@ -0,0 +1,10 @@ +import sys + +from .main import main + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + print("Program canceled by user") + sys.exit(1) diff --git a/refget/app.py b/refget/seqcolapi/app.py similarity index 95% rename from refget/app.py rename to refget/seqcolapi/app.py index 63cf369..128e168 100644 --- a/refget/app.py +++ b/refget/seqcolapi/app.py @@ -26,16 +26,26 @@ Typical use:: - from refget.app import create_seqcol_app + from refget.seqcolapi import create_seqcol_app app.mount("/seqcol", create_seqcol_app(store_path=store_url, remote=True)) + +fastapi and starlette are imported at module level here, as ordinary Python. +That is safe because this module lives inside :mod:`refget.seqcolapi`, which no +base-install code path imports -- see that package's docstring. """ import json import logging import os -from .const import ALL_VERSIONS, SEQCOL_SCHEMA_PATH, SEQCOL_SPEC_VERSION +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from refget.const import ALL_VERSIONS, SEQCOL_SCHEMA_PATH, SEQCOL_SPEC_VERSION +from refget.middleware import StoreFreshnessMiddleware +from refget.router import create_refget_router, setup_backend +from refget.store import RefgetStore _LOGGER = logging.getLogger(__name__) @@ -69,8 +79,6 @@ def prepare_store( Returns: A ReadonlyRefgetStore suitable for concurrent serving. """ - from .store import RefgetStore - if remote: store = RefgetStore.open_remote(cache_dir, store_path) else: @@ -216,10 +224,6 @@ def create_seqcol_app( Returns: A FastAPI application ready to serve standalone or to ``app.mount()``. """ - from fastapi import FastAPI - - from .router import create_refget_router, setup_backend - if store is None and not defer_backend: if store_path is None: raise ValueError( @@ -233,8 +237,6 @@ def create_seqcol_app( app = FastAPI(title=title, version=ALL_VERSIONS["refget_version"]) if cors: - from fastapi.middleware.cors import CORSMiddleware - app.add_middleware( CORSMiddleware, allow_origins=["*"], @@ -261,8 +263,6 @@ def create_seqcol_app( if freshness: if not store_path: raise ValueError("freshness requires `store_path` to poll for rgstore.json") - from .middleware import StoreFreshnessMiddleware - app.add_middleware( StoreFreshnessMiddleware, store_url=store_path, diff --git a/refget/seqcolapi/const.py b/refget/seqcolapi/const.py new file mode 100644 index 0000000..3274cb7 --- /dev/null +++ b/refget/seqcolapi/const.py @@ -0,0 +1,10 @@ +import os + +# ALL_VERSIONS lives in refget.const (the base package) so that service-info can +# be built without importing anything from this optional subpackage. Re-exported +# here because both seqcolapi apps and the top-level `seqcolapi` compatibility +# shim read it from this module. +from refget.const import ALL_VERSIONS # noqa: F401 + +STATIC_DIRNAME = "static" +STATIC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), STATIC_DIRNAME) diff --git a/refget/seqcolapi/examples.py b/refget/seqcolapi/examples.py new file mode 100644 index 0000000..032b863 --- /dev/null +++ b/refget/seqcolapi/examples.py @@ -0,0 +1,143 @@ +# Models +# Used for documentation examples in OpenAPI + +from fastapi import Body, Path + +example_digest = Path( + ..., + description="Sequence collection digest", + pattern=r"^[-\w]+$", + max_length=64, + min_length=32, + examples="a6748aa0f6a1e165f871dbed5e54ba62", +) + +example_digest_2 = Path( + ..., + description="Sequence collection digest", + pattern=r"^[-\w]+$", + max_length=64, + min_length=32, + examples="2786eb8a921aa97018c214f64b9960a0", +) + +example_digest_hg38 = Path( + ..., + description="Sequence collection digest", + pattern=r"^[-\w]+$", + max_length=64, + min_length=32, + examples="514c871928a74885ce981faa61ccbb1a", +) + +example_digest_hg38_primary = Path( + ..., + description="Sequence collection digest", + pattern=r"^[-\w]+$", + max_length=64, + min_length=32, + examples="c345e091cce0b1df78bfc124b03fba1c", +) + +example_sequence = Path( + ..., + description="Refget sequence digest", + pattern=r"^[-\w]+$", + max_length=64, + min_length=32, + examples="76f9f3315fa4b831e93c36cd88196480", +) + +example_hg38_sc = Body( + { + "lengths": [ + "248956422", + "242193529", + "198295559", + "190214555", + "181538259", + "170805979", + "159345973", + "145138636", + "138394717", + "133797422", + "135086622", + "133275309", + "114364328", + "107043718", + "101991189", + "90338345", + "83257441", + "80373285", + "58617616", + "64444167", + "46709983", + "50818468", + "16569", + "156040895", + "57227415", + ], + "names": [ + "chr1", + "chr2", + "chr3", + "chr4", + "chr5", + "chr6", + "chr7", + "chr8", + "chr9", + "chr10", + "chr11", + "chr12", + "chr13", + "chr14", + "chr15", + "chr16", + "chr17", + "chr18", + "chr19", + "chr20", + "chr21", + "chr22", + "chrM", + "chrX", + "chrY", + ], + "sequences": [ + "a004bc1b0bf05fc668cab6bbfd93d3eb", + "0ccf3a67666ac53f99fcad19768f2dde", + "bda7b228789169ae811dd8d676d517ca", + "88a6091e2d9a609f4ea7eaef937cd4c2", + "0f1725f15e8046a6a04e32de629b1e10", + "08c3702d62a2c476a081d3ccd15ea30c", + "cac9e313d08cdf40c9eeafe62b17879a", + "9a2ebb88dc34c2af023d50219248c815", + "41bbec590d36e711864dc6f030f0264b", + "6b420cbb22daea77d7cc930c0a00f812", + "0d4e0be5c4e5bc0f12912894f21a5dd8", + "e1507ba70028a65b3f5a81b594e6f0fe", + "7110500758388b169fe631b212b7e56c", + "f37e77fdbacb1a0f1be5e2bf25df343d", + "3f14ce1984dada290682eb1f564934ee", + "88169bd58f0c5f9fd083030d1357d908", + "0bbc162a7d963574b5989adab5651ac5", + "388e8c7cd11a23eebf84a02d5e442bb7", + "1c927775585df1cb09ec7c7dd1b32a6a", + "c37960f60eff5e2cfbde87e53d262efa", + "f0324d60ccf85288a26a47a7ca25a54a", + "f7479d5a2a3169e2e44d97d7f2a13db1", + "6ab1f3c8f4941e148463c40408c89e43", + "6bdaf93397b486a58fd60b55aa2e21ca", + "9bd609da53b41a50a724f2a0131ee9c1", + ], + } +) + +reclimit_ex = Path( + ..., + description="Recursion limit, the number of times to recurse to populate digests in the structure", + gt=-1, + lt=2, + examples=0, +) diff --git a/refget/seqcolapi/main.py b/refget/seqcolapi/main.py new file mode 100644 index 0000000..a51a308 --- /dev/null +++ b/refget/seqcolapi/main.py @@ -0,0 +1,300 @@ +"""The seqcolapi services: the PostgreSQL-backed ``app`` and the store-backed +``store_app``. + +Importing this module builds a FastAPI app at module scope, and -- unless +``REFGET_STORE_URL`` or ``REFGET_STORE_PATH`` is set -- connects to PostgreSQL. +That is why :mod:`refget.seqcolapi` does not import it for you; ask for it by +name (``uvicorn refget.seqcolapi.main:app``). +""" + +import logging +import os +from contextlib import asynccontextmanager + +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse +from sqlmodel import Session, select +from starlette.requests import Request +from starlette.staticfiles import StaticFiles + +from refget.agents import RefgetDBAgent +from refget.const import HUMANS_SAMPLE_LIST, MOUSE_SAMPLES_LIST +from refget.models import HumanReadableNames +from refget.router import _ROUTER_CONFIG, _SAMPLE_DIGESTS, create_refget_router, setup_backend + +from .app import create_seqcol_app +from .const import ALL_VERSIONS, STATIC_DIRNAME, STATIC_PATH +from .examples import * + +global _LOGGER +_LOGGER = logging.getLogger(__name__) + + +def _load_scom_config(store_path: str, remote: bool): + """Load SCOM target digests from a JSON config. + + Checks (in order): + 1. SCOM_CONFIG_URL environment variable (any HTTP URL) + 2. scom_config.json next to the store (convention) + + Format: {"human": ["digest1", "digest2", ...], "mouse": [...]} + """ + import json + import os + import urllib.request + + # Try env var first + config_url = os.environ.get("SCOM_CONFIG_URL") + + # Fall back to store convention + if not config_url: + if remote: + config_url = store_path.rstrip("/") + "/scom_config.json" + else: + config_path = os.path.join(store_path, "scom_config.json") + if os.path.exists(config_path): + with open(config_path) as f: + config = json.load(f) + for species, digests in config.items(): + _SAMPLE_DIGESTS[species] = digests + _LOGGER.info(f"SCOM: loaded {len(digests)} target digests for '{species}'") + return + else: + _LOGGER.info( + "No SCOM_CONFIG_URL set and no scom_config.json found. SCOM disabled." + ) + return + + try: + with urllib.request.urlopen(config_url, timeout=10) as resp: + config = json.loads(resp.read()) + for species, digests in config.items(): + _SAMPLE_DIGESTS[species] = digests + _LOGGER.info(f"SCOM: loaded {len(digests)} target digests for '{species}'") + except Exception as e: + _LOGGER.info(f"Could not load SCOM config from {config_url} ({e}). SCOM disabled.") + + +for key, value in ALL_VERSIONS.items(): + _LOGGER.info(f"{key}: {value}") + + +@asynccontextmanager +async def lifespan_loader(app): + """ + Lifespan event to pre-load sample names and their digests + """ + _LOGGER.info("Starting lifespan: Loading sample data...") + + # Initialize backend via setup_backend + setup_backend(app, engine=RefgetDBAgent().engine) + + species_samples = {"human": HUMANS_SAMPLE_LIST, "mouse": MOUSE_SAMPLES_LIST} + + for species, sample_names in species_samples.items(): + try: + _LOGGER.info(f"Loading {len(sample_names)} sample names for {species}") + + with Session(app.state.dbagent.engine) as session: + statement = select(HumanReadableNames).where( + HumanReadableNames.human_readable_name.in_(sample_names) + ) + results = session.exec(statement).all() + + target_digests = [result.digest for result in results] + + _SAMPLE_DIGESTS[species] = target_digests + _LOGGER.info(f"Pre-loaded {len(target_digests)} digests for {species}") + + except Exception as e: + _LOGGER.error(f"Error loading sample data for {species}: {e}") + _SAMPLE_DIGESTS[species] = [] + + _LOGGER.info("Lifespan startup complete: Sample data loaded") + + yield # Application runs here + + # Cleanup + _LOGGER.info("Lifespan shutdown: Cleaning up sample data...") + _SAMPLE_DIGESTS.clear() + + +app = FastAPI( + title="Sequence Collections API", + description="An API providing metadata such as names, lengths, and other values for collections of reference sequences", + version=ALL_VERSIONS["refget_version"], + lifespan=lifespan_loader, +) + +origins = ["*"] + +app.add_middleware( # This is a public API, so we allow all origins + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Configuration +# RefgetStore URL (set to None if not using a backing store) +REFGET_STORE_URL = None # e.g., "s3://my-bucket/store/" + +# This is where the magic happens +# This will add the seqcol endpoints to the app +refget_router = create_refget_router( + sequences=False, + pangenomes=False, + fasta_drs=True, + refget_store_url=REFGET_STORE_URL, +) +app.include_router(refget_router) + + +# Catch-all error handler for any uncaught exceptions, return a 500 error with detailed information +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + return await http_exception_handler( + request, HTTPException(status_code=500, detail=str(exc)) + ) # Pass it to HTTP handler + + +# General Exception Handler (Covers All HTTPExceptions) +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + return JSONResponse( + status_code=exc.status_code, + content={ + "error": f"http-{exc.status_code}", # Generic error code + "detail": exc.detail, # FastAPI-style error message + "status": exc.status_code, + "path": str(request.url), # URL of the request + }, + ) + + +@app.exception_handler(ValueError) +async def value_error_handler(request: Request, exc: Exception): + raise HTTPException(status_code=404, detail=str(exc)) + + +@app.get("favicon.ico", include_in_schema=False) +async def favicon(): + return FileResponse("/static/favicon.ico") + + +@app.get("/", summary="Home page", tags=["General endpoints"], response_class=HTMLResponse) +async def index(request: Request): + """ + Returns a landing page HTML with the server resources ready to download. No inputs required. + """ + with open(f"{STATIC_PATH}/index.html", "r") as file: + content = file.read() + return HTMLResponse(content=content) + + +@app.get("/service-info", summary="GA4GH service info", tags=["General endpoints"]) +async def service_info(): + # Build seqcol capabilities object + seqcol_info = { + "schema": getattr(app.state.dbagent, "schema_dict", None) + if hasattr(app.state, "dbagent") + else None, + "sorted_name_length_pairs": True, + "fasta_drs": {"enabled": _ROUTER_CONFIG.get("fasta_drs", False)}, + } + + # Get backend capabilities + backend = getattr(app.state, "backend", None) + caps = backend.capabilities() if backend and hasattr(backend, "capabilities") else {} + + # Add refget_store info + store_url = _ROUTER_CONFIG.get("refget_store_url") + if store_url: + seqcol_info["refget_store"] = {"enabled": True, "url": store_url, **caps} + else: + seqcol_info["refget_store"] = {"enabled": False} + + # Advertise alias + FHR availability independent of refget_store_url + seqcol_info["aliases"] = { + "enabled": bool( + caps.get("collection_alias_namespaces") or caps.get("sequence_alias_namespaces") + ) + } + seqcol_info["fhr_metadata"] = {"enabled": bool(caps.get("fhr_metadata_collections"))} + + return { + "id": "org.databio.seqcolapi", + "name": "Sequence collections", + "type": { + "group": "org.ga4gh", + "artifact": "refget-seqcol", + "version": ALL_VERSIONS["seqcol_spec_version"], + }, + "description": "An API providing metadata such as names, lengths, and other values for collections of reference sequences", + "organization": {"name": "Databio Lab", "url": "https://databio.org"}, + "contactUrl": "https://github.com/refgenie/refget/issues", + "documentationUrl": "https://seqcolapi.databio.org", + "updatedAt": "2025-02-20T00:00:00Z", + "environment": "dev", + "version": ALL_VERSIONS, + "seqcol": seqcol_info, + } + + +# Mount statics after other routes for lower precedence +app.mount("/", StaticFiles(directory=STATIC_PATH), name=STATIC_DIRNAME) + + +def create_store_app(store_path: str, remote: bool = False, cache_dir: str = "/tmp/seqcol_cache"): + """Create a seqcolapi FastAPI app backed by a RefgetStore (no database). + + Thin wrapper over :func:`refget.seqcolapi.create_seqcol_app`, which owns the + shared store-backed wiring (readonly store, backend, router, freshness, + GA4GH service-info). Everything seqcolapi adds on top is SCOM, which is not + a refget concern, so it is injected through ``service_info_extra``. + + Args: + store_path: Path to store on disk, or S3 URL for remote stores. + remote: If True, open as a remote (S3) store. + cache_dir: Local cache directory for remote stores. + + Returns: + FastAPI app with store-backed seqcol endpoints. + """ + # Load SCOM config: check SCOM_CONFIG_URL env var, then fall back to store convention + _load_scom_config(store_path, remote) + + def _scom_block(): + # Evaluated per request, because _SAMPLE_DIGESTS can be repopulated. + return { + "scom": { + "enabled": bool(_SAMPLE_DIGESTS), + "species": list(_SAMPLE_DIGESTS.keys()), + } + } + + return create_seqcol_app( + store_path=store_path, + remote=remote, + cache_dir=cache_dir, + # Advertise the store path (REFGET_STORE_HTTP_URL overrides it inside + # store_service_info), matching the pre-extraction service-info exactly. + store_url=store_path, + service_info_id="org.databio.seqcolapi.store", + service_info_name="Sequence collections (store-backed)", + service_info_extra=_scom_block, + ) + + +_STORE_URL_ENV = os.environ.get("REFGET_STORE_URL") +_STORE_PATH_ENV = os.environ.get("REFGET_STORE_PATH") + +if _STORE_URL_ENV: + store_app = create_store_app(_STORE_URL_ENV, remote=True) +elif _STORE_PATH_ENV: + store_app = create_store_app(_STORE_PATH_ENV, remote=False) + +if __name__ != "__main__" and not _STORE_URL_ENV and not _STORE_PATH_ENV: + setup_backend(app, engine=RefgetDBAgent().engine) diff --git a/seqcolapi/static/css/databio.css b/refget/seqcolapi/static/css/databio.css similarity index 100% rename from seqcolapi/static/css/databio.css rename to refget/seqcolapi/static/css/databio.css diff --git a/seqcolapi/static/css/style.css b/refget/seqcolapi/static/css/style.css similarity index 100% rename from seqcolapi/static/css/style.css rename to refget/seqcolapi/static/css/style.css diff --git a/seqcolapi/static/favicon.ico b/refget/seqcolapi/static/favicon.ico similarity index 100% rename from seqcolapi/static/favicon.ico rename to refget/seqcolapi/static/favicon.ico diff --git a/seqcolapi/static/index.html b/refget/seqcolapi/static/index.html similarity index 100% rename from seqcolapi/static/index.html rename to refget/seqcolapi/static/index.html diff --git a/seqcolapi/static/logo_databio_long.svg b/refget/seqcolapi/static/logo_databio_long.svg similarity index 100% rename from seqcolapi/static/logo_databio_long.svg rename to refget/seqcolapi/static/logo_databio_long.svg diff --git a/seqcolapi/static/seqcol_logo.svg b/refget/seqcolapi/static/seqcol_logo.svg similarity index 100% rename from seqcolapi/static/seqcol_logo.svg rename to refget/seqcolapi/static/seqcol_logo.svg diff --git a/seqcolapi/__init__.py b/seqcolapi/__init__.py index e69de29..6673e9f 100644 --- a/seqcolapi/__init__.py +++ b/seqcolapi/__init__.py @@ -0,0 +1,13 @@ +"""Compatibility shim: ``seqcolapi`` now lives in :mod:`refget.seqcolapi`. + +The service code moved into the refget package so that it actually ships in the +wheel. This top-level package is kept only so that existing deployments -- whose +``Dockerfile`` does ``COPY seqcolapi/ /app/seqcolapi`` and then runs +``uvicorn seqcolapi.main:app`` or ``uvicorn seqcolapi.main:store_app`` -- keep +working unchanged. + +New code should import from :mod:`refget.seqcolapi` instead. +""" + +from refget.seqcolapi import * # noqa: F401,F403 +from refget.seqcolapi import __all__ as __all__ # noqa: PLC0414 diff --git a/seqcolapi/__main__.py b/seqcolapi/__main__.py index 3ea99ba..66eb1a2 100644 --- a/seqcolapi/__main__.py +++ b/seqcolapi/__main__.py @@ -1,10 +1,5 @@ -import sys +"""Compatibility shim: ``python -m seqcolapi`` -> ``python -m refget.seqcolapi``.""" -from .main import main +import runpy -if __name__ == "__main__": - try: - sys.exit(main()) - except KeyboardInterrupt: - print("Program canceled by user") - sys.exit(1) +runpy.run_module("refget.seqcolapi", run_name="__main__", alter_sys=True) diff --git a/seqcolapi/const.py b/seqcolapi/const.py index 6b03ef6..35712a7 100644 --- a/seqcolapi/const.py +++ b/seqcolapi/const.py @@ -1,9 +1,7 @@ -import os +"""Compatibility shim for :mod:`refget.seqcolapi.const`.""" -# ALL_VERSIONS moved into the distributed `refget` package (refget/const.py) so -# the shared store-backed app factory can build service-info without seqcolapi, -# which is not shipped in the refget wheel. Re-exported here for compatibility. -from refget.const import ALL_VERSIONS # noqa: F401 - -STATIC_DIRNAME = "static" -STATIC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), STATIC_DIRNAME) +from refget.seqcolapi.const import ( # noqa: F401 + ALL_VERSIONS, + STATIC_DIRNAME, + STATIC_PATH, +) diff --git a/seqcolapi/examples.py b/seqcolapi/examples.py index 032b863..3ec5725 100644 --- a/seqcolapi/examples.py +++ b/seqcolapi/examples.py @@ -1,143 +1,3 @@ -# Models -# Used for documentation examples in OpenAPI +"""Compatibility shim for :mod:`refget.seqcolapi.examples`.""" -from fastapi import Body, Path - -example_digest = Path( - ..., - description="Sequence collection digest", - pattern=r"^[-\w]+$", - max_length=64, - min_length=32, - examples="a6748aa0f6a1e165f871dbed5e54ba62", -) - -example_digest_2 = Path( - ..., - description="Sequence collection digest", - pattern=r"^[-\w]+$", - max_length=64, - min_length=32, - examples="2786eb8a921aa97018c214f64b9960a0", -) - -example_digest_hg38 = Path( - ..., - description="Sequence collection digest", - pattern=r"^[-\w]+$", - max_length=64, - min_length=32, - examples="514c871928a74885ce981faa61ccbb1a", -) - -example_digest_hg38_primary = Path( - ..., - description="Sequence collection digest", - pattern=r"^[-\w]+$", - max_length=64, - min_length=32, - examples="c345e091cce0b1df78bfc124b03fba1c", -) - -example_sequence = Path( - ..., - description="Refget sequence digest", - pattern=r"^[-\w]+$", - max_length=64, - min_length=32, - examples="76f9f3315fa4b831e93c36cd88196480", -) - -example_hg38_sc = Body( - { - "lengths": [ - "248956422", - "242193529", - "198295559", - "190214555", - "181538259", - "170805979", - "159345973", - "145138636", - "138394717", - "133797422", - "135086622", - "133275309", - "114364328", - "107043718", - "101991189", - "90338345", - "83257441", - "80373285", - "58617616", - "64444167", - "46709983", - "50818468", - "16569", - "156040895", - "57227415", - ], - "names": [ - "chr1", - "chr2", - "chr3", - "chr4", - "chr5", - "chr6", - "chr7", - "chr8", - "chr9", - "chr10", - "chr11", - "chr12", - "chr13", - "chr14", - "chr15", - "chr16", - "chr17", - "chr18", - "chr19", - "chr20", - "chr21", - "chr22", - "chrM", - "chrX", - "chrY", - ], - "sequences": [ - "a004bc1b0bf05fc668cab6bbfd93d3eb", - "0ccf3a67666ac53f99fcad19768f2dde", - "bda7b228789169ae811dd8d676d517ca", - "88a6091e2d9a609f4ea7eaef937cd4c2", - "0f1725f15e8046a6a04e32de629b1e10", - "08c3702d62a2c476a081d3ccd15ea30c", - "cac9e313d08cdf40c9eeafe62b17879a", - "9a2ebb88dc34c2af023d50219248c815", - "41bbec590d36e711864dc6f030f0264b", - "6b420cbb22daea77d7cc930c0a00f812", - "0d4e0be5c4e5bc0f12912894f21a5dd8", - "e1507ba70028a65b3f5a81b594e6f0fe", - "7110500758388b169fe631b212b7e56c", - "f37e77fdbacb1a0f1be5e2bf25df343d", - "3f14ce1984dada290682eb1f564934ee", - "88169bd58f0c5f9fd083030d1357d908", - "0bbc162a7d963574b5989adab5651ac5", - "388e8c7cd11a23eebf84a02d5e442bb7", - "1c927775585df1cb09ec7c7dd1b32a6a", - "c37960f60eff5e2cfbde87e53d262efa", - "f0324d60ccf85288a26a47a7ca25a54a", - "f7479d5a2a3169e2e44d97d7f2a13db1", - "6ab1f3c8f4941e148463c40408c89e43", - "6bdaf93397b486a58fd60b55aa2e21ca", - "9bd609da53b41a50a724f2a0131ee9c1", - ], - } -) - -reclimit_ex = Path( - ..., - description="Recursion limit, the number of times to recurse to populate digests in the structure", - gt=-1, - lt=2, - examples=0, -) +from refget.seqcolapi.examples import * # noqa: F401,F403 diff --git a/seqcolapi/main.py b/seqcolapi/main.py index 3b0287d..55d8a48 100644 --- a/seqcolapi/main.py +++ b/seqcolapi/main.py @@ -1,292 +1,23 @@ -import logging -import os -from contextlib import asynccontextmanager +"""Compatibility shim for :mod:`refget.seqcolapi.main`. -from fastapi import FastAPI, HTTPException -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse, HTMLResponse, JSONResponse -from sqlmodel import Session, select -from starlette.requests import Request -from starlette.staticfiles import StaticFiles +``uvicorn seqcolapi.main:app`` and ``uvicorn seqcolapi.main:store_app`` resolve +through here. Attribute access is forwarded rather than star-imported because +``store_app`` only exists when ``REFGET_STORE_URL`` / ``REFGET_STORE_PATH`` is +set, and a plain ``from ... import store_app`` would turn a missing store into +an ImportError at the wrong moment. +""" -from refget.agents import RefgetDBAgent -from refget.const import HUMANS_SAMPLE_LIST, MOUSE_SAMPLES_LIST -from refget.models import HumanReadableNames -from refget.router import _ROUTER_CONFIG, _SAMPLE_DIGESTS, create_refget_router, setup_backend +from refget.seqcolapi import main as _main -from .const import ALL_VERSIONS, STATIC_DIRNAME, STATIC_PATH -from .examples import * +__doc__ = _main.__doc__ or __doc__ -global _LOGGER -_LOGGER = logging.getLogger(__name__) - - -def _load_scom_config(store_path: str, remote: bool): - """Load SCOM target digests from a JSON config. - - Checks (in order): - 1. SCOM_CONFIG_URL environment variable (any HTTP URL) - 2. scom_config.json next to the store (convention) - - Format: {"human": ["digest1", "digest2", ...], "mouse": [...]} - """ - import json - import os - import urllib.request - - # Try env var first - config_url = os.environ.get("SCOM_CONFIG_URL") - - # Fall back to store convention - if not config_url: - if remote: - config_url = store_path.rstrip("/") + "/scom_config.json" - else: - config_path = os.path.join(store_path, "scom_config.json") - if os.path.exists(config_path): - with open(config_path) as f: - config = json.load(f) - for species, digests in config.items(): - _SAMPLE_DIGESTS[species] = digests - _LOGGER.info(f"SCOM: loaded {len(digests)} target digests for '{species}'") - return - else: - _LOGGER.info( - "No SCOM_CONFIG_URL set and no scom_config.json found. SCOM disabled." - ) - return +def __getattr__(name): try: - with urllib.request.urlopen(config_url, timeout=10) as resp: - config = json.loads(resp.read()) - for species, digests in config.items(): - _SAMPLE_DIGESTS[species] = digests - _LOGGER.info(f"SCOM: loaded {len(digests)} target digests for '{species}'") - except Exception as e: - _LOGGER.info(f"Could not load SCOM config from {config_url} ({e}). SCOM disabled.") - - -for key, value in ALL_VERSIONS.items(): - _LOGGER.info(f"{key}: {value}") - - -@asynccontextmanager -async def lifespan_loader(app): - """ - Lifespan event to pre-load sample names and their digests - """ - _LOGGER.info("Starting lifespan: Loading sample data...") - - # Initialize backend via setup_backend - setup_backend(app, engine=RefgetDBAgent().engine) - - species_samples = {"human": HUMANS_SAMPLE_LIST, "mouse": MOUSE_SAMPLES_LIST} - - for species, sample_names in species_samples.items(): - try: - _LOGGER.info(f"Loading {len(sample_names)} sample names for {species}") - - with Session(app.state.dbagent.engine) as session: - statement = select(HumanReadableNames).where( - HumanReadableNames.human_readable_name.in_(sample_names) - ) - results = session.exec(statement).all() - - target_digests = [result.digest for result in results] - - _SAMPLE_DIGESTS[species] = target_digests - _LOGGER.info(f"Pre-loaded {len(target_digests)} digests for {species}") - - except Exception as e: - _LOGGER.error(f"Error loading sample data for {species}: {e}") - _SAMPLE_DIGESTS[species] = [] - - _LOGGER.info("Lifespan startup complete: Sample data loaded") - - yield # Application runs here - - # Cleanup - _LOGGER.info("Lifespan shutdown: Cleaning up sample data...") - _SAMPLE_DIGESTS.clear() - - -app = FastAPI( - title="Sequence Collections API", - description="An API providing metadata such as names, lengths, and other values for collections of reference sequences", - version=ALL_VERSIONS["refget_version"], - lifespan=lifespan_loader, -) - -origins = ["*"] - -app.add_middleware( # This is a public API, so we allow all origins - CORSMiddleware, - allow_origins=origins, - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -# Configuration -# RefgetStore URL (set to None if not using a backing store) -REFGET_STORE_URL = None # e.g., "s3://my-bucket/store/" - -# This is where the magic happens -# This will add the seqcol endpoints to the app -refget_router = create_refget_router( - sequences=False, - pangenomes=False, - fasta_drs=True, - refget_store_url=REFGET_STORE_URL, -) -app.include_router(refget_router) - - -# Catch-all error handler for any uncaught exceptions, return a 500 error with detailed information -@app.exception_handler(Exception) -async def generic_exception_handler(request: Request, exc: Exception): - return await http_exception_handler( - request, HTTPException(status_code=500, detail=str(exc)) - ) # Pass it to HTTP handler - - -# General Exception Handler (Covers All HTTPExceptions) -@app.exception_handler(HTTPException) -async def http_exception_handler(request: Request, exc: HTTPException): - return JSONResponse( - status_code=exc.status_code, - content={ - "error": f"http-{exc.status_code}", # Generic error code - "detail": exc.detail, # FastAPI-style error message - "status": exc.status_code, - "path": str(request.url), # URL of the request - }, - ) - - -@app.exception_handler(ValueError) -async def value_error_handler(request: Request, exc: Exception): - raise HTTPException(status_code=404, detail=str(exc)) - - -@app.get("favicon.ico", include_in_schema=False) -async def favicon(): - return FileResponse("/static/favicon.ico") - - -@app.get("/", summary="Home page", tags=["General endpoints"], response_class=HTMLResponse) -async def index(request: Request): - """ - Returns a landing page HTML with the server resources ready to download. No inputs required. - """ - with open(f"{STATIC_PATH}/index.html", "r") as file: - content = file.read() - return HTMLResponse(content=content) - - -@app.get("/service-info", summary="GA4GH service info", tags=["General endpoints"]) -async def service_info(): - # Build seqcol capabilities object - seqcol_info = { - "schema": getattr(app.state.dbagent, "schema_dict", None) - if hasattr(app.state, "dbagent") - else None, - "sorted_name_length_pairs": True, - "fasta_drs": {"enabled": _ROUTER_CONFIG.get("fasta_drs", False)}, - } - - # Get backend capabilities - backend = getattr(app.state, "backend", None) - caps = backend.capabilities() if backend and hasattr(backend, "capabilities") else {} - - # Add refget_store info - store_url = _ROUTER_CONFIG.get("refget_store_url") - if store_url: - seqcol_info["refget_store"] = {"enabled": True, "url": store_url, **caps} - else: - seqcol_info["refget_store"] = {"enabled": False} - - # Advertise alias + FHR availability independent of refget_store_url - seqcol_info["aliases"] = { - "enabled": bool( - caps.get("collection_alias_namespaces") or caps.get("sequence_alias_namespaces") - ) - } - seqcol_info["fhr_metadata"] = {"enabled": bool(caps.get("fhr_metadata_collections"))} - - return { - "id": "org.databio.seqcolapi", - "name": "Sequence collections", - "type": { - "group": "org.ga4gh", - "artifact": "refget-seqcol", - "version": ALL_VERSIONS["seqcol_spec_version"], - }, - "description": "An API providing metadata such as names, lengths, and other values for collections of reference sequences", - "organization": {"name": "Databio Lab", "url": "https://databio.org"}, - "contactUrl": "https://github.com/refgenie/refget/issues", - "documentationUrl": "https://seqcolapi.databio.org", - "updatedAt": "2025-02-20T00:00:00Z", - "environment": "dev", - "version": ALL_VERSIONS, - "seqcol": seqcol_info, - } - - -# Mount statics after other routes for lower precedence -app.mount("/", StaticFiles(directory=STATIC_PATH), name=STATIC_DIRNAME) - - -def create_store_app(store_path: str, remote: bool = False, cache_dir: str = "/tmp/seqcol_cache"): - """Create a seqcolapi FastAPI app backed by a RefgetStore (no database). - - Thin wrapper over ``refget.app.create_seqcol_app``, which owns the shared - store-backed wiring (readonly store, backend, router, freshness, GA4GH - service-info). Everything seqcolapi adds on top is SCOM, which is not a - refget concern, so it is injected through ``service_info_extra``. - - Args: - store_path: Path to store on disk, or S3 URL for remote stores. - remote: If True, open as a remote (S3) store. - cache_dir: Local cache directory for remote stores. - - Returns: - FastAPI app with store-backed seqcol endpoints. - """ - from refget.app import create_seqcol_app - - # Load SCOM config: check SCOM_CONFIG_URL env var, then fall back to store convention - _load_scom_config(store_path, remote) - - def _scom_block(): - # Evaluated per request, because _SAMPLE_DIGESTS can be repopulated. - return { - "scom": { - "enabled": bool(_SAMPLE_DIGESTS), - "species": list(_SAMPLE_DIGESTS.keys()), - } - } - - return create_seqcol_app( - store_path=store_path, - remote=remote, - cache_dir=cache_dir, - # Advertise the store path (REFGET_STORE_HTTP_URL overrides it inside - # store_service_info), matching the pre-extraction service-info exactly. - store_url=store_path, - service_info_id="org.databio.seqcolapi.store", - service_info_name="Sequence collections (store-backed)", - service_info_extra=_scom_block, - ) - - -_STORE_URL_ENV = os.environ.get("REFGET_STORE_URL") -_STORE_PATH_ENV = os.environ.get("REFGET_STORE_PATH") + return getattr(_main, name) + except AttributeError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None -if _STORE_URL_ENV: - store_app = create_store_app(_STORE_URL_ENV, remote=True) -elif _STORE_PATH_ENV: - store_app = create_store_app(_STORE_PATH_ENV, remote=False) -if __name__ != "__main__" and not _STORE_URL_ENV and not _STORE_PATH_ENV: - setup_backend(app, engine=RefgetDBAgent().engine) +def __dir__(): + return sorted(set(globals()) | set(dir(_main))) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f67f829..5e49ba1 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -80,7 +80,7 @@ def loaded_dbagent(test_dbagent, test_fasta_path): def client(loaded_dbagent): """Create TestClient with test database""" from refget.router import get_dbagent - from seqcolapi.main import app + from refget.seqcolapi.main import app def override_get_dbagent(): return loaded_dbagent @@ -134,7 +134,7 @@ def test_server(request): import uvicorn from refget.router import get_dbagent - from seqcolapi.main import app + from refget.seqcolapi.main import app def override_get_dbagent(): return loaded_dbagent @@ -192,8 +192,8 @@ def store_test_server(tmp_path_factory): from fastapi.middleware.cors import CORSMiddleware from refget.router import create_refget_router, setup_backend + from refget.seqcolapi.const import ALL_VERSIONS from refget.store import RefgetStore - from seqcolapi.const import ALL_VERSIONS # Create store and load test FASTAs store_dir = tmp_path_factory.mktemp("store") diff --git a/tests/local/test_app_factory.py b/tests/local/test_app_factory.py index eb8deab..ee93667 100644 --- a/tests/local/test_app_factory.py +++ b/tests/local/test_app_factory.py @@ -1,4 +1,4 @@ -"""Tests for the shared store-backed seqcol app factory (refget.app). +"""Tests for the shared store-backed seqcol app factory (refget.seqcolapi). Covers the three properties the factory exists to guarantee: @@ -24,8 +24,8 @@ except ImportError: _RUST_BINDINGS_AVAILABLE = False -from refget.app import create_seqcol_app, store_service_info from refget.router import create_refget_router, setup_backend +from refget.seqcolapi import create_seqcol_app, store_service_info TEST_FASTA_DIR = Path("test_fasta") BASE_FASTA = TEST_FASTA_DIR / "base.fa" diff --git a/tests/local/test_import_gating.py b/tests/local/test_import_gating.py new file mode 100644 index 0000000..18089eb --- /dev/null +++ b/tests/local/test_import_gating.py @@ -0,0 +1,75 @@ +"""The service subpackage must stay off the base-install import path. + +``refget.seqcolapi`` imports fastapi, starlette and (via ``main``) sqlmodel at +module level, which is fine only for as long as nothing on the plain +``import refget`` path reaches it. These tests are the tripwire for that: they +run in a fresh interpreter, because ``sys.modules`` in the pytest process is +already polluted by the rest of the suite. +""" + +import subprocess +import sys + +GATED_TOP_LEVEL = ("fastapi", "starlette", "uvicorn", "sqlmodel", "sqlalchemy", "psycopg2") + + +def _run(code: str) -> str: + proc = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=False + ) + assert proc.returncode == 0, proc.stderr + return proc.stdout.strip() + + +def test_import_refget_does_not_load_the_service_subpackage(): + loaded = _run( + "import sys, refget;" + "print(','.join(sorted(m for m in sys.modules if m.startswith('refget.seqcolapi'))))" + ) + assert loaded == "" + + +def test_import_refget_does_not_load_heavy_dependencies(): + loaded = _run( + "import sys, refget;" + f"tops = {GATED_TOP_LEVEL!r};" + "print(','.join(sorted({m.split('.')[0] for m in sys.modules} & set(tops))))" + ) + assert loaded == "" + + +def test_import_refget_cli_does_not_load_heavy_dependencies(): + loaded = _run( + "import sys, refget.cli;" + f"tops = {GATED_TOP_LEVEL!r};" + "print(','.join(sorted({m.split('.')[0] for m in sys.modules} & set(tops))))" + ) + assert loaded == "" + + +def test_service_subpackage_is_importable_when_deps_are_present(): + # The suite installs the `test` extra, so fastapi is available here. + loaded = _run( + "import sys, refget.seqcolapi;" + "print('fastapi' in sys.modules and hasattr(refget.seqcolapi, 'create_seqcol_app'))" + ) + assert loaded == "True" + + +def test_missing_service_deps_raise_an_actionable_error(): + # Simulate a base install by making `fastapi` unimportable. + message = _run( + "import sys;" + "sys.meta_path.insert(0, type('Blocker', (), {" + " 'find_spec': staticmethod(" + " lambda name, path=None, target=None: " + " (_ for _ in ()).throw(ImportError('blocked')) if name.split('.')[0] == 'fastapi' " + " else None)" + "})());" + "\ntry:\n" + " import refget.seqcolapi\n" + "except ImportError as e:\n" + " print(str(e).replace(chr(10), ' '))\n" + ) + assert "refget[seqcolapi]" in message + assert "pip install" in message From cc09f42af15dbd2add90e8db8bd8466539ec5d52 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 17:44:48 -0400 Subject: [PATCH 04/21] Name the wheel package explicitly --- pyproject.toml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 06f9845..c19330c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,19 @@ Homepage = "https://github.com/refgenie/refget" [tool.hatch.version] path = "refget/_version.py" +# Ship exactly one package: refget/. Stated explicitly rather than left to +# hatchling's name-based auto-detection, because there is now a second +# importable top-level directory -- the seqcolapi/ compatibility shim -- which +# must NOT go in the wheel. The shim exists only for deployments that COPY it +# out of the repo; the real code is refget/seqcolapi/. +# +# refget/seqcolapi/static/ needs no force-include: it lives inside the package +# directory, and hatchling ships every non-VCS-ignored file it finds there. +# Verify after touching .gitignore -- an ignore rule that matched, say, *.ico +# would silently drop package data from the wheel. +[tool.hatch.build.targets.wheel] +packages = ["refget"] + # Bundle the compliance known-answer fixtures into the installed package so the # compliance runner is self-contained on a pip-installed deploy (the source # files live at the repo root, outside the refget/ package). From 66a0f5bfd62583714de459d144f440f7bebf51b6 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 17:44:48 -0400 Subject: [PATCH 05/21] Say which dependency the gate is missing --- pyproject.toml | 5 +++++ refget/seqcolapi/__init__.py | 33 +++++++++++++++++++++++-------- tests/local/test_import_gating.py | 13 ++++++++---- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c19330c..301ab60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,11 @@ exclude = [ select = ["E", "F", "I"] ignore = ["F403", "F405", "E501"] +[tool.ruff.lint.per-file-ignores] +# The dependency gate has to run before the service imports it guards, so those +# imports are deliberately not at the top of the file. +"refget/seqcolapi/__init__.py" = ["E402"] + [tool.ruff.lint.isort] known-first-party = ["refget"] diff --git a/refget/seqcolapi/__init__.py b/refget/seqcolapi/__init__.py index 5d23cca..e76cc88 100644 --- a/refget/seqcolapi/__init__.py +++ b/refget/seqcolapi/__init__.py @@ -26,14 +26,31 @@ (``uvicorn refget.seqcolapi.main:app``) when you want that. """ -try: # The gate. Fail here, once, with something actionable. - import fastapi # noqa: F401 -except ImportError as e: # pragma: no cover - exercised in a bare venv - raise ImportError( - "refget.seqcolapi requires the optional sequence-collections service " - "dependencies (fastapi, uvicorn, ...), which are not installed.\n" - "Install them with: pip install 'refget[seqcolapi]'" - ) from e + +def _gate(): + """Fail once, here, with something actionable -- not with a bare + ModuleNotFoundError from four imports deep. + + Probes rather than wrapping the imports below, so that a genuine ImportError + inside our own service code still surfaces as itself. + """ + missing = [] + for dep in ("fastapi", "sqlmodel"): # sqlmodel arrives via refget.router + try: + __import__(dep) + except ImportError: + missing.append(dep) + if missing: + raise ImportError( + "refget.seqcolapi requires the optional sequence-collections " + f"service dependencies, and {', '.join(missing)} " + f"{'is' if len(missing) == 1 else 'are'} not installed.\n" + "Install them with: pip install 'refget[seqcolapi]'" + ) + + +_gate() +del _gate from refget.const import ALL_VERSIONS diff --git a/tests/local/test_import_gating.py b/tests/local/test_import_gating.py index 18089eb..58fa89f 100644 --- a/tests/local/test_import_gating.py +++ b/tests/local/test_import_gating.py @@ -10,6 +10,8 @@ import subprocess import sys +import pytest + GATED_TOP_LEVEL = ("fastapi", "starlette", "uvicorn", "sqlmodel", "sqlalchemy", "psycopg2") @@ -56,15 +58,17 @@ def test_service_subpackage_is_importable_when_deps_are_present(): assert loaded == "True" -def test_missing_service_deps_raise_an_actionable_error(): - # Simulate a base install by making `fastapi` unimportable. +@pytest.mark.parametrize("blocked", ["fastapi", "sqlmodel"]) +def test_missing_service_deps_raise_an_actionable_error(blocked): + # Simulate a base install by making one gated dependency unimportable. message = _run( "import sys;" + f"blocked = {blocked!r};" "sys.meta_path.insert(0, type('Blocker', (), {" " 'find_spec': staticmethod(" " lambda name, path=None, target=None: " - " (_ for _ in ()).throw(ImportError('blocked')) if name.split('.')[0] == 'fastapi' " - " else None)" + " (_ for _ in ()).throw(ImportError('blocked')) " + " if name.split('.')[0] == blocked else None)" "})());" "\ntry:\n" " import refget.seqcolapi\n" @@ -73,3 +77,4 @@ def test_missing_service_deps_raise_an_actionable_error(): ) assert "refget[seqcolapi]" in message assert "pip install" in message + assert blocked in message From 03bab6e1f93e6c7d4665026a54aacffb6af4d515 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 17:44:48 -0400 Subject: [PATCH 06/21] Give the database deps their own extra --- README.md | 39 ++++- pyproject.toml | 47 ++++-- refget/_deps.py | 52 +++++++ refget/agents.py | 45 ++++-- refget/models.py | 74 +++++----- refget/response_models.py | 63 ++++++++ refget/router.py | 8 +- refget/seqcolapi/__init__.py | 70 ++++----- refget/seqcolapi/dbapp.py | 231 +++++++++++++++++++++++++++++ refget/seqcolapi/main.py | 237 ++++++------------------------ tests/local/test_import_gating.py | 176 +++++++++++++++++----- 11 files changed, 712 insertions(+), 330 deletions(-) create mode 100644 refget/_deps.py create mode 100644 refget/response_models.py create mode 100644 refget/seqcolapi/dbapp.py diff --git a/README.md b/README.md index 57b59f4..7ef51e5 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,39 @@ User-facing documentation is hosted at [refgenie.org/refget](https://refgenie.or This repository includes: 1. `/refget`: The `refget` Python package, which provides a Python interface to both remote and local use of refget standards. It has clients and functions for both refget sequences and refget sequence collections (seqcol). -2. `/refget/seqcolapi`: Sequence collections API software, a FastAPI wrapper built on top of the `refget` package. It provides a bare-bones Sequence Collections API service. It ships in the `refget` wheel, but its dependencies do not: install them with `pip install 'refget[seqcolapi]'`. Nothing on the plain `import refget` path imports this subpackage, so a base install never pays for fastapi/uvicorn/sqlmodel/psycopg2. +2. `/refget/seqcolapi`: Sequence collections API software, a FastAPI wrapper built on top of the `refget` package. It provides a bare-bones Sequence Collections API service. It ships in the `refget` wheel, but its dependencies do not — see [Installation](#installation). Nothing on the plain `import refget` path imports this subpackage, so a base install never pays for fastapi/uvicorn/sqlmodel/psycopg2. 3. `/seqcolapi`: A thin compatibility shim re-exporting `refget.seqcolapi`, kept so existing deployments that run `uvicorn seqcolapi.main:app` (or `:store_app`) keep working. New code should import `refget.seqcolapi`. 4. `/deployment`: Server configurations for demo instances and public deployed instances. There are also github workflows (in `.github/workflows`) that deploy the demo server instance from this repository. 5. `/test_fasta` and `/test_api`: Dummy data and a compliance test, to test external implementations of the Refget Sequence Collections API. 6. `/frontend`: a React seqcolapi front-end. +## Installation + +The base install is deliberately light — a client library and CLI, no web +server and no ORM. Everything heavier is an extra, and the extras compose: + +| Install | Adds | Use it for | +| --- | --- | --- | +| `pip install refget` | — | The Python library and the `refget` CLI: digests, `RefgetStore`, API clients, compliance. No fastapi, no sqlalchemy. | +| `pip install 'refget[db]'` | sqlmodel, psycopg2-binary | The SQLModel layer: `refget.models`, `refget.agents.RefgetDBAgent`, `refget admin`. A library capability — you can want the ORM without wanting a server. | +| `pip install 'refget[seqcolapi]'` | fastapi, uvicorn | **Serving a RefgetStore.** `uvicorn seqcolapi.main:store_app`, `refget store serve`, `refget.seqcolapi.create_seqcol_app`. No database of any kind. | +| `pip install 'refget[seqcolapi-db]'` | both of the above | **The PostgreSQL-backed service** (`uvicorn seqcolapi.main:app`), i.e. what runs seqcolapi.databio.org. | + +The two service extras correspond to the two deployment modes described under +[Development and deployment: Backend](#development-and-deployment-backend). The +store-backed mode is the common case and the cheaper one; it needs no database +dependencies at all. + +These boundaries are enforced by module structure, not convention: the +database code lives in `refget/models.py`, `refget/agents.py` and +`refget/seqcolapi/dbapp.py`, and nothing else imports them at module level. The +router's response bodies live in `refget/response_models.py` (plain pydantic) +precisely so that serving the API does not require an ORM. Importing a module +without its extra raises an error naming the extra to install, rather than a +bare `ModuleNotFoundError`. `tests/local/test_import_gating.py` is the tripwire. + + ## Deploy to AWS ECS To deploy the public demo instance, you can either: @@ -46,7 +72,7 @@ This starts the test database, runs tests, and cleans up automatically. ### Store-backed (no database) -The store-backed seqcolapi uses a RefgetStore (local files) instead of PostgreSQL. This is the simplest way to run the API. +The store-backed seqcolapi uses a RefgetStore (local files) instead of PostgreSQL. This is the simplest way to run the API, and it needs only `pip install 'refget[seqcolapi]'` — fastapi and uvicorn, no sqlmodel, no sqlalchemy, no psycopg2. For safe concurrent serving, the store is fully loaded and converted to a read-only store (`RefgetStore.into_readonly()`) before serving, so HTTP reads borrow immutably across request threads. The `refget store serve` CLI does this by default; pass `--lazy` to serve directly from the mutable, lazy-loading store instead (single-reader-oriented, not recommended for concurrent production serving). @@ -87,7 +113,7 @@ REFGET_STORE_URL=https://example.com/store uvicorn seqcolapi.main:store_app --po ### DB-backed (PostgreSQL) -If you need a database-backed instance (e.g., for mutable data, advanced queries), use the DB-backed workflow. In a moment I'll show you how to do these steps individually, but if you're in a hurry, the easy way to get a development API running for testing is to just use my very simple shell script like this (no data persistence, just loads demo data): +If you need a database-backed instance (e.g., for mutable data, advanced queries), use the DB-backed workflow. This one needs `pip install 'refget[seqcolapi-db]'`. In a moment I'll show you how to do these steps individually, but if you're in a hurry, the easy way to get a development API running for testing is to just use my very simple shell script like this (no data persistence, just loads demo data): ```console bash deployment/demo_up.sh @@ -244,11 +270,16 @@ interface SeqColResult { ### Models -The objects and attributes are represented as SQLModel objects in `refget/models.py`. To add a new attribute: +The database objects and attributes are represented as SQLModel objects in `refget/models.py` (requires `refget[db]`). To add a new attribute: 1. create a new model. This will create a table for that model, etc. 2. change the function that creates the objects, to populate the new attribute. +HTTP response bodies are *not* defined there. They are plain pydantic models in +`refget/response_models.py`, so that `refget.router` — and therefore the +store-backed service — can be imported without an ORM. Put a new response +schema there unless it genuinely maps to a database table. + ## Example of loading reference fasta datasets: ``` diff --git a/pyproject.toml b/pyproject.toml index 301ab60..e5b2dcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] # Base install must stay light: no fastapi, no uvicorn, no sqlmodel, no # psycopg2. Those are service/database dependencies, gated behind the -# `seqcolapi` extra and behind the refget.seqcolapi / refget.agents / +# `seqcolapi` / `db` extras and behind the refget.seqcolapi / refget.agents / # refget.models module boundaries -- nothing on the `import refget` path # touches them. dependencies = [ @@ -42,17 +42,33 @@ refget = "refget.cli:main" [project.optional-dependencies] test = ["pytest", "pytest-cov>=6.0.0", "fastapi", "httpx", "sqlmodel"] -# Everything needed to run the services in refget/seqcolapi/. psycopg2-binary is -# only used by the PostgreSQL-backed app (refget.seqcolapi.main:app); the -# store-backed app needs none of it. Splitting it into its own extra is a -# judgement call left to the maintainer. -seqcolapi = [ - "fastapi", - "psycopg2-binary", - "sqlmodel", - "uvicorn>=0.30.0", - "ubiquerg>=0.6.1", -] + +# There are three axes here, and they compose: +# +# db the SQLModel/SQLAlchemy layer -- refget.models, refget.agents, +# `refget admin`. Its own extra, not a corner of `seqcolapi`, +# because it is a *library* capability: you can want the ORM +# without wanting a web server. +# seqcolapi the web service -- fastapi + uvicorn, nothing more. This is +# all the RefgetStore-backed deployment needs. No ORM: nothing +# reachable from refget.seqcolapi.app imports sqlmodel (the +# router's response models live in refget/response_models.py, +# and setup_backend imports RefgetDBAgent only in its `engine` +# branch). +# seqcolapi-db the PostgreSQL-backed deployment (seqcolapi.databio.org): +# both of the above. Named rather than left for the user to +# spell as `refget[seqcolapi,db]`, because the two service +# modes are a real deployment choice and should be +# discoverable from the extras list. +# +# `seqcolapi` keeps its name and its meaning-for-most-people (run the service), +# so nobody's existing install line breaks; it just stopped dragging in an ORM +# that the store-backed app never touches. +# +# ubiquerg was listed here and is imported by nothing under refget/ -- dropped. +db = ["psycopg2-binary", "sqlmodel"] +seqcolapi = ["fastapi", "uvicorn>=0.30.0"] +seqcolapi-db = ["refget[db,seqcolapi]"] [project.urls] Homepage = "https://github.com/refgenie/refget" @@ -94,9 +110,12 @@ select = ["E", "F", "I"] ignore = ["F403", "F405", "E501"] [tool.ruff.lint.per-file-ignores] -# The dependency gate has to run before the service imports it guards, so those -# imports are deliberately not at the top of the file. +# The dependency gate has to run before the imports it guards, so those imports +# are deliberately not at the top of the file. "refget/seqcolapi/__init__.py" = ["E402"] +"refget/seqcolapi/dbapp.py" = ["E402"] +"refget/models.py" = ["E402"] +"refget/agents.py" = ["E402"] [tool.ruff.lint.isort] known-first-party = ["refget"] diff --git a/refget/_deps.py b/refget/_deps.py new file mode 100644 index 0000000..9bd18a3 --- /dev/null +++ b/refget/_deps.py @@ -0,0 +1,52 @@ +"""Optional-dependency gates. + +refget ships several modules that require optional dependencies: the service +subpackage needs fastapi, the database modules need sqlmodel/sqlalchemy. Those +dependencies are imported at module level inside the modules that own them -- +see the module-boundary rule in ``refget/models.py`` and +``refget/seqcolapi/__init__.py``. This module supplies the one thing that has +to run *before* those imports: a check that turns a missing extra into a +sentence telling you what to type, instead of a ``ModuleNotFoundError`` raised +from four imports deep in somebody else's package. + +Keep this module import-light. It is loaded on paths that must not pull in the +very dependencies it is checking for, so it probes with +``importlib.util.find_spec`` rather than importing. +""" + +from importlib.util import find_spec + +__all__ = ["require"] + + +def _findable(name: str) -> bool: + try: + return find_spec(name) is not None + except (ImportError, ValueError): + # ImportError: a parent package is itself missing or broken. + # ValueError: the module is present in sys.modules but has no spec. + return False + + +def require(what: str, extra: str, *modules: str) -> None: + """Raise an actionable ImportError if any of ``modules`` is unavailable. + + Probes rather than wrapping the caller's imports, so that a genuine + ImportError raised from *inside* an installed dependency still surfaces as + itself instead of being mistranslated into "not installed". + + Args: + what: What the caller is, phrased for the error message, e.g. + ``"refget.models defines the SQLModel database tables and"``. + extra: The pip extra that supplies the dependencies, e.g. ``"db"``. + Rendered as ``pip install 'refget[db]'``. + *modules: Top-level module names to probe. + """ + missing = [name for name in modules if not _findable(name)] + if not missing: + return + raise ImportError( + f"{what} requires the optional '{extra}' dependencies, and " + f"{', '.join(missing)} {'is' if len(missing) == 1 else 'are'} not " + f"installed.\nInstall them with: pip install 'refget[{extra}]'" + ) diff --git a/refget/agents.py b/refget/agents.py index c0672f4..5a26567 100644 --- a/refget/agents.py +++ b/refget/agents.py @@ -1,22 +1,39 @@ +"""PostgreSQL-backed refget agent. + +Like :mod:`refget.models`, this is a database module: sqlmodel and sqlalchemy +are imported at module level, and the gate is the module boundary -- nothing on +the ``import refget`` path imports it. ``refget.router.setup_backend`` imports +:class:`RefgetDBAgent` inside its ``engine is not None`` branch precisely so +that store-backed deployments never load this file. + +Requires ``pip install 'refget[db]'``. +""" + from __future__ import annotations import json import os -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional import requests -from sqlmodel import Session, SQLModel, create_engine, delete, func, select -if TYPE_CHECKING: - import peppy -from typing import List, Optional +from ._deps import require -from sqlalchemy import URL -from sqlalchemy.engine import Engine as SqlalchemyDatabaseEngine -from sqlalchemy.orm import selectinload +# Fail once, here, with something actionable. Only what this module *imports* +# is gated: psycopg2 is deliberately not required here, because the driver is +# resolved by SQLAlchemy at connect time from the URL, and gating on it would +# reject a caller who supplies an engine for some other database. The `db` +# extra ships psycopg2-binary regardless, since the default URL RefgetDBAgent +# builds is a driverless ``postgresql://``. +require("refget.agents (the PostgreSQL-backed agent)", "db", "sqlmodel", "sqlalchemy") -from .const import _LOGGER, DEFAULT_INHERENT_ATTRS, SEQCOL_SCHEMA_PATH -from .models import ( +from sqlalchemy import URL # noqa: E402 +from sqlalchemy.engine import Engine as SqlalchemyDatabaseEngine # noqa: E402 +from sqlalchemy.orm import selectinload # noqa: E402 +from sqlmodel import Session, SQLModel, create_engine, delete, func, select # noqa: E402 + +from .const import _LOGGER, DEFAULT_INHERENT_ATTRS, SEQCOL_SCHEMA_PATH # noqa: E402 +from .models import ( # noqa: E402 AccessMethod, AccessURL, CollectionNamesAttr, @@ -25,21 +42,23 @@ LengthsAttr, NameLengthPairsAttr, NamesAttr, - PaginationResult, Pangenome, - ResultsSequenceCollections, Sequence, SequenceCollection, SequencesAttr, SortedSequencesAttr, ) -from .utils import ( +from .response_models import PaginationResult, ResultsSequenceCollections # noqa: E402 +from .utils import ( # noqa: E402 build_pangenome_model, calc_jaccard_similarities, compare_seqcols, fasta_to_seqcol_dict, ) +if TYPE_CHECKING: + import peppy + ATTR_TYPE_MAP = { "sequences": SequencesAttr, "names": NamesAttr, diff --git a/refget/models.py b/refget/models.py index b6e4e87..9f5a745 100644 --- a/refget/models.py +++ b/refget/models.py @@ -1,14 +1,41 @@ +"""SQLModel/SQLAlchemy table definitions for the PostgreSQL-backed refget. + +This is the database module. sqlmodel and sqlalchemy are imported at the top of +the file, in ordinary Python style, because **the module boundary is the +gate**: nothing on the ``import refget`` path imports ``refget.models``, so a +base install never loads it and never pays for the ORM. Either you asked for +the database side of refget and this module loads, or it is never loaded at +all -- there is no middle state and no function-local imports to maintain. + +Plain-pydantic response bodies are deliberately *not* defined here; they live +in :mod:`refget.response_models` so that :mod:`refget.router` can serve the +sequence collections API without a database. They are re-exported below, so +``from refget.models import Similarities`` keeps working. +""" + import json import logging from copy import copy from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional +from typing import TYPE_CHECKING, List, Literal, Optional + +from ._deps import require + +# Fail once, here, with something actionable -- not with a bare +# ModuleNotFoundError from the middle of a table definition. +require("refget.models (the SQLModel database tables)", "db", "sqlmodel", "sqlalchemy") -from pydantic import BaseModel, field_serializer, field_validator -from sqlalchemy.types import TypeDecorator -from sqlmodel import JSON, Column, Field, Relationship, SQLModel +from pydantic import field_serializer, field_validator # noqa: E402 +from sqlalchemy.types import TypeDecorator # noqa: E402 +from sqlmodel import JSON, Column, Field, Relationship, SQLModel # noqa: E402 -from .digests import sha512t24u_digest +from .digests import sha512t24u_digest # noqa: E402 +from .response_models import ( # noqa: E402,F401 + PaginatedDigestList, + PaginationResult, + ResultsSequenceCollections, + Similarities, +) if TYPE_CHECKING: from gtars.refget import SequenceCollection as gtarsSequenceCollection @@ -732,36 +759,13 @@ class NameLengthPairsAttr(SQLModel, table=True): collection: List["SequenceCollection"] = Relationship(back_populates="name_length_pairs") -class PaginationResult(BaseModel): - page: int = 0 - page_size: int = 10 - total: int - - -class ResultsSequenceCollections(BaseModel): - """ - Sequence collection results with pagination - """ - - pagination: PaginationResult - results: Dict[str, dict] - - -class Similarities(BaseModel): - """ - Model to contain results from similarities calculations - """ - - similarities: List[Dict[str, Any]] - pagination: PaginationResult - reference_digest: Optional[str] = None - - -class PaginatedDigestList(BaseModel): - """Paginated list of digests, used by list endpoints""" - - pagination: PaginationResult - results: List[str] +# PaginationResult, ResultsSequenceCollections, Similarities and +# PaginatedDigestList used to be defined here. They are plain pydantic response +# bodies with no database involvement, and they now live in +# refget/response_models.py so that refget.router can import them without +# dragging sqlmodel/sqlalchemy into a store-backed deployment. They are +# imported at the top of this module, so `from refget.models import +# Similarities` still works. # This is now a transient attribute, so we don't need to store it in the database. diff --git a/refget/response_models.py b/refget/response_models.py new file mode 100644 index 0000000..d8d1869 --- /dev/null +++ b/refget/response_models.py @@ -0,0 +1,63 @@ +"""Plain-pydantic HTTP response bodies for the sequence collections API. + +These live here, and not in :mod:`refget.models`, purely so that they can be +imported without a database. ``refget.models`` is the *database* module: it +imports sqlmodel and sqlalchemy at module level and therefore requires +``pip install 'refget[db]'``. The types below are ordinary +``pydantic.BaseModel`` subclasses -- they describe what goes over the wire, not +what is stored -- so making :mod:`refget.router` depend on them costs nothing. + +That is what lets the store-backed app (``refget.seqcolapi.create_seqcol_app``, +``uvicorn refget.seqcolapi.main:store_app``) run on ``refget[seqcolapi]`` +alone, with no sqlmodel, sqlalchemy or psycopg2 anywhere in the environment. +Before this split, ``refget.router`` reached sqlalchemy transitively through +``refget.models`` and dragged the whole ORM into a deployment that never +touches PostgreSQL. + +Keep this module free of sqlmodel/sqlalchemy imports. ``refget.models`` +re-exports every name defined here, so existing +``from refget.models import Similarities`` code keeps working. +""" + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel + +__all__ = [ + "PaginationResult", + "ResultsSequenceCollections", + "Similarities", + "PaginatedDigestList", +] + + +class PaginationResult(BaseModel): + page: int = 0 + page_size: int = 10 + total: int + + +class ResultsSequenceCollections(BaseModel): + """ + Sequence collection results with pagination + """ + + pagination: PaginationResult + results: Dict[str, dict] + + +class Similarities(BaseModel): + """ + Model to contain results from similarities calculations + """ + + similarities: List[Dict[str, Any]] + pagination: PaginationResult + reference_digest: Optional[str] = None + + +class PaginatedDigestList(BaseModel): + """Paginated list of digests, used by list endpoints""" + + pagination: PaginationResult + results: List[str] diff --git a/refget/router.py b/refget/router.py index cb86477..65b90d4 100644 --- a/refget/router.py +++ b/refget/router.py @@ -24,6 +24,12 @@ the *same* store from both. To serve a store from a sub-path (or to leave room for serving several stores), prefer ``refget.seqcolapi.create_seqcol_app()``, which returns a self-contained app to mount. + +This module needs fastapi (``pip install 'refget[seqcolapi]'``) but **no +database**: its response models come from :mod:`refget.response_models`, not +from the SQLModel tables in :mod:`refget.models`. Only ``setup_backend(engine=)`` +touches the database, and it imports :class:`refget.agents.RefgetDBAgent` +inside that branch. """ import logging @@ -33,7 +39,7 @@ from .backend import SeqColBackend from .examples import * -from .models import PaginatedDigestList, Similarities +from .response_models import PaginatedDigestList, Similarities _LOGGER = logging.getLogger(__name__) diff --git a/refget/seqcolapi/__init__.py b/refget/seqcolapi/__init__.py index e76cc88..938b752 100644 --- a/refget/seqcolapi/__init__.py +++ b/refget/seqcolapi/__init__.py @@ -1,58 +1,48 @@ """Sequence Collections API service — the optional, server-side half of refget. -Everything in this subpackage requires the optional service dependencies -(``pip install 'refget[seqcolapi]'``). They are imported here at module level, -in ordinary Python style, because **the package boundary is the gate**: nothing -in ``refget/__init__.py`` — or on any other base-install code path — imports -``refget.seqcolapi``. Either you asked for the service and the whole subpackage -loads, or it is never loaded at all. There is no middle state, and no -function-local imports scattered through the service code to maintain. +Everything in this subpackage requires the optional web-service dependencies +(``pip install 'refget[seqcolapi]'``: fastapi and uvicorn). They are imported +here at module level, in ordinary Python style, because **the package boundary +is the gate**: nothing in ``refget/__init__.py`` — or on any other base-install +code path — imports ``refget.seqcolapi``. Either you asked for the service and +the whole subpackage loads, or it is never loaded at all. There is no middle +state, and no function-local imports scattered through the service code to +maintain. -Two applications ship here: +Two applications ship here, and they do **not** cost the same: -* :mod:`refget.seqcolapi.main` — the PostgreSQL-backed ``app`` that runs - seqcolapi.databio.org, plus the ``store_app`` built from the environment. * :func:`create_seqcol_app` — the store-backed factory, which returns a self-contained, mountable app served out of a - :class:`~refget.store.RefgetStore`. + :class:`~refget.store.RefgetStore`. Needs ``refget[seqcolapi]`` only. No + database, no ORM: nothing it reaches imports sqlmodel. +* :mod:`refget.seqcolapi.main` — the PostgreSQL-backed ``app`` that runs + seqcolapi.databio.org, plus the ``store_app`` built from the environment. + Needs ``refget[seqcolapi,db]``, because it goes through + :mod:`refget.agents` and :mod:`refget.models`. Import the package, not its submodules:: from refget.seqcolapi import create_seqcol_app, prepare_store -:mod:`refget.seqcolapi.main` is deliberately *not* imported here: it builds a -FastAPI app at module scope and, absent ``REFGET_STORE_URL`` / -``REFGET_STORE_PATH``, connects to PostgreSQL on import. Ask for it by name -(``uvicorn refget.seqcolapi.main:app``) when you want that. +:mod:`refget.seqcolapi.main` is deliberately *not* imported here: it reads the +environment and builds an app at module scope. Ask for it by name +(``uvicorn refget.seqcolapi.main:store_app``, or ``:app`` for the PostgreSQL +one) when you want that. """ +from refget._deps import require -def _gate(): - """Fail once, here, with something actionable -- not with a bare - ModuleNotFoundError from four imports deep. - - Probes rather than wrapping the imports below, so that a genuine ImportError - inside our own service code still surfaces as itself. - """ - missing = [] - for dep in ("fastapi", "sqlmodel"): # sqlmodel arrives via refget.router - try: - __import__(dep) - except ImportError: - missing.append(dep) - if missing: - raise ImportError( - "refget.seqcolapi requires the optional sequence-collections " - f"service dependencies, and {', '.join(missing)} " - f"{'is' if len(missing) == 1 else 'are'} not installed.\n" - "Install them with: pip install 'refget[seqcolapi]'" - ) - - -_gate() -del _gate +# Fail once, here, with something actionable -- not with a bare +# ModuleNotFoundError from four imports deep. +# +# Only fastapi is required. The store-backed app in .app needs no database: +# refget.router takes its response models from refget.response_models, and +# setup_backend imports RefgetDBAgent lazily inside its engine branch. The +# database gate belongs to .main, which is the only module here that imports +# sqlmodel -- and .main is deliberately not imported by this package. +require("refget.seqcolapi (the sequence collections service)", "seqcolapi", "fastapi") -from refget.const import ALL_VERSIONS +from refget.const import ALL_VERSIONS # noqa: E402 from .app import ( DEFAULT_CACHE_DIR, diff --git a/refget/seqcolapi/dbapp.py b/refget/seqcolapi/dbapp.py new file mode 100644 index 0000000..339cc6f --- /dev/null +++ b/refget/seqcolapi/dbapp.py @@ -0,0 +1,231 @@ +"""The PostgreSQL-backed seqcolapi application (``seqcolapi.databio.org``). + +This is the database half of the service, and it is a separate module for the +same reason :mod:`refget.models` is: **the module boundary is the gate**. +sqlmodel, :mod:`refget.agents` and :mod:`refget.models` are imported here at +top level, in ordinary Python style, and nothing imports this module unless you +actually asked for the PostgreSQL app -- :mod:`refget.seqcolapi.main` reaches +it only through a module-level ``__getattr__`` on the name ``app``. + +That is what lets ``uvicorn seqcolapi.main:store_app`` run on +``pip install 'refget[seqcolapi]'`` with no ORM in the environment at all, +while ``uvicorn seqcolapi.main:app`` still needs +``pip install 'refget[seqcolapi,db]'``. + +Importing this module connects to PostgreSQL (via +``setup_backend(app, engine=RefgetDBAgent().engine)``) unless ``REFGET_STORE_URL`` +or ``REFGET_STORE_PATH`` is set. Ask for the app by name; do not import this +module for its side effects. +""" + +import logging +import os +from contextlib import asynccontextmanager + +from refget._deps import require + +# Fail once, here, with something actionable. This is the one module in the +# service subpackage that needs the database extra, so it carries its own gate +# rather than making every store-only deployment pay for one. +require( + "refget.seqcolapi.dbapp (the PostgreSQL-backed seqcolapi app)", + "db", + "sqlmodel", + "sqlalchemy", +) + +from fastapi import FastAPI, HTTPException # noqa: E402 +from fastapi.middleware.cors import CORSMiddleware # noqa: E402 +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse # noqa: E402 +from sqlmodel import Session, select # noqa: E402 +from starlette.requests import Request # noqa: E402 +from starlette.staticfiles import StaticFiles # noqa: E402 + +from refget.agents import RefgetDBAgent # noqa: E402 +from refget.const import HUMANS_SAMPLE_LIST, MOUSE_SAMPLES_LIST # noqa: E402 +from refget.models import HumanReadableNames # noqa: E402 +from refget.router import ( # noqa: E402 + _ROUTER_CONFIG, + _SAMPLE_DIGESTS, + create_refget_router, + setup_backend, +) + +from .const import ALL_VERSIONS, STATIC_DIRNAME, STATIC_PATH # noqa: E402 +from .examples import * # noqa: E402,F401,F403 + +_LOGGER = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan_loader(app): + """ + Lifespan event to pre-load sample names and their digests + """ + _LOGGER.info("Starting lifespan: Loading sample data...") + + # Initialize backend via setup_backend + setup_backend(app, engine=RefgetDBAgent().engine) + + species_samples = {"human": HUMANS_SAMPLE_LIST, "mouse": MOUSE_SAMPLES_LIST} + + for species, sample_names in species_samples.items(): + try: + _LOGGER.info(f"Loading {len(sample_names)} sample names for {species}") + + with Session(app.state.dbagent.engine) as session: + statement = select(HumanReadableNames).where( + HumanReadableNames.human_readable_name.in_(sample_names) + ) + results = session.exec(statement).all() + + target_digests = [result.digest for result in results] + + _SAMPLE_DIGESTS[species] = target_digests + _LOGGER.info(f"Pre-loaded {len(target_digests)} digests for {species}") + + except Exception as e: + _LOGGER.error(f"Error loading sample data for {species}: {e}") + _SAMPLE_DIGESTS[species] = [] + + _LOGGER.info("Lifespan startup complete: Sample data loaded") + + yield # Application runs here + + # Cleanup + _LOGGER.info("Lifespan shutdown: Cleaning up sample data...") + _SAMPLE_DIGESTS.clear() + + +app = FastAPI( + title="Sequence Collections API", + description="An API providing metadata such as names, lengths, and other values for collections of reference sequences", + version=ALL_VERSIONS["refget_version"], + lifespan=lifespan_loader, +) + +origins = ["*"] + +app.add_middleware( # This is a public API, so we allow all origins + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Configuration +# RefgetStore URL (set to None if not using a backing store) +REFGET_STORE_URL = None # e.g., "s3://my-bucket/store/" + +# This is where the magic happens +# This will add the seqcol endpoints to the app +refget_router = create_refget_router( + sequences=False, + pangenomes=False, + fasta_drs=True, + refget_store_url=REFGET_STORE_URL, +) +app.include_router(refget_router) + + +# Catch-all error handler for any uncaught exceptions, return a 500 error with detailed information +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + return await http_exception_handler( + request, HTTPException(status_code=500, detail=str(exc)) + ) # Pass it to HTTP handler + + +# General Exception Handler (Covers All HTTPExceptions) +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + return JSONResponse( + status_code=exc.status_code, + content={ + "error": f"http-{exc.status_code}", # Generic error code + "detail": exc.detail, # FastAPI-style error message + "status": exc.status_code, + "path": str(request.url), # URL of the request + }, + ) + + +@app.exception_handler(ValueError) +async def value_error_handler(request: Request, exc: Exception): + raise HTTPException(status_code=404, detail=str(exc)) + + +@app.get("favicon.ico", include_in_schema=False) +async def favicon(): + return FileResponse("/static/favicon.ico") + + +@app.get("/", summary="Home page", tags=["General endpoints"], response_class=HTMLResponse) +async def index(request: Request): + """ + Returns a landing page HTML with the server resources ready to download. No inputs required. + """ + with open(f"{STATIC_PATH}/index.html", "r") as file: + content = file.read() + return HTMLResponse(content=content) + + +@app.get("/service-info", summary="GA4GH service info", tags=["General endpoints"]) +async def service_info(): + # Build seqcol capabilities object + seqcol_info = { + "schema": getattr(app.state.dbagent, "schema_dict", None) + if hasattr(app.state, "dbagent") + else None, + "sorted_name_length_pairs": True, + "fasta_drs": {"enabled": _ROUTER_CONFIG.get("fasta_drs", False)}, + } + + # Get backend capabilities + backend = getattr(app.state, "backend", None) + caps = backend.capabilities() if backend and hasattr(backend, "capabilities") else {} + + # Add refget_store info + store_url = _ROUTER_CONFIG.get("refget_store_url") + if store_url: + seqcol_info["refget_store"] = {"enabled": True, "url": store_url, **caps} + else: + seqcol_info["refget_store"] = {"enabled": False} + + # Advertise alias + FHR availability independent of refget_store_url + seqcol_info["aliases"] = { + "enabled": bool( + caps.get("collection_alias_namespaces") or caps.get("sequence_alias_namespaces") + ) + } + seqcol_info["fhr_metadata"] = {"enabled": bool(caps.get("fhr_metadata_collections"))} + + return { + "id": "org.databio.seqcolapi", + "name": "Sequence collections", + "type": { + "group": "org.ga4gh", + "artifact": "refget-seqcol", + "version": ALL_VERSIONS["seqcol_spec_version"], + }, + "description": "An API providing metadata such as names, lengths, and other values for collections of reference sequences", + "organization": {"name": "Databio Lab", "url": "https://databio.org"}, + "contactUrl": "https://github.com/refgenie/refget/issues", + "documentationUrl": "https://seqcolapi.databio.org", + "updatedAt": "2025-02-20T00:00:00Z", + "environment": "dev", + "version": ALL_VERSIONS, + "seqcol": seqcol_info, + } + + +# Mount statics after other routes for lower precedence +app.mount("/", StaticFiles(directory=STATIC_PATH), name=STATIC_DIRNAME) + + +# Bind the database backend at import time, exactly as before the split -- but +# not when the environment names a store, because then the caller wants +# `store_app` and this module was only reached by an incidental attribute look-up. +if not os.environ.get("REFGET_STORE_URL") and not os.environ.get("REFGET_STORE_PATH"): + setup_backend(app, engine=RefgetDBAgent().engine) diff --git a/refget/seqcolapi/main.py b/refget/seqcolapi/main.py index a51a308..f3552d8 100644 --- a/refget/seqcolapi/main.py +++ b/refget/seqcolapi/main.py @@ -1,35 +1,44 @@ -"""The seqcolapi services: the PostgreSQL-backed ``app`` and the store-backed -``store_app``. +"""The seqcolapi service entry points: ``store_app`` and ``app``. -Importing this module builds a FastAPI app at module scope, and -- unless -``REFGET_STORE_URL`` or ``REFGET_STORE_PATH`` is set -- connects to PostgreSQL. -That is why :mod:`refget.seqcolapi` does not import it for you; ask for it by -name (``uvicorn refget.seqcolapi.main:app``). +Two deployments live behind this one module name, and they cost different +things to install: + +``store_app`` -- RefgetStore-backed, no database + Built here, eagerly, when ``REFGET_STORE_URL`` or ``REFGET_STORE_PATH`` is + set. Needs ``pip install 'refget[seqcolapi]'`` and nothing else: no + sqlmodel, no sqlalchemy, no psycopg2 anywhere in the environment:: + + REFGET_STORE_PATH=/path/to/store uvicorn seqcolapi.main:store_app + +``app`` -- PostgreSQL-backed (this is seqcolapi.databio.org) + Lives in :mod:`refget.seqcolapi.dbapp` and is resolved here through a + module-level ``__getattr__``. Needs + ``pip install 'refget[seqcolapi-db]'``:: + + uvicorn seqcolapi.main:app + +The indirection is the point. If ``app`` were built at module scope, importing +this module would import sqlmodel, :mod:`refget.agents` and +:mod:`refget.models`, and every store-only deployment would have to install the +ORM to serve a directory of FASTA digests. Touching the name ``app`` is the +gate; ``uvicorn seqcolapi.main:app`` does exactly that and nothing else does. """ import logging import os -from contextlib import asynccontextmanager - -from fastapi import FastAPI, HTTPException -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse, HTMLResponse, JSONResponse -from sqlmodel import Session, select -from starlette.requests import Request -from starlette.staticfiles import StaticFiles -from refget.agents import RefgetDBAgent -from refget.const import HUMANS_SAMPLE_LIST, MOUSE_SAMPLES_LIST -from refget.models import HumanReadableNames -from refget.router import _ROUTER_CONFIG, _SAMPLE_DIGESTS, create_refget_router, setup_backend +from refget.router import _SAMPLE_DIGESTS from .app import create_seqcol_app -from .const import ALL_VERSIONS, STATIC_DIRNAME, STATIC_PATH -from .examples import * +from .const import ALL_VERSIONS global _LOGGER _LOGGER = logging.getLogger(__name__) +# `app` is resolved by __getattr__ below, and `store_app` only exists when the +# environment names a store -- neither is a module-level binding here. +__all__ = ["app", "store_app", "create_store_app"] # noqa: F822 + def _load_scom_config(store_path: str, remote: bool): """Load SCOM target digests from a JSON config. @@ -80,173 +89,6 @@ def _load_scom_config(store_path: str, remote: bool): _LOGGER.info(f"{key}: {value}") -@asynccontextmanager -async def lifespan_loader(app): - """ - Lifespan event to pre-load sample names and their digests - """ - _LOGGER.info("Starting lifespan: Loading sample data...") - - # Initialize backend via setup_backend - setup_backend(app, engine=RefgetDBAgent().engine) - - species_samples = {"human": HUMANS_SAMPLE_LIST, "mouse": MOUSE_SAMPLES_LIST} - - for species, sample_names in species_samples.items(): - try: - _LOGGER.info(f"Loading {len(sample_names)} sample names for {species}") - - with Session(app.state.dbagent.engine) as session: - statement = select(HumanReadableNames).where( - HumanReadableNames.human_readable_name.in_(sample_names) - ) - results = session.exec(statement).all() - - target_digests = [result.digest for result in results] - - _SAMPLE_DIGESTS[species] = target_digests - _LOGGER.info(f"Pre-loaded {len(target_digests)} digests for {species}") - - except Exception as e: - _LOGGER.error(f"Error loading sample data for {species}: {e}") - _SAMPLE_DIGESTS[species] = [] - - _LOGGER.info("Lifespan startup complete: Sample data loaded") - - yield # Application runs here - - # Cleanup - _LOGGER.info("Lifespan shutdown: Cleaning up sample data...") - _SAMPLE_DIGESTS.clear() - - -app = FastAPI( - title="Sequence Collections API", - description="An API providing metadata such as names, lengths, and other values for collections of reference sequences", - version=ALL_VERSIONS["refget_version"], - lifespan=lifespan_loader, -) - -origins = ["*"] - -app.add_middleware( # This is a public API, so we allow all origins - CORSMiddleware, - allow_origins=origins, - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -# Configuration -# RefgetStore URL (set to None if not using a backing store) -REFGET_STORE_URL = None # e.g., "s3://my-bucket/store/" - -# This is where the magic happens -# This will add the seqcol endpoints to the app -refget_router = create_refget_router( - sequences=False, - pangenomes=False, - fasta_drs=True, - refget_store_url=REFGET_STORE_URL, -) -app.include_router(refget_router) - - -# Catch-all error handler for any uncaught exceptions, return a 500 error with detailed information -@app.exception_handler(Exception) -async def generic_exception_handler(request: Request, exc: Exception): - return await http_exception_handler( - request, HTTPException(status_code=500, detail=str(exc)) - ) # Pass it to HTTP handler - - -# General Exception Handler (Covers All HTTPExceptions) -@app.exception_handler(HTTPException) -async def http_exception_handler(request: Request, exc: HTTPException): - return JSONResponse( - status_code=exc.status_code, - content={ - "error": f"http-{exc.status_code}", # Generic error code - "detail": exc.detail, # FastAPI-style error message - "status": exc.status_code, - "path": str(request.url), # URL of the request - }, - ) - - -@app.exception_handler(ValueError) -async def value_error_handler(request: Request, exc: Exception): - raise HTTPException(status_code=404, detail=str(exc)) - - -@app.get("favicon.ico", include_in_schema=False) -async def favicon(): - return FileResponse("/static/favicon.ico") - - -@app.get("/", summary="Home page", tags=["General endpoints"], response_class=HTMLResponse) -async def index(request: Request): - """ - Returns a landing page HTML with the server resources ready to download. No inputs required. - """ - with open(f"{STATIC_PATH}/index.html", "r") as file: - content = file.read() - return HTMLResponse(content=content) - - -@app.get("/service-info", summary="GA4GH service info", tags=["General endpoints"]) -async def service_info(): - # Build seqcol capabilities object - seqcol_info = { - "schema": getattr(app.state.dbagent, "schema_dict", None) - if hasattr(app.state, "dbagent") - else None, - "sorted_name_length_pairs": True, - "fasta_drs": {"enabled": _ROUTER_CONFIG.get("fasta_drs", False)}, - } - - # Get backend capabilities - backend = getattr(app.state, "backend", None) - caps = backend.capabilities() if backend and hasattr(backend, "capabilities") else {} - - # Add refget_store info - store_url = _ROUTER_CONFIG.get("refget_store_url") - if store_url: - seqcol_info["refget_store"] = {"enabled": True, "url": store_url, **caps} - else: - seqcol_info["refget_store"] = {"enabled": False} - - # Advertise alias + FHR availability independent of refget_store_url - seqcol_info["aliases"] = { - "enabled": bool( - caps.get("collection_alias_namespaces") or caps.get("sequence_alias_namespaces") - ) - } - seqcol_info["fhr_metadata"] = {"enabled": bool(caps.get("fhr_metadata_collections"))} - - return { - "id": "org.databio.seqcolapi", - "name": "Sequence collections", - "type": { - "group": "org.ga4gh", - "artifact": "refget-seqcol", - "version": ALL_VERSIONS["seqcol_spec_version"], - }, - "description": "An API providing metadata such as names, lengths, and other values for collections of reference sequences", - "organization": {"name": "Databio Lab", "url": "https://databio.org"}, - "contactUrl": "https://github.com/refgenie/refget/issues", - "documentationUrl": "https://seqcolapi.databio.org", - "updatedAt": "2025-02-20T00:00:00Z", - "environment": "dev", - "version": ALL_VERSIONS, - "seqcol": seqcol_info, - } - - -# Mount statics after other routes for lower precedence -app.mount("/", StaticFiles(directory=STATIC_PATH), name=STATIC_DIRNAME) - - def create_store_app(store_path: str, remote: bool = False, cache_dir: str = "/tmp/seqcol_cache"): """Create a seqcolapi FastAPI app backed by a RefgetStore (no database). @@ -296,5 +138,22 @@ def _scom_block(): elif _STORE_PATH_ENV: store_app = create_store_app(_STORE_PATH_ENV, remote=False) -if __name__ != "__main__" and not _STORE_URL_ENV and not _STORE_PATH_ENV: - setup_backend(app, engine=RefgetDBAgent().engine) + +def __getattr__(name): + """Resolve ``app`` -- and only ``app`` -- by importing the database module. + + PEP 562. This runs on the first ``seqcolapi.main.app`` look-up, which for a + ``uvicorn seqcolapi.main:app`` deployment is at startup, so the observable + behaviour (build the app, connect to PostgreSQL) is unchanged. What changed + is that a store-only deployment never triggers it and therefore never needs + the `db` extra installed. + """ + if name in ("app", "lifespan_loader", "refget_router", "service_info"): + from . import dbapp + + return getattr(dbapp, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + return sorted(set(globals()) | {"app", "lifespan_loader", "refget_router", "service_info"}) diff --git a/tests/local/test_import_gating.py b/tests/local/test_import_gating.py index 58fa89f..2b227d4 100644 --- a/tests/local/test_import_gating.py +++ b/tests/local/test_import_gating.py @@ -1,28 +1,74 @@ -"""The service subpackage must stay off the base-install import path. +"""Tripwires for the optional-dependency boundaries. -``refget.seqcolapi`` imports fastapi, starlette and (via ``main``) sqlmodel at -module level, which is fine only for as long as nothing on the plain -``import refget`` path reaches it. These tests are the tripwire for that: they -run in a fresh interpreter, because ``sys.modules`` in the pytest process is -already polluted by the rest of the suite. +refget has three install tiers, and each one is a promise about what does *not* +get imported: + +* base (``pip install refget``) -- no fastapi, no sqlmodel, no sqlalchemy. +* ``refget[seqcolapi]`` -- fastapi, but still **no database**. The + RefgetStore-backed service must run with no ORM in the environment. +* ``refget[seqcolapi-db]`` -- everything, for the PostgreSQL app. + +Those promises are enforced by module boundaries, which are easy to break by +adding one innocent-looking top-level import. These tests run in a fresh +interpreter, because ``sys.modules`` in the pytest process is already polluted +by the rest of the suite. """ +import os import subprocess import sys import pytest -GATED_TOP_LEVEL = ("fastapi", "starlette", "uvicorn", "sqlmodel", "sqlalchemy", "psycopg2") +DB_TOP_LEVEL = ("sqlmodel", "sqlalchemy", "psycopg2") +GATED_TOP_LEVEL = ("fastapi", "starlette", "uvicorn") + DB_TOP_LEVEL + +# Blocks top-level distributions from being imported *or* found by find_spec, +# which is how refget._deps.require probes. Simulates a narrower install. +_BLOCKER = ( + "import sys;" + "blocked = {blocked!r};" + "sys.meta_path.insert(0, type('Blocker', (), {{" + " 'find_spec': staticmethod(" + " lambda name, path=None, target=None: " + " (_ for _ in ()).throw(ImportError('blocked')) " + " if name.split('.')[0] in blocked else None)" + "}})());" +) -def _run(code: str) -> str: +def _run(code: str, env: dict = None) -> str: proc = subprocess.run( - [sys.executable, "-c", code], capture_output=True, text=True, check=False + [sys.executable, "-c", code], capture_output=True, text=True, check=False, env=env ) assert proc.returncode == 0, proc.stderr return proc.stdout.strip() +def _blocker(*blocked: str) -> str: + return _BLOCKER.format(blocked=blocked) + + +def _report_loaded(*tops: str) -> str: + """Print the gated top-level packages that ended up in sys.modules.""" + return ( + f"tops = {tops!r};" + "print('LOADED:' + ','.join(sorted({m.split('.')[0] for m in sys.modules} & set(tops))))" + ) + + +def _loaded_line(output: str) -> str: + for line in output.splitlines(): + if line.startswith("LOADED:"): + return line[len("LOADED:") :] + raise AssertionError(f"no LOADED: line in output: {output!r}") + + +# -------------------------------------------------------------------------- +# Base install: nothing heavy. +# -------------------------------------------------------------------------- + + def test_import_refget_does_not_load_the_service_subpackage(): loaded = _run( "import sys, refget;" @@ -32,21 +78,19 @@ def test_import_refget_does_not_load_the_service_subpackage(): def test_import_refget_does_not_load_heavy_dependencies(): - loaded = _run( - "import sys, refget;" - f"tops = {GATED_TOP_LEVEL!r};" - "print(','.join(sorted({m.split('.')[0] for m in sys.modules} & set(tops))))" - ) - assert loaded == "" + out = _run("import sys, refget;" + _report_loaded(*GATED_TOP_LEVEL)) + assert _loaded_line(out) == "" def test_import_refget_cli_does_not_load_heavy_dependencies(): - loaded = _run( - "import sys, refget.cli;" - f"tops = {GATED_TOP_LEVEL!r};" - "print(','.join(sorted({m.split('.')[0] for m in sys.modules} & set(tops))))" - ) - assert loaded == "" + out = _run("import sys, refget.cli;" + _report_loaded(*GATED_TOP_LEVEL)) + assert _loaded_line(out) == "" + + +# -------------------------------------------------------------------------- +# refget[seqcolapi]: fastapi, but no database. This is the boundary the `db` +# extra exists to make real -- a store-backed deployment must not need an ORM. +# -------------------------------------------------------------------------- def test_service_subpackage_is_importable_when_deps_are_present(): @@ -58,23 +102,87 @@ def test_service_subpackage_is_importable_when_deps_are_present(): assert loaded == "True" -@pytest.mark.parametrize("blocked", ["fastapi", "sqlmodel"]) -def test_missing_service_deps_raise_an_actionable_error(blocked): - # Simulate a base install by making one gated dependency unimportable. +def test_router_does_not_load_the_database_layer(): + """refget.router's response models come from refget.response_models. + + If someone moves them back into refget.models, this fails -- and every + store-backed deployment silently starts requiring sqlalchemy again. + """ + out = _run("import sys, refget.router;" + _report_loaded(*DB_TOP_LEVEL)) + assert _loaded_line(out) == "" + + +def test_service_subpackage_does_not_load_the_database_layer(): + out = _run("import sys, refget.seqcolapi;" + _report_loaded(*DB_TOP_LEVEL)) + assert _loaded_line(out) == "" + + +def test_store_backed_app_builds_without_the_database_layer(tmp_path): + """The headline capability: build the store app with sqlmodel unimportable.""" + out = _run( + _blocker(*DB_TOP_LEVEL) + "import refget.seqcolapi as s;" + f"app = s.create_seqcol_app(store_path={str(tmp_path)!r});" + "print(type(app).__name__);" + _report_loaded(*DB_TOP_LEVEL) + ) + assert out.splitlines()[0] == "FastAPI" + assert _loaded_line(out) == "" + + +def test_main_module_store_app_does_not_load_the_database_layer(tmp_path): + """``uvicorn seqcolapi.main:store_app`` must not drag in the ORM. + + ``app`` lives in refget.seqcolapi.dbapp and is resolved through main's + module-level ``__getattr__``, so importing main and reading ``store_app`` + off it stays database-free. + """ + env = dict(os.environ, REFGET_STORE_PATH=str(tmp_path)) + out = _run( + "import sys, refget.seqcolapi.main as m;" + "print(type(m.store_app).__name__);" + _report_loaded(*DB_TOP_LEVEL), + env=env, + ) + assert out.splitlines()[0] == "FastAPI" + assert _loaded_line(out) == "" + + +def test_missing_fastapi_points_at_the_seqcolapi_extra(): message = _run( - "import sys;" - f"blocked = {blocked!r};" - "sys.meta_path.insert(0, type('Blocker', (), {" - " 'find_spec': staticmethod(" - " lambda name, path=None, target=None: " - " (_ for _ in ()).throw(ImportError('blocked')) " - " if name.split('.')[0] == blocked else None)" - "})());" - "\ntry:\n" + _blocker("fastapi") + "\ntry:\n" " import refget.seqcolapi\n" "except ImportError as e:\n" " print(str(e).replace(chr(10), ' '))\n" ) assert "refget[seqcolapi]" in message assert "pip install" in message - assert blocked in message + assert "fastapi" in message + + +# -------------------------------------------------------------------------- +# refget[db]: the ORM modules name their own extra. +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("module", ["refget.models", "refget.agents", "refget.seqcolapi.dbapp"]) +def test_missing_sqlmodel_points_at_the_db_extra(module): + message = _run( + _blocker(*DB_TOP_LEVEL) + "\ntry:\n" + f" import {module}\n" + "except ImportError as e:\n" + " print(str(e).replace(chr(10), ' '))\n" + ) + assert "refget[db]" in message, message + assert "pip install" in message, message + assert "sqlmodel" in message, message + # Names itself, rather than being a bare ModuleNotFoundError from four + # imports deep in somebody else's package. + assert module in message, message + + +def test_models_still_reexports_the_response_models(): + """Moving them to refget.response_models must not break existing imports.""" + loaded = _run( + "from refget.models import PaginatedDigestList, PaginationResult, " + "ResultsSequenceCollections, Similarities;" + "print('ok')" + ) + assert loaded == "ok" From e551ce0007abc23c0fc8331a175438d3c586ec3b Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 17:44:48 -0400 Subject: [PATCH 07/21] Gate refget.router on fastapi too --- pyproject.toml | 1 + refget/router.py | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e5b2dcc..c0d7e21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,7 @@ ignore = ["F403", "F405", "E501"] "refget/seqcolapi/dbapp.py" = ["E402"] "refget/models.py" = ["E402"] "refget/agents.py" = ["E402"] +"refget/router.py" = ["E402"] [tool.ruff.lint.isort] known-first-party = ["refget"] diff --git a/refget/router.py b/refget/router.py index 65b90d4..63806f5 100644 --- a/refget/router.py +++ b/refget/router.py @@ -34,12 +34,19 @@ import logging -from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response -from fastapi.responses import StreamingResponse +from ._deps import require -from .backend import SeqColBackend -from .examples import * -from .response_models import PaginatedDigestList, Similarities +# This module is a documented entry point in its own right +# (``from refget.router import create_refget_router``), so it carries its own +# gate rather than relying on being reached through refget.seqcolapi. +require("refget.router (the sequence collections router)", "seqcolapi", "fastapi") + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response # noqa: E402 +from fastapi.responses import StreamingResponse # noqa: E402 + +from .backend import SeqColBackend # noqa: E402 +from .examples import * # noqa: E402 +from .response_models import PaginatedDigestList, Similarities # noqa: E402 _LOGGER = logging.getLogger(__name__) From 1c83adb91d5ef65975387aed8885e657fbf7e859 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 17:44:48 -0400 Subject: [PATCH 08/21] Name the extra when no web server --- refget/cli/store.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/refget/cli/store.py b/refget/cli/store.py index 2d6c56b..01b3a54 100644 --- a/refget/cli/store.py +++ b/refget/cli/store.py @@ -1846,7 +1846,15 @@ def serve( try: import uvicorn except ImportError: - print_error("uvicorn is required: pip install uvicorn", EXIT_FAILURE) + # Name the extra, not just the package: fastapi is needed here too, and + # `refget[seqcolapi]` is the supported way to ask for both. No `db` + # needed -- serving a store touches no ORM. + print_error( + "Serving requires the optional web-service dependencies " + "(fastapi, uvicorn), and uvicorn is not installed.\n" + "Install them with: pip install 'refget[seqcolapi]'", + EXIT_FAILURE, + ) from refget.backend import RefgetStoreBackend From eebb1e7a28d2c1af4c7a9693ed8dd5125e0da264 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 17:44:48 -0400 Subject: [PATCH 09/21] Build the store image without db packages --- deployment/seqcolapi-store/Dockerfile | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/deployment/seqcolapi-store/Dockerfile b/deployment/seqcolapi-store/Dockerfile index 10fad3d..b53532c 100644 --- a/deployment/seqcolapi-store/Dockerfile +++ b/deployment/seqcolapi-store/Dockerfile @@ -1,11 +1,15 @@ FROM tiangolo/uvicorn-gunicorn:python3.11-slim LABEL authors="Nathan Sheffield" -RUN pip install https://github.com/refgenie/refget/archive/dev.zip -RUN pip install gtars -COPY seqcolapi/requirements/requirements-seqcolapi.txt requirements-seqcolapi.txt -RUN pip install -r requirements-seqcolapi.txt --no-cache-dir +# The store-backed service needs no database packages. `refget[seqcolapi]` pulls +# fastapi and uvicorn only, so sqlmodel, sqlalchemy and psycopg2 stay out of the +# image. The Postgres-backed service is a separate deployment and installs +# `refget[seqcolapi-db]`. +# +# seqcolapi now ships inside the refget wheel (refget/seqcolapi/), so there is +# nothing to COPY -- the app is importable straight from site-packages. gtars is +# a base dependency of refget and no longer needs its own install line. +RUN pip install --no-cache-dir \ + "refget[seqcolapi] @ https://github.com/refgenie/refget/archive/dev.zip" -COPY seqcolapi/ /app/seqcolapi - -CMD ["uvicorn", "seqcolapi.main:store_app", "--host", "0.0.0.0", "--port", "80"] +CMD ["uvicorn", "refget.seqcolapi.main:store_app", "--host", "0.0.0.0", "--port", "80"] From d619f0e1a13160efcb8ed77bb72c9b3480c98a25 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 18:25:34 -0400 Subject: [PATCH 10/21] Format python blocks in markdown docs --- models_notes.md | 2 -- refget/schemas/README.md | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models_notes.md b/models_notes.md index b85b901..c5f3b7a 100644 --- a/models_notes.md +++ b/models_notes.md @@ -49,7 +49,6 @@ Digest is a string, and value is a JSON column, where I put the content of the a This is some old deprecated models I had been working on, under different modeling approaches; probably can delete these soon if I don't need to revisit it. ```python - # DigestedSequenceCollection = create_model( # 'DigestedSequenceCollection', digest=(str, ""), # __base__= SequenceCollection, @@ -134,7 +133,6 @@ This is some old deprecated models I had been working on, under different modeli if False: - from pydantic import create_model from typing import List from sqlmodel import Field, ARRAY, SQLModel, create_engine, Column, Float, Relationship diff --git a/refget/schemas/README.md b/refget/schemas/README.md index f8c1891..a140082 100644 --- a/refget/schemas/README.md +++ b/refget/schemas/README.md @@ -42,6 +42,7 @@ class AccessionsAttr(SQLModel, table=True): value: list = Field(sa_column=Column(JSON), default_factory=list) collection: List["SequenceCollection"] = Relationship(back_populates="accessions") + class SequenceCollection(SQLModel, table=True): # ... existing fields ... accessions_digest: str = Field(foreign_key="accessionsattr.digest") @@ -107,6 +108,7 @@ class MetadataAttr(SQLModel, table=True): value: dict = Field(sa_column=Column(JSON)) collection: List["SequenceCollection"] = Relationship(back_populates="metadata") + class SequenceCollection(SQLModel, table=True): # ... existing fields ... metadata_id: str = Field(foreign_key="metadataattr.id") From a0a6e92c57921e9fb3d63ea95b31dd1dce239bbe Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 18:25:34 -0400 Subject: [PATCH 11/21] Read routes from the OpenAPI schema --- tests/local/test_app_factory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/local/test_app_factory.py b/tests/local/test_app_factory.py index ee93667..e5135eb 100644 --- a/tests/local/test_app_factory.py +++ b/tests/local/test_app_factory.py @@ -156,7 +156,9 @@ class TestDeferredBackend: def test_routes_exist_before_a_store_is_bound(self): seqcol = create_seqcol_app(defer_backend=True, store_url="https://a/store/", cors=False) assert not hasattr(seqcol.state, "backend") - assert "/list/collection" in {r.path for r in seqcol.routes} + # Read paths from the OpenAPI schema: `app.routes` nests included routers + # differently across starlette versions, but the schema is public API. + assert "/list/collection" in seqcol.openapi()["paths"] # service-info still answers; it just reports no backend capabilities. info = TestClient(seqcol).get("/service-info").json() assert info["seqcol"]["refget_store"]["url"] == "https://a/store/" From 2e7260b9ba80a8628783fa9d3f2c7fe638730462 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 20:29:23 -0400 Subject: [PATCH 12/21] Delete the __main__ modules that never worked --- refget/seqcolapi/__main__.py | 10 ---------- seqcolapi/__main__.py | 5 ----- 2 files changed, 15 deletions(-) delete mode 100644 refget/seqcolapi/__main__.py delete mode 100644 seqcolapi/__main__.py diff --git a/refget/seqcolapi/__main__.py b/refget/seqcolapi/__main__.py deleted file mode 100644 index 3ea99ba..0000000 --- a/refget/seqcolapi/__main__.py +++ /dev/null @@ -1,10 +0,0 @@ -import sys - -from .main import main - -if __name__ == "__main__": - try: - sys.exit(main()) - except KeyboardInterrupt: - print("Program canceled by user") - sys.exit(1) diff --git a/seqcolapi/__main__.py b/seqcolapi/__main__.py deleted file mode 100644 index 66eb1a2..0000000 --- a/seqcolapi/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compatibility shim: ``python -m seqcolapi`` -> ``python -m refget.seqcolapi``.""" - -import runpy - -runpy.run_module("refget.seqcolapi", run_name="__main__", alter_sys=True) From 60f1646b5bfcd8d3bb8d7a159e35fbc12161f72f Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 20:29:24 -0400 Subject: [PATCH 13/21] Point run instructions at refget.seqcolapi --- README.md | 39 ++++++++++++++++++++++++++++--------- deployment/demo_up.sh | 2 +- deployment/store_demo_up.sh | 2 +- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 7ef51e5..f5a89a8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This repository includes: 1. `/refget`: The `refget` Python package, which provides a Python interface to both remote and local use of refget standards. It has clients and functions for both refget sequences and refget sequence collections (seqcol). 2. `/refget/seqcolapi`: Sequence collections API software, a FastAPI wrapper built on top of the `refget` package. It provides a bare-bones Sequence Collections API service. It ships in the `refget` wheel, but its dependencies do not — see [Installation](#installation). Nothing on the plain `import refget` path imports this subpackage, so a base install never pays for fastapi/uvicorn/sqlmodel/psycopg2. -3. `/seqcolapi`: A thin compatibility shim re-exporting `refget.seqcolapi`, kept so existing deployments that run `uvicorn seqcolapi.main:app` (or `:store_app`) keep working. New code should import `refget.seqcolapi`. +3. `/seqcolapi`: A thin compatibility shim re-exporting `refget.seqcolapi`, kept so existing deployments that run `uvicorn seqcolapi.main:app` (or `:store_app`) keep working. **This shim is deliberately excluded from the wheel** — it is importable only from a checkout of this repository (or from a deployment that `COPY`s the directory), never from `pip install refget`. Every instruction below therefore uses `refget.seqcolapi.main`, which works everywhere. 4. `/deployment`: Server configurations for demo instances and public deployed instances. There are also github workflows (in `.github/workflows`) that deploy the demo server instance from this repository. 5. `/test_fasta` and `/test_api`: Dummy data and a compliance test, to test external implementations of the Refget Sequence Collections API. 6. `/frontend`: a React seqcolapi front-end. @@ -23,8 +23,15 @@ server and no ORM. Everything heavier is an extra, and the extras compose: | --- | --- | --- | | `pip install refget` | — | The Python library and the `refget` CLI: digests, `RefgetStore`, API clients, compliance. No fastapi, no sqlalchemy. | | `pip install 'refget[db]'` | sqlmodel, psycopg2-binary | The SQLModel layer: `refget.models`, `refget.agents.RefgetDBAgent`, `refget admin`. A library capability — you can want the ORM without wanting a server. | -| `pip install 'refget[seqcolapi]'` | fastapi, uvicorn | **Serving a RefgetStore.** `uvicorn seqcolapi.main:store_app`, `refget store serve`, `refget.seqcolapi.create_seqcol_app`. No database of any kind. | -| `pip install 'refget[seqcolapi-db]'` | both of the above | **The PostgreSQL-backed service** (`uvicorn seqcolapi.main:app`), i.e. what runs seqcolapi.databio.org. | +| `pip install 'refget[seqcolapi]'` | fastapi, uvicorn | **Serving a RefgetStore.** `uvicorn refget.seqcolapi.main:store_app`, `refget store serve`, `refget.seqcolapi.create_seqcol_app`. No database of any kind. | +| `pip install 'refget[seqcolapi-db]'` | both of the above | **The PostgreSQL-backed service** (`uvicorn refget.seqcolapi.main:app`), i.e. what runs seqcolapi.databio.org. | + +The importable module path is `refget.seqcolapi.main`, not `seqcolapi.main`. The +bare `seqcolapi` package in this repository is a compatibility shim that is +**not** shipped in the wheel; `uvicorn seqcolapi.main:store_app` resolves only +from a checkout of this repository, and raises +`ModuleNotFoundError: No module named 'seqcolapi'` in a pip-installed +environment. The two service extras correspond to the two deployment modes described under [Development and deployment: Backend](#development-and-deployment-backend). The @@ -78,6 +85,9 @@ For safe concurrent serving, the store is fully loaded and converted to a read-o #### Quick start +*Requires a checkout of this repository* (the script and the demo FASTA files +live here): + ```console bash deployment/store_demo_up.sh ``` @@ -91,16 +101,25 @@ No Docker or database required. #### Step-by-step -1. Build a store from FASTA files: +1. Build a store from FASTA files. `data_loaders/` is not part of the wheel, so + this step needs a checkout of this repository: ```console python data_loaders/demo_build_store.py test_fasta /tmp/refget_demo_store ``` -2. Start the store-backed API: + From a pip install, build a store with the CLI instead: + +```console +refget store init -p /tmp/refget_demo_store +refget store add -p /tmp/refget_demo_store +``` + +2. Start the store-backed API. This works anywhere `refget[seqcolapi]` is + installed: ```console -REFGET_STORE_PATH=/tmp/refget_demo_store uvicorn seqcolapi.main:store_app --reload --port 8100 +REFGET_STORE_PATH=/tmp/refget_demo_store uvicorn refget.seqcolapi.main:store_app --reload --port 8100 ``` #### Remote store @@ -108,7 +127,7 @@ REFGET_STORE_PATH=/tmp/refget_demo_store uvicorn seqcolapi.main:store_app --relo To run against a remote (S3) store: ```console -REFGET_STORE_URL=https://example.com/store uvicorn seqcolapi.main:store_app --port 8100 +REFGET_STORE_URL=https://example.com/store uvicorn refget.seqcolapi.main:store_app --port 8100 ``` ### DB-backed (PostgreSQL) @@ -167,12 +186,14 @@ refget add-fasta -p test_fasta/test_fasta_metadata.csv -r test_fasta Run the demo `seqcolapi` service like this: ``` -uvicorn seqcolapi.main:app --reload --port 8100 +uvicorn refget.seqcolapi.main:app --reload --port 8100 ``` #### Running with docker -To build the docker file, first build the image from the root of this repository: +To build the docker file, first build the image from a checkout of this +repository (the build context is the `seqcolapi/` compatibility shim, which +exists only here): ``` docker build -f deployment/dockerhub/Dockerfile -t databio/seqcolapi seqcolapi diff --git a/deployment/demo_up.sh b/deployment/demo_up.sh index 96595c8..deaf042 100644 --- a/deployment/demo_up.sh +++ b/deployment/demo_up.sh @@ -35,7 +35,7 @@ docker run --rm -d --name refget-postgres -p 127.0.0.1:5432:5432 \ echo "Postgres is up - continuing" echo "Running uvicorn API service..." -uvicorn seqcolapi.main:app --reload --port 8100 & +uvicorn refget.seqcolapi.main:app --reload --port 8100 & PID=$! echo "Loading demo sequence collections..." diff --git a/deployment/store_demo_up.sh b/deployment/store_demo_up.sh index 7e3bb24..c1e9808 100755 --- a/deployment/store_demo_up.sh +++ b/deployment/store_demo_up.sh @@ -44,7 +44,7 @@ STORE_HTTP_PID=$! export REFGET_STORE_HTTP_URL="http://localhost:$STORE_HTTP_PORT" echo "Running store-backed uvicorn API service..." -uvicorn seqcolapi.main:store_app --reload --port ${SEQCOLAPI_PORT:-8100} & +uvicorn refget.seqcolapi.main:store_app --reload --port ${SEQCOLAPI_PORT:-8100} & PID=$! echo "" From 11df7598c854c85f6cb1e0b65a473dd970e9ece9 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 20:29:39 -0400 Subject: [PATCH 14/21] Import STATIC_PATH from the shipped package --- refget/cli/store.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/refget/cli/store.py b/refget/cli/store.py index 01b3a54..7569c69 100644 --- a/refget/cli/store.py +++ b/refget/cli/store.py @@ -113,8 +113,8 @@ def _find_spa_dir(frontend_dir: Optional[Path]) -> Path: Resolution order: 1. ``--frontend-dir`` override (must contain index.html). - 2. Packaged build at ``seqcolapi.const.STATIC_PATH`` (when the SPA has - been bundled into the installed package). + 2. Packaged build at ``refget.seqcolapi.const.STATIC_PATH`` (when the + SPA has been bundled into the installed package). 3. Repo-relative ``frontend/dist`` for development. Auto-discovered candidates must look like the Explorer build (see @@ -131,10 +131,16 @@ def _find_spa_dir(frontend_dir: Optional[Path]) -> Path: candidates: List[Path] = [] try: - from seqcolapi.const import STATIC_PATH + # `refget.seqcolapi`, not the top-level `seqcolapi` shim: the shim is + # excluded from the wheel, so importing it silently fails on every + # pip-installed environment. Importing the submodule runs the package + # gate in refget/seqcolapi/__init__.py, so ImportError is the expected + # miss when fastapi is not installed -- and it is the only one we want + # to swallow here. + from refget.seqcolapi.const import STATIC_PATH candidates.append(Path(STATIC_PATH)) - except Exception: + except ImportError: pass # repos/refget/refget/cli/store.py -> parents[2] == repos/refget candidates.append(Path(__file__).resolve().parents[2] / "frontend" / "dist") From 446e75d4b0619fa26d112268bb296efce60cac69 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 20:29:47 -0400 Subject: [PATCH 15/21] Split the factory names, fix the isolation claim --- refget/__init__.py | 2 +- refget/seqcolapi/__init__.py | 13 ++++++------ refget/seqcolapi/app.py | 40 ++++++++++++++--------------------- refget/seqcolapi/main.py | 27 +++++++++++++++-------- tests/integration/conftest.py | 2 +- 5 files changed, 43 insertions(+), 41 deletions(-) diff --git a/refget/__init__.py b/refget/__init__.py index 51a2aa4..80c5b03 100644 --- a/refget/__init__.py +++ b/refget/__init__.py @@ -7,7 +7,7 @@ from refget.utils import compare_seqcols, validate_seqcol, seqcol_digest from refget.clients import SequenceCollectionClient, FastaDrsClient from refget.router import create_refget_router - from refget.seqcolapi import create_seqcol_app, create_store_app, prepare_store + from refget.seqcolapi import create_seqcol_app, prepare_store from refget.agents import RefgetDBAgent `refget.seqcolapi` (the API service) and `refget.router` require the optional diff --git a/refget/seqcolapi/__init__.py b/refget/seqcolapi/__init__.py index 938b752..147ef05 100644 --- a/refget/seqcolapi/__init__.py +++ b/refget/seqcolapi/__init__.py @@ -14,11 +14,14 @@ * :func:`create_seqcol_app` — the store-backed factory, which returns a self-contained, mountable app served out of a :class:`~refget.store.RefgetStore`. Needs ``refget[seqcolapi]`` only. No - database, no ORM: nothing it reaches imports sqlmodel. + database, no ORM: nothing it reaches imports sqlmodel. This is the only app + factory exported here; it is the one to call from your own code. * :mod:`refget.seqcolapi.main` — the PostgreSQL-backed ``app`` that runs - seqcolapi.databio.org, plus the ``store_app`` built from the environment. - Needs ``refget[seqcolapi,db]``, because it goes through - :mod:`refget.agents` and :mod:`refget.models`. + seqcolapi.databio.org, plus the ``store_app`` built from the environment by + :func:`refget.seqcolapi.main.create_seqcolapi_store_app` (``create_seqcol_app`` + plus seqcolapi's own service identity and SCOM block). ``app`` needs + ``refget[seqcolapi,db]``, because it goes through :mod:`refget.agents` and + :mod:`refget.models`. Import the package, not its submodules:: @@ -47,7 +50,6 @@ from .app import ( DEFAULT_CACHE_DIR, create_seqcol_app, - create_store_app, load_seqcol_schema, prepare_store, store_service_info, @@ -60,7 +62,6 @@ "STATIC_DIRNAME", "STATIC_PATH", "create_seqcol_app", - "create_store_app", "load_seqcol_schema", "prepare_store", "store_service_info", diff --git a/refget/seqcolapi/app.py b/refget/seqcolapi/app.py index 128e168..07e4c61 100644 --- a/refget/seqcolapi/app.py +++ b/refget/seqcolapi/app.py @@ -15,14 +15,28 @@ of the same app therefore does **not** give you two stores; both mounts resolve through the same ``app.state.backend``. Returning a self-contained sub-application instead means each store gets its own ``state.backend``, its -own service-info and its own freshness policy, so a host app can do:: +own ``/service-info`` route and its own freshness middleware, so a host app can +do:: host.mount("/jungle", create_seqcol_app(store_path=url_a, remote=True)) host.mount("/other", create_seqcol_app(store_path=url_b, remote=True)) Serving several stores from one process is not a requirement today, and nothing here implements it. The point is only that the shape does not preclude -it. +it -- and the shape is not the whole story. Two process-global dicts in +:mod:`refget.router` are shared by every router and every app in the process: + +* ``_ROUTER_CONFIG``, which :func:`refget.router.create_refget_router` + overwrites on each call, so it ends up describing whichever router was + created last. This app's ``/service-info`` never reads it (it closes over its + own arguments instead), but :mod:`refget.seqcolapi.dbapp` and refgenie's + server both do, so a process that builds several routers cannot trust it. +* ``_SAMPLE_DIGESTS``, the SCOM target digests, which are likewise per-process + and not per-app. + +So: per-mount backend, service-info and freshness, yes. Per-mount *router +configuration*, no. Genuinely serving several distinct stores from one process +would have to deal with those two globals first. Typical use:: @@ -288,25 +302,3 @@ async def service_info(): ) return app - - -def create_store_app( - store_path: str, - remote: bool = False, - cache_dir: str = DEFAULT_CACHE_DIR, - **kwargs, -): - """Create a standalone store-backed seqcol app (no database). - - Thin convenience wrapper around :func:`create_seqcol_app` for serving a - single store at the root of its own process. - - Args: - store_path: Path to store on disk, or S3/HTTP URL for remote stores. - remote: If True, open as a remote store. - cache_dir: Local cache directory for remote stores. - - Returns: - FastAPI app with store-backed seqcol endpoints at the root. - """ - return create_seqcol_app(store_path=store_path, remote=remote, cache_dir=cache_dir, **kwargs) diff --git a/refget/seqcolapi/main.py b/refget/seqcolapi/main.py index f3552d8..c1c8405 100644 --- a/refget/seqcolapi/main.py +++ b/refget/seqcolapi/main.py @@ -37,7 +37,7 @@ # `app` is resolved by __getattr__ below, and `store_app` only exists when the # environment names a store -- neither is a module-level binding here. -__all__ = ["app", "store_app", "create_store_app"] # noqa: F822 +__all__ = ["app", "store_app", "create_seqcolapi_store_app"] # noqa: F822 def _load_scom_config(store_path: str, remote: bool): @@ -89,13 +89,22 @@ def _load_scom_config(store_path: str, remote: bool): _LOGGER.info(f"{key}: {value}") -def create_store_app(store_path: str, remote: bool = False, cache_dir: str = "/tmp/seqcol_cache"): - """Create a seqcolapi FastAPI app backed by a RefgetStore (no database). +def create_seqcolapi_store_app( + store_path: str, remote: bool = False, cache_dir: str = "/tmp/seqcol_cache" +): + """Create *the seqcolapi deployment's* store-backed app (no database). - Thin wrapper over :func:`refget.seqcolapi.create_seqcol_app`, which owns the - shared store-backed wiring (readonly store, backend, router, freshness, - GA4GH service-info). Everything seqcolapi adds on top is SCOM, which is not - a refget concern, so it is injected through ``service_info_extra``. + Not to be confused with :func:`refget.seqcolapi.create_seqcol_app`, which is + the generic factory. This function is the seqcolapi.databio.org flavour of + it: same wiring, but a different service identity + (``org.databio.seqcolapi.store`` rather than the generic + ``org.ga4gh.seqcol.store``) and an extra SCOM block in ``/service-info``. + Everything else -- readonly store, backend, router, freshness, the GA4GH + service-info body -- is owned by ``create_seqcol_app``; SCOM is not a refget + concern, so it is injected through ``service_info_extra``. + + Renamed from ``create_store_app`` so that the two factories in this package + no longer share a name while differing in service identity. Args: store_path: Path to store on disk, or S3 URL for remote stores. @@ -134,9 +143,9 @@ def _scom_block(): _STORE_PATH_ENV = os.environ.get("REFGET_STORE_PATH") if _STORE_URL_ENV: - store_app = create_store_app(_STORE_URL_ENV, remote=True) + store_app = create_seqcolapi_store_app(_STORE_URL_ENV, remote=True) elif _STORE_PATH_ENV: - store_app = create_store_app(_STORE_PATH_ENV, remote=False) + store_app = create_seqcolapi_store_app(_STORE_PATH_ENV, remote=False) def __getattr__(name): diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 5e49ba1..805c37e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -180,7 +180,7 @@ def store_test_server(tmp_path_factory): and runs a store-backed uvicorn server in a background thread. No database required. - Note: We build the app manually (instead of create_store_app) so we can + Note: We build the app manually (instead of create_seqcol_app) so we can reuse the same store instance that loaded the FASTAs, preserving correct array ordering. Opening a new store from the same path would lose FASTA-order due to a gtars hash-map ordering issue. From d9ebf83f9ac9d77b174065600f1843733e83639c Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 20:30:06 -0400 Subject: [PATCH 16/21] Build store serve through the seqcol factory --- refget/cli/store.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/refget/cli/store.py b/refget/cli/store.py index 7569c69..23fe8db 100644 --- a/refget/cli/store.py +++ b/refget/cli/store.py @@ -1839,6 +1839,10 @@ def serve( ): """Serve a seqcol API backed by a RefgetStore (no database required). + The app is built by :func:`refget.seqcolapi.create_seqcol_app`, the same + factory the deployments use, so this serves the same routes they do + (including ``/service-info``). + By default the store is fully loaded and converted to a ReadonlyRefgetStore, whose methods borrow immutably and are safe to share across request threads for concurrent serving. Use --lazy to skip loading and serve from the mutable @@ -1862,8 +1866,6 @@ def serve( EXIT_FAILURE, ) - from refget.backend import RefgetStoreBackend - if remote: store = _load_store(path=None, remote=remote) elif path: @@ -1871,27 +1873,27 @@ def serve( else: store = _load_store(None) - if lazy: - backend = RefgetStoreBackend(store) - else: + if not lazy: # Load all collections and convert to a thread-safe readonly store. # Sequence endpoints are disabled below, so sequences are not loaded; # enabling them later should also pass load_sequences=True here. - readonly_store = _into_readonly(store, load_sequences=False) - backend = RefgetStoreBackend(readonly_store) - - from fastapi import FastAPI - - from refget.router import create_refget_router - - app = FastAPI(title="Sequence Collections API (Store-backed)") - app.state.backend = backend - router = create_refget_router( + store = _into_readonly(store, load_sequences=False) + + # Go through the shared factory rather than hand-wiring FastAPI here, so + # that this CLI serves exactly what the deployments serve -- including + # /service-info, which a hand-rolled app silently omitted. The store is + # already open (and possibly already readonly), so it is passed in directly; + # freshness polling is off because that is a deployment concern and needs a + # store_path to poll. + from refget.seqcolapi import create_seqcol_app + + app = create_seqcol_app( + store=store, + store_url=remote, sequences=False, pangenomes=False, - refget_store_url=remote, + freshness=False, ) - app.include_router(router) typer.echo(f"Serving store-backed seqcol API on {host}:{port}") uvicorn.run(app, host=host, port=port) From 10b363f17b9631ed28bce697e4542f301cae429e Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 20:30:06 -0400 Subject: [PATCH 17/21] Test the store image the way we build it --- scripts/test-store-docker.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/test-store-docker.sh b/scripts/test-store-docker.sh index 2dcd790..ef68ae0 100755 --- a/scripts/test-store-docker.sh +++ b/scripts/test-store-docker.sh @@ -66,12 +66,14 @@ rsync -a --exclude='.git' --exclude='node_modules' --exclude='__pycache__' \ echo "Building Docker image from local source..." docker build -t "$IMAGE_NAME" -f - "$CONTEXT_DIR" < Date: Sat, 25 Jul 2026 20:30:06 -0400 Subject: [PATCH 18/21] Restore the changelog and log these changes --- changelog.md | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 changelog.md diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..fe72b8e --- /dev/null +++ b/changelog.md @@ -0,0 +1,121 @@ +# Changelog + +All notable changes to the refget package will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- **`refget.seqcolapi` ships in the wheel.** The service code moved from the + top-level `seqcolapi/` directory into `refget/seqcolapi/`, so a + `pip install 'refget[seqcolapi]'` can serve the API with no repository + checkout. Run it as `uvicorn refget.seqcolapi.main:store_app` (store-backed) + or `uvicorn refget.seqcolapi.main:app` (PostgreSQL-backed). +- **`refget.seqcolapi.create_seqcol_app`**: a store-backed app factory that + returns a self-contained, mountable FastAPI app, so a host application can + mount the seqcol API under a prefix and keep its own `/service-info`. +- **`create_refget_router(mount_prefix=...)`**: lets the compliance endpoints + self-target the seqcol service rather than the server root when the router is + included under a prefix. +- **`db` extra**: `pip install 'refget[db]'` installs just the SQLModel layer + (`refget.models`, `refget.agents`, `refget admin`) with no web server. +- Importing a module without its extra now raises an error naming the extra to + install, rather than a bare `ModuleNotFoundError`. + +### Changed + +- **Distribution: `sqlmodel` is no longer a base dependency.** It moved out of + the base install into the new `db` extra, and out of the `seqcolapi` extra, + which now installs fastapi and uvicorn only. **If you import + `refget.models`, `refget.agents`, or run `refget admin`, install + `refget[db]`.** The PostgreSQL-backed service now needs + `refget[seqcolapi-db]` (equivalent to `refget[seqcolapi,db]`); the + store-backed service needs only `refget[seqcolapi]` and pulls no ORM. +- **`ubiquerg` removed from the `seqcolapi` extra.** Nothing under `refget/` + imports it. +- `refget store serve` now builds its app with + `refget.seqcolapi.create_seqcol_app` instead of hand-wiring FastAPI, so it + serves the same routes the deployments do — notably `/service-info`, which it + previously omitted — plus a permissive CORS middleware. +- `refget.seqcolapi.main.create_store_app` was renamed to + `create_seqcolapi_store_app`, so that it is not confused with the generic + `refget.seqcolapi.create_seqcol_app`: it carries the seqcolapi.databio.org + service identity (`org.databio.seqcolapi.store`) and the SCOM service-info + block. The `uvicorn refget.seqcolapi.main:store_app` entry point is unchanged. +- HTTP response models moved from `refget.models` (SQLModel) to + `refget.response_models` (plain pydantic), so serving the API needs no ORM. + +### Removed + +- **`refget.router.compliance_router`** — the module-level router singleton is + gone. It is replaced by `create_refget_router(compliance=True)` (the default), + which builds the compliance routes per router so they can self-target a + mounted prefix. No deprecation alias is provided on purpose: a module-level + singleton can only be built with an empty `mount_prefix`, and would therefore + silently reintroduce the root-targeting bug this replacement fixes. Nothing + in-tree used the name. +- `python -m seqcolapi` / `python -m refget.seqcolapi` — both `__main__.py` + modules imported a `main()` that has never existed, so both raised + `ImportError` on every invocation. Use the `refget` CLI, or uvicorn with one + of the app paths above. + +## [0.11.0] - 2026-02-28 + +This is a major release with significant restructuring, new features, and improved tooling. + +### Added + +- **CLI overhaul**: New `refget` CLI built with Typer, including subcommands for `store`, `seqcol`, `fasta`, `config`, and `admin` +- **Local store**: `refget store pull` command to pull sequence collections from remote servers to a local store +- **FASTA digesting**: `refget fasta digest` CLI command for computing sequence collection digests from FASTA files +- **Sequence collection similarities**: `calc_similarities` and `calc_similarities_from_json` functions with Jaccard similarity metrics and API endpoint +- **FASTA DRS objects**: `FastaDrsObject` model for serving FASTA files via DRS endpoints +- **Comparison interpreter**: Local sequence collection comparison interpretation module (SCIM) +- **Species filtering**: Filter similarities endpoint by species +- **Human-readable names**: `human_readable_name` field on `SequenceCollection` model +- **Pydantic API models**: Structured response models for API endpoints (fixes #33) +- **Swagger documentation**: API query parameter documentation +- **Frontend features**: Strip plots, one-to-many comparison view, FASTA digest tool, species selector, SCIM integration, dynamic version display +- **Compliance testing**: Comprehensive API compliance test suite +- **Integration test framework**: New integration test infrastructure with ephemeral databases +- **CLI test suite**: Extensive CLI tests covering store, seqcol, fasta, config, admin, and help commands +- **Service info**: `/service-info` endpoints for fasta_drs and refget_store features +- **Attribute listing**: `/list/attributes` endpoint per GA4GH paging guide +- **Bulk query**: Preload and bulk query support for sequence collections +- **R package**: First pass at `refget-r` R bindings (experimental) + +### Changed + +- **Switched to gtars**: Replaced pyfaidx and henge with gtars for FASTA parsing and digest computation +- **Major code restructure**: Consolidated schemas, reorganized modules, reduced code duplication +- **Improved error messages**: Better dependency error messages (fixes #49), clearer import errors +- **Performance optimizations**: Faster level 2 retrieval using `get_many`, optimized similarity calculations +- **Updated GA4GH compliance**: Aligned with latest refget sequence collections specification +- **Schema consolidation**: Single unified schema replacing multiple schema files +- **Collated attribute validation**: Validation for collated attributes in sequence collections +- **Frontend overhaul**: Updated comparison view, heatmap aliases, loading states, error handling + +### Removed + +- **Henge dependency**: Removed henge and biopython requirements +- **Legacy code**: Removed old flags code, duplicate functions, unused yacman imports + +### Fixed + +- `from_PySequenceCollection` construction and associated tests +- Circular dependency import issues in utilities +- Level 1 model representation +- Comparison links +- Cancel handling in frontend +- Various linting and type hint improvements + +### Security + +- Bumped frontend dependencies: vite, minimatch, rollup, esbuild, js-yaml, vega + +## [0.10.1] - 2025-06-01 + +Previous release. See git history for details. From 603c577ac2b1bd4cce77f7b1a936ca8dbe750c77 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 20:32:03 -0400 Subject: [PATCH 19/21] Replace the removed add-fasta command in docs --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f5a89a8..19aa09f 100644 --- a/README.md +++ b/README.md @@ -175,10 +175,10 @@ If you need to load test data into your server, then you have to install [gtars] PYTHONPATH=. python data_loaders/load_demo_seqcols.py ``` -or: +or, with the CLI (`refget add-fasta` was replaced by `refget admin load`): ``` -refget add-fasta -p test_fasta/test_fasta_metadata.csv -r test_fasta +refget admin load --pep test_fasta/test_fasta_metadata.csv --fa-root test_fasta ``` #### Running the seqcolapi API backend @@ -303,6 +303,9 @@ schema there unless it genuinely maps to a database table. ## Example of loading reference fasta datasets: +Needs `pip install 'refget[seqcolapi-db]'` and a configured PostgreSQL +connection (see [DB-backed (PostgreSQL)](#db-backed-postgresql)): + ``` -refget add-fasta -p ref_fasta.csv -r $BRICKYARD/datasets_downloaded/pangenome_fasta/reference_fasta +refget admin load --pep ref_fasta.csv --fa-root $BRICKYARD/datasets_downloaded/pangenome_fasta/reference_fasta ``` From 915e23e6a880df24ef7bcbeeba7940459854ece8 Mon Sep 17 00:00:00 2001 From: nsheff Date: Sat, 25 Jul 2026 21:02:35 -0400 Subject: [PATCH 20/21] Keep the changelog in the docs repo --- changelog.md | 121 --------------------------------------------------- mkdocs.yml | 4 +- 2 files changed, 3 insertions(+), 122 deletions(-) delete mode 100644 changelog.md diff --git a/changelog.md b/changelog.md deleted file mode 100644 index fe72b8e..0000000 --- a/changelog.md +++ /dev/null @@ -1,121 +0,0 @@ -# Changelog - -All notable changes to the refget package will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -### Added - -- **`refget.seqcolapi` ships in the wheel.** The service code moved from the - top-level `seqcolapi/` directory into `refget/seqcolapi/`, so a - `pip install 'refget[seqcolapi]'` can serve the API with no repository - checkout. Run it as `uvicorn refget.seqcolapi.main:store_app` (store-backed) - or `uvicorn refget.seqcolapi.main:app` (PostgreSQL-backed). -- **`refget.seqcolapi.create_seqcol_app`**: a store-backed app factory that - returns a self-contained, mountable FastAPI app, so a host application can - mount the seqcol API under a prefix and keep its own `/service-info`. -- **`create_refget_router(mount_prefix=...)`**: lets the compliance endpoints - self-target the seqcol service rather than the server root when the router is - included under a prefix. -- **`db` extra**: `pip install 'refget[db]'` installs just the SQLModel layer - (`refget.models`, `refget.agents`, `refget admin`) with no web server. -- Importing a module without its extra now raises an error naming the extra to - install, rather than a bare `ModuleNotFoundError`. - -### Changed - -- **Distribution: `sqlmodel` is no longer a base dependency.** It moved out of - the base install into the new `db` extra, and out of the `seqcolapi` extra, - which now installs fastapi and uvicorn only. **If you import - `refget.models`, `refget.agents`, or run `refget admin`, install - `refget[db]`.** The PostgreSQL-backed service now needs - `refget[seqcolapi-db]` (equivalent to `refget[seqcolapi,db]`); the - store-backed service needs only `refget[seqcolapi]` and pulls no ORM. -- **`ubiquerg` removed from the `seqcolapi` extra.** Nothing under `refget/` - imports it. -- `refget store serve` now builds its app with - `refget.seqcolapi.create_seqcol_app` instead of hand-wiring FastAPI, so it - serves the same routes the deployments do — notably `/service-info`, which it - previously omitted — plus a permissive CORS middleware. -- `refget.seqcolapi.main.create_store_app` was renamed to - `create_seqcolapi_store_app`, so that it is not confused with the generic - `refget.seqcolapi.create_seqcol_app`: it carries the seqcolapi.databio.org - service identity (`org.databio.seqcolapi.store`) and the SCOM service-info - block. The `uvicorn refget.seqcolapi.main:store_app` entry point is unchanged. -- HTTP response models moved from `refget.models` (SQLModel) to - `refget.response_models` (plain pydantic), so serving the API needs no ORM. - -### Removed - -- **`refget.router.compliance_router`** — the module-level router singleton is - gone. It is replaced by `create_refget_router(compliance=True)` (the default), - which builds the compliance routes per router so they can self-target a - mounted prefix. No deprecation alias is provided on purpose: a module-level - singleton can only be built with an empty `mount_prefix`, and would therefore - silently reintroduce the root-targeting bug this replacement fixes. Nothing - in-tree used the name. -- `python -m seqcolapi` / `python -m refget.seqcolapi` — both `__main__.py` - modules imported a `main()` that has never existed, so both raised - `ImportError` on every invocation. Use the `refget` CLI, or uvicorn with one - of the app paths above. - -## [0.11.0] - 2026-02-28 - -This is a major release with significant restructuring, new features, and improved tooling. - -### Added - -- **CLI overhaul**: New `refget` CLI built with Typer, including subcommands for `store`, `seqcol`, `fasta`, `config`, and `admin` -- **Local store**: `refget store pull` command to pull sequence collections from remote servers to a local store -- **FASTA digesting**: `refget fasta digest` CLI command for computing sequence collection digests from FASTA files -- **Sequence collection similarities**: `calc_similarities` and `calc_similarities_from_json` functions with Jaccard similarity metrics and API endpoint -- **FASTA DRS objects**: `FastaDrsObject` model for serving FASTA files via DRS endpoints -- **Comparison interpreter**: Local sequence collection comparison interpretation module (SCIM) -- **Species filtering**: Filter similarities endpoint by species -- **Human-readable names**: `human_readable_name` field on `SequenceCollection` model -- **Pydantic API models**: Structured response models for API endpoints (fixes #33) -- **Swagger documentation**: API query parameter documentation -- **Frontend features**: Strip plots, one-to-many comparison view, FASTA digest tool, species selector, SCIM integration, dynamic version display -- **Compliance testing**: Comprehensive API compliance test suite -- **Integration test framework**: New integration test infrastructure with ephemeral databases -- **CLI test suite**: Extensive CLI tests covering store, seqcol, fasta, config, admin, and help commands -- **Service info**: `/service-info` endpoints for fasta_drs and refget_store features -- **Attribute listing**: `/list/attributes` endpoint per GA4GH paging guide -- **Bulk query**: Preload and bulk query support for sequence collections -- **R package**: First pass at `refget-r` R bindings (experimental) - -### Changed - -- **Switched to gtars**: Replaced pyfaidx and henge with gtars for FASTA parsing and digest computation -- **Major code restructure**: Consolidated schemas, reorganized modules, reduced code duplication -- **Improved error messages**: Better dependency error messages (fixes #49), clearer import errors -- **Performance optimizations**: Faster level 2 retrieval using `get_many`, optimized similarity calculations -- **Updated GA4GH compliance**: Aligned with latest refget sequence collections specification -- **Schema consolidation**: Single unified schema replacing multiple schema files -- **Collated attribute validation**: Validation for collated attributes in sequence collections -- **Frontend overhaul**: Updated comparison view, heatmap aliases, loading states, error handling - -### Removed - -- **Henge dependency**: Removed henge and biopython requirements -- **Legacy code**: Removed old flags code, duplicate functions, unused yacman imports - -### Fixed - -- `from_PySequenceCollection` construction and associated tests -- Circular dependency import issues in utilities -- Level 1 model representation -- Comparison links -- Cancel handling in frontend -- Various linting and type hint improvements - -### Security - -- Bumped frontend dependencies: vite, minimatch, rollup, esbuild, js-yaml, vega - -## [0.10.1] - 2025-06-01 - -Previous release. See git history for details. diff --git a/mkdocs.yml b/mkdocs.yml index bc1f4b8..34a9dcc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -10,7 +10,9 @@ nav: - "Tutorial": tutorial.md - Reference: - Support: https://github.com/refgenie/refget/issues - - Changelog: changelog.md + # The changelog lives in the refgenie-docs repo, at + # src/content/docs/refget/reference/changelog.md + - Changelog: https://refgenie.org/refget/reference/changelog/ theme: databio From a0bdeac09c365d70a2781766c76ce130f5436e69 Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 28 Jul 2026 14:47:07 -0400 Subject: [PATCH 21/21] store: adapt the import CLI to ImportReport and the renamed stats keys add_sequence_collections_from_fastas returns an ImportReport rather than a list of (metadata, was_new) pairs, and PyImportReport defines neither __iter__ nor __getitem__, so iterating it and calling len() on it both raised TypeError. Read .collections for the per-file results, and surface the report's per-run ingest counters in the JSON output. Also update the stats docstring, which still advertised n_collections_loaded: that key is now n_collections_in_memory, and the docstring now says what those gauges actually measure -- RAM residency at the moment stats() is called, typically 0 for a disk-backed store -- so they stop being read as ingest counts. --- refget/cli/store.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/refget/cli/store.py b/refget/cli/store.py index 23fe8db..640ebaf 100644 --- a/refget/cli/store.py +++ b/refget/cli/store.py @@ -411,9 +411,13 @@ def add( { "results": [ {"digest": m.digest, "sequences": m.n_sequences, "was_new": new} - for (m, new) in results + for (m, new) in results.collections ], - "count": len(results), + "count": len(results.collections), + # Per-run ingest counters carried by the ImportReport. + "n_collections_new": results.n_collections_new, + "n_sequences_written": results.n_sequences_written, + "n_sequences_deduped": results.n_sequences_deduped, } ) raise typer.Exit(EXIT_SUCCESS) @@ -1090,11 +1094,17 @@ def stats( Display store statistics. Outputs the store's stats dict: n_sequences, n_collections, - n_collections_loaded, and storage_mode. + n_collections_in_memory, n_sequences_in_memory, logical_sequence_bytes, + and storage_mode. + + The `*_in_memory` fields are RAM-residency gauges, not ingest counts -- + they report how much is loaded right now, so they are typically 0 for a + disk-backed store. For what a build actually wrote, see the ImportReport + returned by an import. Example output: - {"n_sequences": 75, "n_collections": 3, "n_collections_loaded": 0, - "storage_mode": "Encoded"} + {"n_sequences": 75, "n_collections": 3, "n_collections_in_memory": 0, + "n_sequences_in_memory": 0, "storage_mode": "Encoded"} """ store = _load_store(path, remote=remote)