diff --git a/builders/server/benchmarks/bench_build.py b/builders/server/benchmarks/bench_build.py index d6c4202..ce14b74 100644 --- a/builders/server/benchmarks/bench_build.py +++ b/builders/server/benchmarks/bench_build.py @@ -93,16 +93,16 @@ def _run_standalone(days: int) -> None: cur.execute(create_index) # patch modules to use test db - import db.connection - import db.datasets - from runtime import config, loader + import core.db.connection + import core.db.datasets + from core.runtime import config, loader @contextmanager def _test_conn(): yield conn - db.connection.get_conn = _test_conn # type: ignore[assignment] # benchmark patching - db.datasets.get_conn = _test_conn # type: ignore[assignment] # benchmark patching + core.db.connection.get_conn = _test_conn # type: ignore[assignment] # benchmark patching + core.db.datasets.get_conn = _test_conn # type: ignore[assignment] # benchmark patching config.SCRIPTS_DIR = scripts_dir loader.SCRIPTS_DIR = scripts_dir diff --git a/builders/server/benchmarks/conftest.py b/builders/server/benchmarks/conftest.py index d3db07c..dbeb3d3 100644 --- a/builders/server/benchmarks/conftest.py +++ b/builders/server/benchmarks/conftest.py @@ -66,21 +66,21 @@ def clean_db(db_conn): @pytest.fixture(autouse=True) def patch_db_conn(db_conn, monkeypatch): """Redirect all production DB calls to the test database.""" - import db.connection - import db.datasets + import core.db.connection + import core.db.datasets @contextmanager def _test_conn(): yield db_conn - monkeypatch.setattr(db.connection, "get_conn", _test_conn) - monkeypatch.setattr(db.datasets, "get_conn", _test_conn) + monkeypatch.setattr(core.db.connection, "get_conn", _test_conn) + monkeypatch.setattr(core.db.datasets, "get_conn", _test_conn) @pytest.fixture(autouse=True) def real_scripts_dir(monkeypatch): """Point SCRIPTS_DIR to the real builders/scripts/ directory.""" - from runtime import config, loader + from core.runtime import config, loader monkeypatch.setattr(config, "SCRIPTS_DIR", REAL_SCRIPTS_DIR) monkeypatch.setattr(loader, "SCRIPTS_DIR", REAL_SCRIPTS_DIR) diff --git a/builders/server/calendars/registry.py b/builders/server/calendars/registry.py deleted file mode 100644 index 49d8e10..0000000 --- a/builders/server/calendars/registry.py +++ /dev/null @@ -1,13 +0,0 @@ -from calendars.definitions.always_open import AlwaysOpenCalendar -from calendars.definitions.everyday import EverydayCalendar -from calendars.definitions.nyse_daily import NyseDailyCalendar -from calendars.definitions.weekday import WeekdayCalendar -from calendars.interface import Calendar - -# registry mapping calendar name -> Calendar instance -CALENDARS_MAP: dict[str, Calendar] = { - "everyday": EverydayCalendar(), - "weekday": WeekdayCalendar(), - "always-open": AlwaysOpenCalendar(), - "nyse-daily": NyseDailyCalendar(), -} diff --git a/builders/server/api/__init__.py b/builders/server/core/api/__init__.py similarity index 100% rename from builders/server/api/__init__.py rename to builders/server/core/api/__init__.py diff --git a/builders/server/api/routes.py b/builders/server/core/api/routes.py similarity index 95% rename from builders/server/api/routes.py rename to builders/server/core/api/routes.py index 239b5e2..ad6a4f6 100644 --- a/builders/server/api/routes.py +++ b/builders/server/core/api/routes.py @@ -3,9 +3,10 @@ import structlog from fastapi import APIRouter, HTTPException, Query from fastapi.responses import JSONResponse -from service.builder import NoValidTimestampsError, build_dataset, get_data -from service.catalog import list_datasets -from utils.semver import SemVer + +from core.service.builder import NoValidTimestampsError, build_dataset, get_data +from core.service.catalog import list_datasets +from core.utils.semver import SemVer logger = structlog.get_logger() diff --git a/builders/server/calendars/__init__.py b/builders/server/core/calendars/__init__.py similarity index 100% rename from builders/server/calendars/__init__.py rename to builders/server/core/calendars/__init__.py diff --git a/builders/server/calendars/definitions/__init__.py b/builders/server/core/calendars/definitions/__init__.py similarity index 100% rename from builders/server/calendars/definitions/__init__.py rename to builders/server/core/calendars/definitions/__init__.py diff --git a/builders/server/calendars/definitions/always_open.py b/builders/server/core/calendars/definitions/always_open.py similarity index 92% rename from builders/server/calendars/definitions/always_open.py rename to builders/server/core/calendars/definitions/always_open.py index f2884f7..803e216 100644 --- a/builders/server/calendars/definitions/always_open.py +++ b/builders/server/core/calendars/definitions/always_open.py @@ -1,6 +1,6 @@ from datetime import datetime, timedelta -from calendars.interface import Calendar +from core.calendars.interface import Calendar class AlwaysOpenCalendar(Calendar): diff --git a/builders/server/calendars/definitions/everyday.py b/builders/server/core/calendars/definitions/everyday.py similarity index 88% rename from builders/server/calendars/definitions/everyday.py rename to builders/server/core/calendars/definitions/everyday.py index 538eb04..d6c4260 100644 --- a/builders/server/calendars/definitions/everyday.py +++ b/builders/server/core/calendars/definitions/everyday.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta -from calendars.interface import Calendar -from calendars.utils import is_midnight +from core.calendars.interface import Calendar +from core.calendars.utils import is_midnight class EverydayCalendar(Calendar): diff --git a/builders/server/calendars/definitions/nyse_daily.py b/builders/server/core/calendars/definitions/nyse_daily.py similarity index 94% rename from builders/server/calendars/definitions/nyse_daily.py rename to builders/server/core/calendars/definitions/nyse_daily.py index b403d6b..13e5326 100644 --- a/builders/server/calendars/definitions/nyse_daily.py +++ b/builders/server/core/calendars/definitions/nyse_daily.py @@ -3,8 +3,8 @@ import exchange_calendars as xcals import pandas as pd -from calendars.interface import Calendar -from calendars.utils import is_midnight +from core.calendars.interface import Calendar +from core.calendars.utils import is_midnight class NyseDailyCalendar(Calendar): diff --git a/builders/server/calendars/definitions/weekday.py b/builders/server/core/calendars/definitions/weekday.py similarity index 90% rename from builders/server/calendars/definitions/weekday.py rename to builders/server/core/calendars/definitions/weekday.py index b541fbb..c139776 100644 --- a/builders/server/calendars/definitions/weekday.py +++ b/builders/server/core/calendars/definitions/weekday.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta -from calendars.interface import Calendar -from calendars.utils import is_midnight +from core.calendars.interface import Calendar +from core.calendars.utils import is_midnight class WeekdayCalendar(Calendar): diff --git a/builders/server/calendars/interface.py b/builders/server/core/calendars/interface.py similarity index 100% rename from builders/server/calendars/interface.py rename to builders/server/core/calendars/interface.py diff --git a/builders/server/core/calendars/registry.py b/builders/server/core/calendars/registry.py new file mode 100644 index 0000000..48d8f4c --- /dev/null +++ b/builders/server/core/calendars/registry.py @@ -0,0 +1,13 @@ +from core.calendars.definitions.always_open import AlwaysOpenCalendar +from core.calendars.definitions.everyday import EverydayCalendar +from core.calendars.definitions.nyse_daily import NyseDailyCalendar +from core.calendars.definitions.weekday import WeekdayCalendar +from core.calendars.interface import Calendar + +# registry mapping calendar name -> Calendar instance +CALENDARS_MAP: dict[str, Calendar] = { + "everyday": EverydayCalendar(), + "weekday": WeekdayCalendar(), + "always-open": AlwaysOpenCalendar(), + "nyse-daily": NyseDailyCalendar(), +} diff --git a/builders/server/calendars/utils.py b/builders/server/core/calendars/utils.py similarity index 100% rename from builders/server/calendars/utils.py rename to builders/server/core/calendars/utils.py diff --git a/builders/server/db/__init__.py b/builders/server/core/db/__init__.py similarity index 100% rename from builders/server/db/__init__.py rename to builders/server/core/db/__init__.py diff --git a/builders/server/db/connection.py b/builders/server/core/db/connection.py similarity index 100% rename from builders/server/db/connection.py rename to builders/server/core/db/connection.py diff --git a/builders/server/db/datasets.py b/builders/server/core/db/datasets.py similarity index 98% rename from builders/server/db/datasets.py rename to builders/server/core/db/datasets.py index 03fde90..16c2592 100644 --- a/builders/server/db/datasets.py +++ b/builders/server/core/db/datasets.py @@ -4,9 +4,9 @@ import structlog from psycopg.rows import dict_row from psycopg.types.json import Jsonb -from utils.semver import SemVer -from db.connection import get_conn +from core.db.connection import get_conn +from core.utils.semver import SemVer logger = structlog.get_logger() diff --git a/builders/server/db/migrations/env.py b/builders/server/core/db/migrations/env.py similarity index 100% rename from builders/server/db/migrations/env.py rename to builders/server/core/db/migrations/env.py diff --git a/builders/server/db/migrations/script.py.mako b/builders/server/core/db/migrations/script.py.mako similarity index 100% rename from builders/server/db/migrations/script.py.mako rename to builders/server/core/db/migrations/script.py.mako diff --git a/builders/server/db/migrations/versions/.gitkeep b/builders/server/core/db/migrations/versions/.gitkeep similarity index 100% rename from builders/server/db/migrations/versions/.gitkeep rename to builders/server/core/db/migrations/versions/.gitkeep diff --git a/builders/server/db/migrations/versions/001_initial_schema.py b/builders/server/core/db/migrations/versions/001_initial_schema.py similarity index 100% rename from builders/server/db/migrations/versions/001_initial_schema.py rename to builders/server/core/db/migrations/versions/001_initial_schema.py diff --git a/builders/server/runtime/__init__.py b/builders/server/core/runtime/__init__.py similarity index 100% rename from builders/server/runtime/__init__.py rename to builders/server/core/runtime/__init__.py diff --git a/builders/server/runtime/config.py b/builders/server/core/runtime/config.py similarity index 98% rename from builders/server/runtime/config.py rename to builders/server/core/runtime/config.py index eb5b711..60a5178 100644 --- a/builders/server/runtime/config.py +++ b/builders/server/core/runtime/config.py @@ -6,9 +6,10 @@ from pathlib import Path import structlog -from calendars.interface import Calendar -from calendars.registry import CALENDARS_MAP -from utils.semver import SemVer + +from core.calendars.interface import Calendar +from core.calendars.registry import CALENDARS_MAP +from core.utils.semver import SemVer logger = structlog.get_logger() diff --git a/builders/server/runtime/loader.py b/builders/server/core/runtime/loader.py similarity index 97% rename from builders/server/runtime/loader.py rename to builders/server/core/runtime/loader.py index 4c19833..f3c7faa 100644 --- a/builders/server/runtime/loader.py +++ b/builders/server/core/runtime/loader.py @@ -4,7 +4,8 @@ from pathlib import Path import structlog -from utils.semver import SemVer + +from core.utils.semver import SemVer logger = structlog.get_logger() diff --git a/builders/server/runtime/registry.py b/builders/server/core/runtime/registry.py similarity index 98% rename from builders/server/runtime/registry.py rename to builders/server/core/runtime/registry.py index b852862..08b32a4 100644 --- a/builders/server/runtime/registry.py +++ b/builders/server/core/runtime/registry.py @@ -3,13 +3,13 @@ from pathlib import Path import structlog -from utils.semver import SemVer -from runtime.config import ( +from core.runtime.config import ( DatasetConfig, normalize_config, validate_config, ) +from core.utils.semver import SemVer logger = structlog.get_logger() diff --git a/builders/server/runtime/runner.py b/builders/server/core/runtime/runner.py similarity index 95% rename from builders/server/runtime/runner.py rename to builders/server/core/runtime/runner.py index 14d4a50..c610496 100644 --- a/builders/server/runtime/runner.py +++ b/builders/server/core/runtime/runner.py @@ -5,14 +5,14 @@ from pathlib import Path import structlog -from utils.retry import retry_with_backoff -from runtime.serialization import ( +from core.runtime.serialization import ( WorkerError, WorkerSuccess, deserialize_output, serialize_input, ) +from core.utils.retry import retry_with_backoff logger = structlog.get_logger() @@ -21,7 +21,7 @@ RETRY_INITIAL_DELAY = 2.0 # seconds RETRY_BACKOFF_FACTOR = 2.0 -WORKER_PATH = Path(__file__).parent.parent / "workers" / "subprocess_worker.py" +WORKER_PATH = Path(__file__).parent.parent.parent / "workers" / "subprocess_worker.py" def run_builder( diff --git a/builders/server/runtime/serialization.py b/builders/server/core/runtime/serialization.py similarity index 100% rename from builders/server/runtime/serialization.py rename to builders/server/core/runtime/serialization.py diff --git a/builders/server/runtime/validator.py b/builders/server/core/runtime/validator.py similarity index 95% rename from builders/server/runtime/validator.py rename to builders/server/core/runtime/validator.py index 16404fd..b59ce5d 100644 --- a/builders/server/runtime/validator.py +++ b/builders/server/core/runtime/validator.py @@ -1,4 +1,4 @@ -from runtime.config import SchemaType +from core.runtime.config import SchemaType class ValidationError(Exception): diff --git a/builders/server/runtime/venv_management.py b/builders/server/core/runtime/venv_management.py similarity index 100% rename from builders/server/runtime/venv_management.py rename to builders/server/core/runtime/venv_management.py diff --git a/builders/server/service/__init__.py b/builders/server/core/service/__init__.py similarity index 100% rename from builders/server/service/__init__.py rename to builders/server/core/service/__init__.py diff --git a/builders/server/service/builder.py b/builders/server/core/service/builder.py similarity index 85% rename from builders/server/service/builder.py rename to builders/server/core/service/builder.py index a7cbcab..4ab4a1a 100644 --- a/builders/server/service/builder.py +++ b/builders/server/core/service/builder.py @@ -1,13 +1,13 @@ from dataclasses import dataclass from datetime import datetime -import db.datasets import structlog -from runtime import registry -from utils.semver import SemVer -from service.orchestrator import run_build -from service.timestamps import NoValidTimestampsError, generate_timestamps +import core.db.datasets +from core.runtime import registry +from core.service.orchestrator import run_build +from core.service.timestamps import NoValidTimestampsError, generate_timestamps +from core.utils.semver import SemVer logger = structlog.get_logger() @@ -62,7 +62,7 @@ def get_data( build_dataset(dataset_name, dataset_version, start, end) total_num_rows = len(generate_timestamps(start, end, cfg.granularity, cfg.calendar)) - data = db.datasets.get_rows_range(dataset_name, dataset_version, start, end) + data = core.db.datasets.get_rows_range(dataset_name, dataset_version, start, end) logger.info( "data fetched", diff --git a/builders/server/service/catalog.py b/builders/server/core/service/catalog.py similarity index 82% rename from builders/server/service/catalog.py rename to builders/server/core/service/catalog.py index 2f0b3bf..9e97a1d 100644 --- a/builders/server/service/catalog.py +++ b/builders/server/core/service/catalog.py @@ -1,7 +1,7 @@ from dataclasses import dataclass -import db.datasets -from runtime.registry import iter_config_keys +import core.db.datasets +from core.runtime.registry import iter_config_keys @dataclass @@ -15,7 +15,7 @@ class DatasetInfo: def list_datasets() -> list[DatasetInfo]: """Return all pre-loaded datasets annotated with whether they have DB rows.""" - has_data = db.datasets.get_datasets_with_data() + has_data = core.db.datasets.get_datasets_with_data() return sorted( [ DatasetInfo( diff --git a/builders/server/service/locks.py b/builders/server/core/service/locks.py similarity index 100% rename from builders/server/service/locks.py rename to builders/server/core/service/locks.py diff --git a/builders/server/service/models.py b/builders/server/core/service/models.py similarity index 96% rename from builders/server/service/models.py rename to builders/server/core/service/models.py index c180db9..1f84e2b 100644 --- a/builders/server/service/models.py +++ b/builders/server/core/service/models.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from datetime import datetime -from utils.semver import SemVer +from core.utils.semver import SemVer @dataclass(frozen=True) diff --git a/builders/server/service/orchestrator.py b/builders/server/core/service/orchestrator.py similarity index 95% rename from builders/server/service/orchestrator.py rename to builders/server/core/service/orchestrator.py index c9fb64c..57e8b7d 100644 --- a/builders/server/service/orchestrator.py +++ b/builders/server/core/service/orchestrator.py @@ -19,10 +19,10 @@ from datetime import datetime import structlog -from utils.semver import SemVer -from service.scheduler import schedule_build -from service.worker import execute_job +from core.service.scheduler import schedule_build +from core.service.worker import execute_job +from core.utils.semver import SemVer logger = structlog.get_logger() diff --git a/builders/server/service/scheduler.py b/builders/server/core/service/scheduler.py similarity index 98% rename from builders/server/service/scheduler.py rename to builders/server/core/service/scheduler.py index f3dfb5b..26aeecd 100644 --- a/builders/server/service/scheduler.py +++ b/builders/server/core/service/scheduler.py @@ -26,10 +26,10 @@ from datetime import datetime import structlog -from runtime import registry -from utils.semver import SemVer -from service.models import BuildPlan, JobDescriptor +from core.runtime import registry +from core.service.models import BuildPlan, JobDescriptor +from core.utils.semver import SemVer logger = structlog.get_logger() diff --git a/builders/server/service/timestamps.py b/builders/server/core/service/timestamps.py similarity index 95% rename from builders/server/service/timestamps.py rename to builders/server/core/service/timestamps.py index 4fdc5d9..cdf3238 100644 --- a/builders/server/service/timestamps.py +++ b/builders/server/core/service/timestamps.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta -from calendars.interface import Calendar +from core.calendars.interface import Calendar class NoValidTimestampsError(Exception): diff --git a/builders/server/service/worker.py b/builders/server/core/service/worker.py similarity index 91% rename from builders/server/service/worker.py rename to builders/server/core/service/worker.py index 4835c73..556f694 100644 --- a/builders/server/service/worker.py +++ b/builders/server/core/service/worker.py @@ -19,13 +19,13 @@ from datetime import datetime from pathlib import Path -import db.datasets import structlog -from runtime import config, registry, runner, validator -from service.locks import get_build_lock -from service.models import JobDescriptor, JobResult -from service.timestamps import NoValidTimestampsError, generate_timestamps +import core.db.datasets +from core.runtime import config, registry, runner, validator +from core.service.locks import get_build_lock +from core.service.models import JobDescriptor, JobResult +from core.service.timestamps import NoValidTimestampsError, generate_timestamps logger = structlog.get_logger() @@ -90,7 +90,7 @@ def _execute( # between the "check missing" read and "insert rows" write with get_build_lock(job.dataset_name, str(job.dataset_version)): existing = set( - db.datasets.get_existing_timestamps( + core.db.datasets.get_existing_timestamps( job.dataset_name, job.dataset_version, job.start, job.end ) ) @@ -143,7 +143,7 @@ def _execute( rows.append((ts, result)) # bulk insert -- only reached if all timestamps succeeded - db.datasets.insert_rows(job.dataset_name, job.dataset_version, rows) + core.db.datasets.insert_rows(job.dataset_name, job.dataset_version, rows) logger.info( "inserted rows", dataset=job.dataset_name, @@ -166,14 +166,16 @@ def _fetch_dep_data( for dep_name, dep_info in cfg.dependencies.items(): if dep_info.lookback_subtract is not None: - dep_rows = db.datasets.get_rows_range( + dep_rows = core.db.datasets.get_rows_range( dep_name, dep_info.version, ts - dep_info.lookback_subtract, ts, ) else: - dep_rows = db.datasets.get_rows_timestamps(dep_name, dep_info.version, [ts]) + dep_rows = core.db.datasets.get_rows_timestamps( + dep_name, dep_info.version, [ts] + ) if not dep_rows: raise RuntimeError( diff --git a/builders/server/tests/api/__init__.py b/builders/server/core/utils/__init__.py similarity index 100% rename from builders/server/tests/api/__init__.py rename to builders/server/core/utils/__init__.py diff --git a/builders/server/utils/retry.py b/builders/server/core/utils/retry.py similarity index 100% rename from builders/server/utils/retry.py rename to builders/server/core/utils/retry.py diff --git a/builders/server/utils/semver.py b/builders/server/core/utils/semver.py similarity index 100% rename from builders/server/utils/semver.py rename to builders/server/core/utils/semver.py diff --git a/builders/server/main.py b/builders/server/main.py index 570786c..42105a7 100644 --- a/builders/server/main.py +++ b/builders/server/main.py @@ -4,13 +4,13 @@ from contextlib import asynccontextmanager import structlog -from api.routes import router -from db.connection import close_pool, open_pool +from core.api.routes import router +from core.db.connection import close_pool, open_pool +from core.runtime.config import SCRIPTS_DIR +from core.runtime.registry import load_all_configs +from core.runtime.venv_management import setup_builder_venvs from fastapi import FastAPI, Request, Response from log_config import setup_logging as _setup_logging -from runtime.config import SCRIPTS_DIR -from runtime.registry import load_all_configs -from runtime.venv_management import setup_builder_venvs _setup_logging() diff --git a/builders/server/tests/conftest.py b/builders/server/tests/conftest.py index 50e5bd5..e69de29 100644 --- a/builders/server/tests/conftest.py +++ b/builders/server/tests/conftest.py @@ -1,46 +0,0 @@ -from collections.abc import Callable -from pathlib import Path - -import pytest - - -@pytest.fixture -def mock_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Create a temp scripts dir and monkeypatch SCRIPTS_DIR.""" - scripts = tmp_path / "scripts" - scripts.mkdir() - from runtime import config, loader - - monkeypatch.setattr(config, "SCRIPTS_DIR", scripts) - monkeypatch.setattr(loader, "SCRIPTS_DIR", scripts) - return scripts - - -@pytest.fixture -def write_config(tmp_path: Path) -> Callable[[Path, str, str, str], Path]: - """Factory fixture to write config.toml files under the scripts dir.""" - - def _write( - scripts_dir: Path, dataset_name: str, dataset_version: str, content: str - ) -> Path: - d = scripts_dir / dataset_name / dataset_version - d.mkdir(parents=True, exist_ok=True) - (d / "config.toml").write_text(content) - return d - - return _write - - -@pytest.fixture -def write_builder(tmp_path: Path) -> Callable[[Path, str, str, str], Path]: - """Factory fixture to write builder.py files under the scripts dir.""" - - def _write( - scripts_dir: Path, dataset_name: str, dataset_version: str, content: str - ) -> Path: - d = scripts_dir / dataset_name / dataset_version - d.mkdir(parents=True, exist_ok=True) - (d / "builder.py").write_text(content) - return d - - return _write diff --git a/builders/server/tests/calendars/__init__.py b/builders/server/tests/core/__init__.py similarity index 100% rename from builders/server/tests/calendars/__init__.py rename to builders/server/tests/core/__init__.py diff --git a/builders/server/tests/calendars/definitions/__init__.py b/builders/server/tests/core/api/__init__.py similarity index 100% rename from builders/server/tests/calendars/definitions/__init__.py rename to builders/server/tests/core/api/__init__.py diff --git a/builders/server/tests/api/test_routes.py b/builders/server/tests/core/api/test_routes.py similarity index 90% rename from builders/server/tests/api/test_routes.py rename to builders/server/tests/core/api/test_routes.py index 90d55e8..e72d7bc 100644 --- a/builders/server/tests/api/test_routes.py +++ b/builders/server/tests/core/api/test_routes.py @@ -1,15 +1,15 @@ from datetime import datetime from unittest.mock import MagicMock, patch +from core.service.builder import DataResult, NoValidTimestampsError +from core.service.catalog import DatasetInfo from fastapi.testclient import TestClient from main import app -from service.builder import DataResult, NoValidTimestampsError -from service.catalog import DatasetInfo client: TestClient = TestClient(app) -@patch("api.routes.build_dataset") +@patch("core.api.routes.build_dataset") def test_build_endpoint_success(mock_build: MagicMock) -> None: """POST returns 200 with status ok.""" mock_build.return_value = None @@ -30,7 +30,10 @@ def test_build_endpoint_invalid_timestamp() -> None: assert resp.status_code == 400 -@patch("api.routes.build_dataset", side_effect=FileNotFoundError("config not found")) +@patch( + "core.api.routes.build_dataset", + side_effect=FileNotFoundError("config not found"), +) def test_build_endpoint_internal_error(mock_build: MagicMock) -> None: """Config not found returns 500.""" resp = client.post("/api/v1/build/ds/0.1.0?start=2024-01-01&end=2024-01-31") @@ -38,7 +41,7 @@ def test_build_endpoint_internal_error(mock_build: MagicMock) -> None: @patch( - "api.routes.build_dataset", + "core.api.routes.build_dataset", side_effect=NoValidTimestampsError("no valid timestamps in range"), ) def test_build_endpoint_no_valid_timestamps(mock_build: MagicMock) -> None: @@ -51,7 +54,7 @@ def test_build_endpoint_no_valid_timestamps(mock_build: MagicMock) -> None: # --- GET /data tests --- -@patch("api.routes.get_data") +@patch("core.api.routes.get_data") def test_data_endpoint_default_build_200(mock_get_data: MagicMock) -> None: """GET with default build-data=true returns 200 with metadata.""" ts = datetime(2024, 1, 2) @@ -77,7 +80,7 @@ def test_data_endpoint_default_build_200(mock_get_data: MagicMock) -> None: assert mock_get_data.call_args.kwargs["build_data"] is True -@patch("api.routes.get_data") +@patch("core.api.routes.get_data") def test_data_endpoint_no_build_complete_200(mock_get_data: MagicMock) -> None: """GET with build-data=false and complete data returns 200.""" ts = datetime(2024, 1, 2) @@ -96,7 +99,7 @@ def test_data_endpoint_no_build_complete_200(mock_get_data: MagicMock) -> None: assert resp.json()["returned_timestamps"] == 1 -@patch("api.routes.get_data") +@patch("core.api.routes.get_data") def test_data_endpoint_no_build_incomplete_206(mock_get_data: MagicMock) -> None: """GET with build-data=false and missing data returns 206.""" mock_get_data.return_value = DataResult( @@ -116,7 +119,7 @@ def test_data_endpoint_no_build_incomplete_206(mock_get_data: MagicMock) -> None assert body["rows"] == [] -@patch("api.routes.get_data") +@patch("core.api.routes.get_data") def test_data_endpoint_no_build_partial_206(mock_get_data: MagicMock) -> None: """GET with build-data=false and partial data returns 206.""" ts = datetime(2024, 1, 1) @@ -146,7 +149,7 @@ def test_data_endpoint_invalid_timestamp() -> None: assert resp.status_code == 400 -@patch("api.routes.get_data", side_effect=FileNotFoundError("config not found")) +@patch("core.api.routes.get_data", side_effect=FileNotFoundError("config not found")) def test_data_endpoint_internal_error(mock_get_data: MagicMock) -> None: """Config not found returns 500.""" resp = client.get("/api/v1/data/ds/0.1.0?start=2024-01-01&end=2024-01-31") @@ -154,7 +157,7 @@ def test_data_endpoint_internal_error(mock_get_data: MagicMock) -> None: @patch( - "api.routes.get_data", + "core.api.routes.get_data", side_effect=NoValidTimestampsError("no valid timestamps in range"), ) def test_data_endpoint_no_valid_timestamps_422(mock_get_data: MagicMock) -> None: @@ -167,7 +170,7 @@ def test_data_endpoint_no_valid_timestamps_422(mock_get_data: MagicMock) -> None # --- GET /datasets tests --- -@patch("api.routes.list_datasets") +@patch("core.api.routes.list_datasets") def test_datasets_endpoint_returns_list(mock_list: MagicMock) -> None: """Returns 200 with datasets array.""" mock_list.return_value = [ @@ -191,7 +194,7 @@ def test_datasets_endpoint_returns_list(mock_list: MagicMock) -> None: } -@patch("api.routes.list_datasets") +@patch("core.api.routes.list_datasets") def test_datasets_endpoint_empty(mock_list: MagicMock) -> None: """Returns 200 with empty list when no datasets discovered.""" mock_list.return_value = [] @@ -200,7 +203,7 @@ def test_datasets_endpoint_empty(mock_list: MagicMock) -> None: assert resp.json() == {"datasets": []} -@patch("api.routes.list_datasets", side_effect=OSError("disk error")) +@patch("core.api.routes.list_datasets", side_effect=OSError("disk error")) def test_datasets_endpoint_internal_error(mock_list: MagicMock) -> None: """Unexpected failure returns 500.""" resp = client.get("/api/v1/datasets") diff --git a/builders/server/tests/db/__init__.py b/builders/server/tests/core/calendars/__init__.py similarity index 100% rename from builders/server/tests/db/__init__.py rename to builders/server/tests/core/calendars/__init__.py diff --git a/builders/server/tests/runtime/__init__.py b/builders/server/tests/core/calendars/definitions/__init__.py similarity index 100% rename from builders/server/tests/runtime/__init__.py rename to builders/server/tests/core/calendars/definitions/__init__.py diff --git a/builders/server/tests/calendars/definitions/test_always_open.py b/builders/server/tests/core/calendars/definitions/test_always_open.py similarity index 92% rename from builders/server/tests/calendars/definitions/test_always_open.py rename to builders/server/tests/core/calendars/definitions/test_always_open.py index f4f5f67..b0702b7 100644 --- a/builders/server/tests/calendars/definitions/test_always_open.py +++ b/builders/server/tests/core/calendars/definitions/test_always_open.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta -from calendars.definitions.always_open import AlwaysOpenCalendar -from calendars.registry import CALENDARS_MAP +from core.calendars.definitions.always_open import AlwaysOpenCalendar +from core.calendars.registry import CALENDARS_MAP def test_always_open_is_open_always_true() -> None: diff --git a/builders/server/tests/calendars/definitions/test_everyday.py b/builders/server/tests/core/calendars/definitions/test_everyday.py similarity index 94% rename from builders/server/tests/calendars/definitions/test_everyday.py rename to builders/server/tests/core/calendars/definitions/test_everyday.py index ca55075..7b56bcd 100644 --- a/builders/server/tests/calendars/definitions/test_everyday.py +++ b/builders/server/tests/core/calendars/definitions/test_everyday.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta -from calendars.definitions.everyday import EverydayCalendar -from calendars.registry import CALENDARS_MAP +from core.calendars.definitions.everyday import EverydayCalendar +from core.calendars.registry import CALENDARS_MAP def test_everyday_is_open_always_true() -> None: diff --git a/builders/server/tests/calendars/definitions/test_nyse_daily.py b/builders/server/tests/core/calendars/definitions/test_nyse_daily.py similarity index 96% rename from builders/server/tests/calendars/definitions/test_nyse_daily.py rename to builders/server/tests/core/calendars/definitions/test_nyse_daily.py index 4026565..0491330 100644 --- a/builders/server/tests/calendars/definitions/test_nyse_daily.py +++ b/builders/server/tests/core/calendars/definitions/test_nyse_daily.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta -from calendars.definitions.nyse_daily import NyseDailyCalendar -from calendars.registry import CALENDARS_MAP +from core.calendars.definitions.nyse_daily import NyseDailyCalendar +from core.calendars.registry import CALENDARS_MAP def test_nyse_daily_name() -> None: diff --git a/builders/server/tests/calendars/definitions/test_weekday.py b/builders/server/tests/core/calendars/definitions/test_weekday.py similarity index 95% rename from builders/server/tests/calendars/definitions/test_weekday.py rename to builders/server/tests/core/calendars/definitions/test_weekday.py index f166399..7f2706d 100644 --- a/builders/server/tests/calendars/definitions/test_weekday.py +++ b/builders/server/tests/core/calendars/definitions/test_weekday.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta -from calendars.definitions.weekday import WeekdayCalendar -from calendars.registry import CALENDARS_MAP +from core.calendars.definitions.weekday import WeekdayCalendar +from core.calendars.registry import CALENDARS_MAP def test_weekday_open_on_weekdays() -> None: diff --git a/builders/server/tests/calendars/test_utils.py b/builders/server/tests/core/calendars/test_utils.py similarity index 94% rename from builders/server/tests/calendars/test_utils.py rename to builders/server/tests/core/calendars/test_utils.py index 1e969ad..2040710 100644 --- a/builders/server/tests/calendars/test_utils.py +++ b/builders/server/tests/core/calendars/test_utils.py @@ -1,6 +1,6 @@ from datetime import datetime -from calendars.utils import is_midnight +from core.calendars.utils import is_midnight def test_is_midnight_positive() -> None: diff --git a/builders/server/tests/core/conftest.py b/builders/server/tests/core/conftest.py new file mode 100644 index 0000000..3f8b61d --- /dev/null +++ b/builders/server/tests/core/conftest.py @@ -0,0 +1,46 @@ +from collections.abc import Callable +from pathlib import Path + +import pytest + + +@pytest.fixture +def mock_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Create a temp scripts dir and monkeypatch SCRIPTS_DIR.""" + scripts = tmp_path / "scripts" + scripts.mkdir() + from core.runtime import config, loader + + monkeypatch.setattr(config, "SCRIPTS_DIR", scripts) + monkeypatch.setattr(loader, "SCRIPTS_DIR", scripts) + return scripts + + +@pytest.fixture +def write_config(tmp_path: Path) -> Callable[[Path, str, str, str], Path]: + """Factory fixture to write config.toml files under the scripts dir.""" + + def _write( + scripts_dir: Path, dataset_name: str, dataset_version: str, content: str + ) -> Path: + d = scripts_dir / dataset_name / dataset_version + d.mkdir(parents=True, exist_ok=True) + (d / "config.toml").write_text(content) + return d + + return _write + + +@pytest.fixture +def write_builder(tmp_path: Path) -> Callable[[Path, str, str, str], Path]: + """Factory fixture to write builder.py files under the scripts dir.""" + + def _write( + scripts_dir: Path, dataset_name: str, dataset_version: str, content: str + ) -> Path: + d = scripts_dir / dataset_name / dataset_version + d.mkdir(parents=True, exist_ok=True) + (d / "builder.py").write_text(content) + return d + + return _write diff --git a/builders/server/tests/service/__init__.py b/builders/server/tests/core/db/__init__.py similarity index 100% rename from builders/server/tests/service/__init__.py rename to builders/server/tests/core/db/__init__.py diff --git a/builders/server/tests/db/test_connection.py b/builders/server/tests/core/db/test_connection.py similarity index 91% rename from builders/server/tests/db/test_connection.py rename to builders/server/tests/core/db/test_connection.py index 72f9d56..628cd6a 100644 --- a/builders/server/tests/db/test_connection.py +++ b/builders/server/tests/core/db/test_connection.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock, patch import pytest -from db import connection +from core.db import connection @pytest.fixture(autouse=True) @@ -13,7 +13,7 @@ def reset_pool() -> Generator[None, None, None]: connection._pool = None -@patch("db.connection.ConnectionPool") +@patch("core.db.connection.ConnectionPool") def test_open_pool_creates_pool(mock_pool_cls: MagicMock) -> None: """open_pool initializes ConnectionPool.""" connection.open_pool("postgresql://test@localhost/test", min_size=1, max_size=5) @@ -26,7 +26,7 @@ def test_open_pool_creates_pool(mock_pool_cls: MagicMock) -> None: assert connection._pool is mock_pool_cls.return_value -@patch("db.connection.ConnectionPool") +@patch("core.db.connection.ConnectionPool") def test_close_pool_closes_and_clears(mock_pool_cls: MagicMock) -> None: """close_pool calls close and sets _pool to None.""" connection.open_pool("postgresql://test@localhost/test") @@ -49,7 +49,7 @@ def test_get_conn_raises_without_pool() -> None: connection.get_conn() -@patch("db.connection.ConnectionPool") +@patch("core.db.connection.ConnectionPool") def test_get_conn_delegates_to_pool(mock_pool_cls: MagicMock) -> None: """get_conn returns pool.connection() context manager.""" mock_pool = mock_pool_cls.return_value diff --git a/builders/server/tests/db/test_datasets.py b/builders/server/tests/core/db/test_datasets.py similarity index 94% rename from builders/server/tests/db/test_datasets.py rename to builders/server/tests/core/db/test_datasets.py index 4ced1d5..0e7cc5c 100644 --- a/builders/server/tests/db/test_datasets.py +++ b/builders/server/tests/core/db/test_datasets.py @@ -2,8 +2,8 @@ from datetime import datetime from unittest.mock import MagicMock, patch -from db import datasets -from utils.semver import SemVer +from core.db import datasets +from core.utils.semver import SemVer V010 = SemVer.parse("0.1.0") @@ -18,7 +18,7 @@ def _get_conn(): return _get_conn -@patch("db.datasets.get_conn") +@patch("core.db.datasets.get_conn") def test_get_existing_timestamps(mock_get_conn: MagicMock) -> None: """Returns list[datetime] from cursor rows.""" mock_cursor = MagicMock() @@ -43,7 +43,7 @@ def test_get_existing_timestamps(mock_get_conn: MagicMock) -> None: assert "DISTINCT" in executed_sql -@patch("db.datasets.get_conn") +@patch("core.db.datasets.get_conn") def test_get_existing_timestamps_empty(mock_get_conn: MagicMock) -> None: """No rows returns empty list.""" mock_cursor = MagicMock() @@ -59,14 +59,14 @@ def test_get_existing_timestamps_empty(mock_get_conn: MagicMock) -> None: assert result == [] -@patch("db.datasets.get_conn") +@patch("core.db.datasets.get_conn") def test_insert_rows_empty_returns_early(mock_get_conn: MagicMock) -> None: """Empty rows list skips DB call.""" datasets.insert_rows("ds", V010, []) mock_get_conn.assert_not_called() -@patch("db.datasets.get_conn") +@patch("core.db.datasets.get_conn") def test_insert_rows_calls_executemany(mock_get_conn: MagicMock) -> None: """Verify executemany is called with correct SQL and args.""" mock_cursor = MagicMock() @@ -89,7 +89,7 @@ def test_insert_rows_calls_executemany(mock_get_conn: MagicMock) -> None: assert len(insert_tuples) == 2 -@patch("db.datasets.get_conn") +@patch("core.db.datasets.get_conn") def test_get_rows_timestamps_empty_timestamps_returns_empty_dict( mock_get_conn: MagicMock, ) -> None: @@ -99,7 +99,7 @@ def test_get_rows_timestamps_empty_timestamps_returns_empty_dict( mock_get_conn.assert_not_called() -@patch("db.datasets.get_conn") +@patch("core.db.datasets.get_conn") def test_get_rows_timestamps_returns_list_per_timestamp( mock_get_conn: MagicMock, ) -> None: @@ -122,7 +122,7 @@ def test_get_rows_timestamps_returns_list_per_timestamp( assert result[ts][1] == {"ticker": "MSFT", "price": 200} -@patch("db.datasets.get_conn") +@patch("core.db.datasets.get_conn") def test_get_rows_range_returns_dict_by_timestamp(mock_get_conn: MagicMock) -> None: """Returns dict[datetime, list[dict]] for a time range.""" ts1 = datetime(2024, 1, 1) @@ -146,7 +146,7 @@ def test_get_rows_range_returns_dict_by_timestamp(mock_get_conn: MagicMock) -> N assert result[ts2][0] == {"val": 20} -@patch("db.datasets.get_conn") +@patch("core.db.datasets.get_conn") def test_get_datasets_with_data_returns_set(mock_get_conn: MagicMock) -> None: """Returns set of (name, version) tuples from cursor rows.""" mock_cursor = MagicMock() @@ -167,7 +167,7 @@ def test_get_datasets_with_data_returns_set(mock_get_conn: MagicMock) -> None: assert "dataset_version" in executed_sql -@patch("db.datasets.get_conn") +@patch("core.db.datasets.get_conn") def test_get_datasets_with_data_empty_table(mock_get_conn: MagicMock) -> None: """Empty table returns empty set.""" mock_cursor = MagicMock() diff --git a/builders/server/tests/utils/__init__.py b/builders/server/tests/core/runtime/__init__.py similarity index 100% rename from builders/server/tests/utils/__init__.py rename to builders/server/tests/core/runtime/__init__.py diff --git a/builders/server/tests/runtime/test_config.py b/builders/server/tests/core/runtime/test_config.py similarity index 98% rename from builders/server/tests/runtime/test_config.py rename to builders/server/tests/core/runtime/test_config.py index 759cb66..5502ae9 100644 --- a/builders/server/tests/runtime/test_config.py +++ b/builders/server/tests/core/runtime/test_config.py @@ -1,7 +1,7 @@ from datetime import timedelta import pytest -from runtime.config import ( +from core.runtime.config import ( SchemaType, normalize_config, parse_lookback, diff --git a/builders/server/tests/runtime/test_loader.py b/builders/server/tests/core/runtime/test_loader.py similarity index 98% rename from builders/server/tests/runtime/test_loader.py rename to builders/server/tests/core/runtime/test_loader.py index d1859dc..2b11fed 100644 --- a/builders/server/tests/runtime/test_loader.py +++ b/builders/server/tests/core/runtime/test_loader.py @@ -3,8 +3,8 @@ from pathlib import Path import pytest -from runtime import loader -from utils.semver import SemVer +from core.runtime import loader +from core.utils.semver import SemVer V010 = SemVer.parse("0.1.0") diff --git a/builders/server/tests/runtime/test_registry.py b/builders/server/tests/core/runtime/test_registry.py similarity index 98% rename from builders/server/tests/runtime/test_registry.py rename to builders/server/tests/core/runtime/test_registry.py index 065a4ce..e9df7c3 100644 --- a/builders/server/tests/runtime/test_registry.py +++ b/builders/server/tests/core/runtime/test_registry.py @@ -1,10 +1,10 @@ from collections.abc import Callable from pathlib import Path +import core.runtime.registry as registry import pytest -import runtime.registry as registry -from runtime.config import DatasetConfig -from utils.semver import SemVer +from core.runtime.config import DatasetConfig +from core.utils.semver import SemVer V010 = SemVer.parse("0.1.0") V020 = SemVer.parse("0.2.0") diff --git a/builders/server/tests/runtime/test_runner.py b/builders/server/tests/core/runtime/test_runner.py similarity index 99% rename from builders/server/tests/runtime/test_runner.py rename to builders/server/tests/core/runtime/test_runner.py index 386804c..14d1621 100644 --- a/builders/server/tests/runtime/test_runner.py +++ b/builders/server/tests/core/runtime/test_runner.py @@ -3,7 +3,7 @@ from pathlib import Path import pytest -from runtime import runner +from core.runtime import runner def _write_builder(tmp_path: Path, code: str) -> Path: diff --git a/builders/server/tests/runtime/test_serialization.py b/builders/server/tests/core/runtime/test_serialization.py similarity index 98% rename from builders/server/tests/runtime/test_serialization.py rename to builders/server/tests/core/runtime/test_serialization.py index b93ce34..6f4feea 100644 --- a/builders/server/tests/runtime/test_serialization.py +++ b/builders/server/tests/core/runtime/test_serialization.py @@ -3,7 +3,7 @@ from pathlib import Path import pytest -from runtime.serialization import ( +from core.runtime.serialization import ( WorkerError, WorkerSuccess, deserialize_output, diff --git a/builders/server/tests/runtime/test_validator.py b/builders/server/tests/core/runtime/test_validator.py similarity index 96% rename from builders/server/tests/runtime/test_validator.py rename to builders/server/tests/core/runtime/test_validator.py index c31e731..d8ee66c 100644 --- a/builders/server/tests/runtime/test_validator.py +++ b/builders/server/tests/core/runtime/test_validator.py @@ -1,6 +1,6 @@ import pytest -from runtime.config import SchemaType -from runtime.validator import ValidationError, validate, validate_rows +from core.runtime.config import SchemaType +from core.runtime.validator import ValidationError, validate, validate_rows def test_valid_data_passes() -> None: diff --git a/builders/server/tests/runtime/test_venv_management.py b/builders/server/tests/core/runtime/test_venv_management.py similarity index 88% rename from builders/server/tests/runtime/test_venv_management.py rename to builders/server/tests/core/runtime/test_venv_management.py index ba3d6c9..6f7a98b 100644 --- a/builders/server/tests/runtime/test_venv_management.py +++ b/builders/server/tests/core/runtime/test_venv_management.py @@ -1,7 +1,7 @@ from pathlib import Path from unittest.mock import patch -from runtime.venv_management import _ensure_venv, setup_builder_venvs +from core.runtime.venv_management import _ensure_venv, setup_builder_venvs def _make_builder(tmp_path: Path, name: str, version: str, reqs: str) -> Path: @@ -13,7 +13,7 @@ def _make_builder(tmp_path: Path, name: str, version: str, reqs: str) -> Path: return d -@patch("runtime.venv_management.subprocess.run") +@patch("core.runtime.venv_management.subprocess.run") def test_ensure_venv_creates_venv(mock_run, tmp_path: Path): """First run creates venv and writes hash file.""" builder_dir = _make_builder(tmp_path, "ds", "0.1.0", "requests==2.32.0\n") @@ -29,7 +29,7 @@ def test_ensure_venv_creates_venv(mock_run, tmp_path: Path): assert hash_file.exists() -@patch("runtime.venv_management.subprocess.run") +@patch("core.runtime.venv_management.subprocess.run") def test_ensure_venv_skips_when_hash_matches(mock_run, tmp_path: Path): """Second run with same requirements skips venv creation.""" builder_dir = _make_builder(tmp_path, "ds", "0.1.0", "requests==2.32.0\n") @@ -43,7 +43,7 @@ def test_ensure_venv_skips_when_hash_matches(mock_run, tmp_path: Path): mock_run.assert_not_called() -@patch("runtime.venv_management.subprocess.run") +@patch("core.runtime.venv_management.subprocess.run") def test_ensure_venv_rebuilds_on_requirements_change(mock_run, tmp_path: Path): """Changing requirements.txt triggers rebuild.""" builder_dir = _make_builder(tmp_path, "ds", "0.1.0", "requests==2.32.0\n") @@ -58,7 +58,7 @@ def test_ensure_venv_rebuilds_on_requirements_change(mock_run, tmp_path: Path): assert mock_run.call_count == 2 -@patch("runtime.venv_management._ensure_venv") +@patch("core.runtime.venv_management._ensure_venv") def test_setup_skips_dirs_without_requirements(mock_ensure, tmp_path: Path): """Directories without requirements.txt are skipped.""" # builder with no requirements.txt @@ -70,7 +70,7 @@ def test_setup_skips_dirs_without_requirements(mock_ensure, tmp_path: Path): mock_ensure.assert_not_called() -@patch("runtime.venv_management._ensure_venv") +@patch("core.runtime.venv_management._ensure_venv") def test_setup_continues_on_error(mock_ensure, tmp_path: Path): """One builder's venv failure doesn't block others.""" _make_builder(tmp_path, "ds1", "0.1.0", "bad\n") diff --git a/builders/server/utils/__init__.py b/builders/server/tests/core/service/__init__.py similarity index 100% rename from builders/server/utils/__init__.py rename to builders/server/tests/core/service/__init__.py diff --git a/builders/server/tests/service/conftest.py b/builders/server/tests/core/service/conftest.py similarity index 77% rename from builders/server/tests/service/conftest.py rename to builders/server/tests/core/service/conftest.py index 4800014..cc29d74 100644 --- a/builders/server/tests/service/conftest.py +++ b/builders/server/tests/core/service/conftest.py @@ -1,9 +1,14 @@ from datetime import datetime, timedelta -from calendars.interface import Calendar -from calendars.registry import CALENDARS_MAP -from runtime.config import DEFAULT_BUILDER, DatasetConfig, DependencyInfo, SchemaType -from utils.semver import SemVer +from core.calendars.interface import Calendar +from core.calendars.registry import CALENDARS_MAP +from core.runtime.config import ( + DEFAULT_BUILDER, + DatasetConfig, + DependencyInfo, + SchemaType, +) +from core.utils.semver import SemVer V010 = SemVer.parse("0.1.0") _1D = timedelta(days=1) diff --git a/builders/server/tests/service/test_builder.py b/builders/server/tests/core/service/test_builder.py similarity index 91% rename from builders/server/tests/service/test_builder.py rename to builders/server/tests/core/service/test_builder.py index 5eeefd0..e955808 100644 --- a/builders/server/tests/service/test_builder.py +++ b/builders/server/tests/core/service/test_builder.py @@ -2,10 +2,10 @@ from unittest.mock import MagicMock, patch import pytest -from calendars.definitions.always_open import AlwaysOpenCalendar -from calendars.definitions.everyday import EverydayCalendar -from calendars.definitions.weekday import WeekdayCalendar -from service.builder import ( +from core.calendars.definitions.always_open import AlwaysOpenCalendar +from core.calendars.definitions.everyday import EverydayCalendar +from core.calendars.definitions.weekday import WeekdayCalendar +from core.service.builder import ( NoValidTimestampsError, build_dataset, generate_timestamps, @@ -127,7 +127,7 @@ def test_generate_timestamps_start_on_closed_day_no_valid_range_returns_empty() # detailed build behavior is tested in test_scheduler, test_worker, test_orchestrator -@patch("service.builder.run_build") +@patch("core.service.builder.run_build") def test_build_dataset_delegates_to_orchestrator(mock_run_build: MagicMock) -> None: """build_dataset delegates to run_build with the same args.""" build_dataset("ds", V010, datetime(2024, 1, 1), datetime(2024, 1, 5)) @@ -137,7 +137,7 @@ def test_build_dataset_delegates_to_orchestrator(mock_run_build: MagicMock) -> N ) -@patch("service.builder.run_build") +@patch("core.service.builder.run_build") def test_build_dataset_propagates_value_error(mock_run_build: MagicMock) -> None: """ValueError from scheduler (end before start-date) propagates.""" mock_run_build.side_effect = ValueError("before start-date") @@ -146,7 +146,7 @@ def test_build_dataset_propagates_value_error(mock_run_build: MagicMock) -> None build_dataset("ds", V010, datetime(2024, 5, 1), datetime(2024, 5, 15)) -@patch("service.builder.run_build") +@patch("core.service.builder.run_build") def test_build_dataset_propagates_runtime_error(mock_run_build: MagicMock) -> None: """RuntimeError from worker failure propagates.""" mock_run_build.side_effect = RuntimeError("build failed") @@ -158,8 +158,8 @@ def test_build_dataset_propagates_runtime_error(mock_run_build: MagicMock) -> No # --- get_data tests --- -@patch("service.builder.db.datasets") -@patch("service.builder.registry") +@patch("core.db.datasets") +@patch("core.service.builder.registry") def test_get_data_no_build_returns_data( mock_registry: MagicMock, mock_db: MagicMock ) -> None: @@ -180,8 +180,8 @@ def test_get_data_no_build_returns_data( mock_db.get_rows_range.assert_called_once_with("ds", V010, start, end) -@patch("service.builder.db.datasets") -@patch("service.builder.registry") +@patch("core.db.datasets") +@patch("core.service.builder.registry") def test_get_data_no_build_empty_result( mock_registry: MagicMock, mock_db: MagicMock ) -> None: @@ -202,9 +202,9 @@ def test_get_data_no_build_empty_result( assert result.total_timestamps == 2 -@patch("service.builder.build_dataset") -@patch("service.builder.db.datasets") -@patch("service.builder.registry") +@patch("core.service.builder.build_dataset") +@patch("core.db.datasets") +@patch("core.service.builder.registry") def test_get_data_with_build_calls_build_dataset( mock_registry: MagicMock, mock_db: MagicMock, @@ -229,8 +229,8 @@ def test_get_data_with_build_calls_build_dataset( assert result.returned_timestamps == 1 -@patch("service.builder.build_dataset") -@patch("service.builder.registry") +@patch("core.service.builder.build_dataset") +@patch("core.service.builder.registry") def test_get_data_with_build_no_valid_timestamps_raises( mock_registry: MagicMock, mock_build: MagicMock, @@ -249,7 +249,7 @@ def test_get_data_with_build_no_valid_timestamps_raises( ) -@patch("service.builder.registry") +@patch("core.service.builder.registry") def test_get_data_config_not_found_raises(mock_registry: MagicMock) -> None: """get_data raises when dataset config doesn't exist in registry.""" mock_registry.get_config.side_effect = ValueError("not found in config registry") diff --git a/builders/server/tests/service/test_catalog.py b/builders/server/tests/core/service/test_catalog.py similarity index 76% rename from builders/server/tests/service/test_catalog.py rename to builders/server/tests/core/service/test_catalog.py index 0d551b8..23f121d 100644 --- a/builders/server/tests/service/test_catalog.py +++ b/builders/server/tests/core/service/test_catalog.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock, patch -from service.catalog import DatasetInfo, list_datasets -from utils.semver import SemVer +from core.service.catalog import DatasetInfo, list_datasets +from core.utils.semver import SemVer V010 = SemVer.parse("0.1.0") V020 = SemVer.parse("0.2.0") @@ -16,8 +16,8 @@ # --- list_datasets tests --- -@patch("service.catalog.db.datasets.get_datasets_with_data") -@patch("service.catalog.iter_config_keys", return_value=_MOCK_KEYS) +@patch("core.db.datasets.get_datasets_with_data") +@patch("core.service.catalog.iter_config_keys", return_value=_MOCK_KEYS) def test_list_datasets_marks_has_data( mock_keys: MagicMock, mock_has_data: MagicMock ) -> None: @@ -34,8 +34,8 @@ def test_list_datasets_marks_has_data( assert result[1] == DatasetInfo(name="mock-ohlc", version="0.1.0", has_data=True) -@patch("service.catalog.db.datasets.get_datasets_with_data") -@patch("service.catalog.iter_config_keys", return_value=_MOCK_KEYS) +@patch("core.db.datasets.get_datasets_with_data") +@patch("core.service.catalog.iter_config_keys", return_value=_MOCK_KEYS) def test_list_datasets_all_no_data( mock_keys: MagicMock, mock_has_data: MagicMock ) -> None: @@ -48,8 +48,8 @@ def test_list_datasets_all_no_data( assert len(result) == 2 -@patch("service.catalog.db.datasets.get_datasets_with_data") -@patch("service.catalog.iter_config_keys", return_value=[]) +@patch("core.db.datasets.get_datasets_with_data") +@patch("core.service.catalog.iter_config_keys", return_value=[]) def test_list_datasets_empty_registry( mock_keys: MagicMock, mock_has_data: MagicMock ) -> None: @@ -61,9 +61,9 @@ def test_list_datasets_empty_registry( assert result == [] -@patch("service.catalog.db.datasets.get_datasets_with_data") +@patch("core.db.datasets.get_datasets_with_data") @patch( - "service.catalog.iter_config_keys", + "core.service.catalog.iter_config_keys", return_value=[("a", V010), ("b", V010), ("c", V010)], ) def test_list_datasets_single_db_call( @@ -77,9 +77,9 @@ def test_list_datasets_single_db_call( mock_has_data.assert_called_once() -@patch("service.catalog.db.datasets.get_datasets_with_data") +@patch("core.db.datasets.get_datasets_with_data") @patch( - "service.catalog.iter_config_keys", + "core.service.catalog.iter_config_keys", return_value=[("z-ds", V010), ("a-ds", V010), ("m-ds", V020)], ) def test_list_datasets_sorted_by_name_then_version( diff --git a/builders/server/tests/service/test_concurrent_builds.py b/builders/server/tests/core/service/test_concurrent_builds.py similarity index 89% rename from builders/server/tests/service/test_concurrent_builds.py rename to builders/server/tests/core/service/test_concurrent_builds.py index dad6f86..72f1c7b 100644 --- a/builders/server/tests/service/test_concurrent_builds.py +++ b/builders/server/tests/core/service/test_concurrent_builds.py @@ -3,8 +3,8 @@ from unittest.mock import MagicMock, patch import pytest -from runtime.config import DependencyInfo -from service.builder import build_dataset +from core.runtime.config import DependencyInfo +from core.service.builder import build_dataset from .conftest import V010, _cfg @@ -20,11 +20,11 @@ def _mock_both_registries(mock_sched_reg, mock_worker_reg, configs): @pytest.mark.parametrize("n_threads", [3, 5, 10, 15]) -@patch("service.worker.validator") -@patch("service.worker.runner") -@patch("service.worker.db.datasets") -@patch("service.worker.registry") -@patch("service.scheduler.registry") +@patch("core.service.worker.validator") +@patch("core.service.worker.runner") +@patch("core.db.datasets") +@patch("core.service.worker.registry") +@patch("core.service.scheduler.registry") def test_concurrent_builds_same_dataset_no_duplicates( mock_sched_reg: MagicMock, mock_worker_reg: MagicMock, @@ -77,11 +77,11 @@ def build_thread(): assert insert_call_count == 1 -@patch("service.worker.validator") -@patch("service.worker.runner") -@patch("service.worker.db.datasets") -@patch("service.worker.registry") -@patch("service.scheduler.registry") +@patch("core.service.worker.validator") +@patch("core.service.worker.runner") +@patch("core.db.datasets") +@patch("core.service.worker.registry") +@patch("core.service.scheduler.registry") def test_concurrent_builds_overlapping_ranges( mock_sched_reg: MagicMock, mock_worker_reg: MagicMock, @@ -138,11 +138,11 @@ def build_range(start, end): assert count == 1, f"Jan {d} inserted {count} times, expected 1" -@patch("service.worker.validator") -@patch("service.worker.runner") -@patch("service.worker.db.datasets") -@patch("service.worker.registry") -@patch("service.scheduler.registry") +@patch("core.service.worker.validator") +@patch("core.service.worker.runner") +@patch("core.db.datasets") +@patch("core.service.worker.registry") +@patch("core.service.scheduler.registry") def test_concurrent_builds_shared_dependency( mock_sched_reg: MagicMock, mock_worker_reg: MagicMock, @@ -207,11 +207,11 @@ def build_thread(name): assert insert_counts.get("shared-dep", 0) == 1 -@patch("service.worker.validator") -@patch("service.worker.runner") -@patch("service.worker.db.datasets") -@patch("service.worker.registry") -@patch("service.scheduler.registry") +@patch("core.service.worker.validator") +@patch("core.service.worker.runner") +@patch("core.db.datasets") +@patch("core.service.worker.registry") +@patch("core.service.scheduler.registry") def test_concurrent_builds_deep_dependency_chain( mock_sched_reg: MagicMock, mock_worker_reg: MagicMock, @@ -279,11 +279,11 @@ def build_thread(name): ) -@patch("service.worker.validator") -@patch("service.worker.runner") -@patch("service.worker.db.datasets") -@patch("service.worker.registry") -@patch("service.scheduler.registry") +@patch("core.service.worker.validator") +@patch("core.service.worker.runner") +@patch("core.db.datasets") +@patch("core.service.worker.registry") +@patch("core.service.scheduler.registry") def test_lock_released_on_builder_failure( mock_sched_reg: MagicMock, mock_worker_reg: MagicMock, @@ -316,11 +316,11 @@ def fake_run_builder(*args, **kwargs): assert mock_runner.run_builder.call_count == 2 -@patch("service.worker.validator") -@patch("service.worker.runner") -@patch("service.worker.db.datasets") -@patch("service.worker.registry") -@patch("service.scheduler.registry") +@patch("core.service.worker.validator") +@patch("core.service.worker.runner") +@patch("core.db.datasets") +@patch("core.service.worker.registry") +@patch("core.service.scheduler.registry") def test_second_request_skips_after_first_completes( mock_sched_reg: MagicMock, mock_worker_reg: MagicMock, diff --git a/builders/server/tests/service/test_locks.py b/builders/server/tests/core/service/test_locks.py similarity index 86% rename from builders/server/tests/service/test_locks.py rename to builders/server/tests/core/service/test_locks.py index a069f5b..ca53cf3 100644 --- a/builders/server/tests/service/test_locks.py +++ b/builders/server/tests/core/service/test_locks.py @@ -1,12 +1,12 @@ import threading from unittest.mock import patch -from service.locks import get_build_lock +from core.service.locks import get_build_lock def test_get_build_lock_returns_same_lock_for_same_key() -> None: """Same (name, version) returns the same Lock instance.""" - with patch("service.locks._lock_map", {}): + with patch("core.service.locks._lock_map", {}): lock1 = get_build_lock("ds", "0.1.0") lock2 = get_build_lock("ds", "0.1.0") assert lock1 is lock2 @@ -14,7 +14,7 @@ def test_get_build_lock_returns_same_lock_for_same_key() -> None: def test_get_build_lock_returns_different_locks_for_different_keys() -> None: """Different (name, version) pairs return distinct Lock instances.""" - with patch("service.locks._lock_map", {}): + with patch("core.service.locks._lock_map", {}): lock_a = get_build_lock("ds-a", "0.1.0") lock_b = get_build_lock("ds-b", "0.1.0") lock_a_v2 = get_build_lock("ds-a", "0.2.0") @@ -24,7 +24,7 @@ def test_get_build_lock_returns_different_locks_for_different_keys() -> None: def test_get_build_lock_is_thread_safe() -> None: """Concurrent calls from multiple threads all get the same lock.""" - with patch("service.locks._lock_map", {}): + with patch("core.service.locks._lock_map", {}): results: list[threading.Lock] = [] barrier = threading.Barrier(10) diff --git a/builders/server/tests/service/test_models.py b/builders/server/tests/core/service/test_models.py similarity index 97% rename from builders/server/tests/service/test_models.py rename to builders/server/tests/core/service/test_models.py index 8810263..3d30662 100644 --- a/builders/server/tests/service/test_models.py +++ b/builders/server/tests/core/service/test_models.py @@ -1,8 +1,8 @@ from datetime import datetime import pytest -from service.models import BuildPlan, JobDescriptor, JobResult -from utils.semver import SemVer +from core.service.models import BuildPlan, JobDescriptor, JobResult +from core.utils.semver import SemVer V010 = SemVer.parse("0.1.0") V020 = SemVer.parse("0.2.0") diff --git a/builders/server/tests/service/test_orchestrator.py b/builders/server/tests/core/service/test_orchestrator.py similarity index 88% rename from builders/server/tests/service/test_orchestrator.py rename to builders/server/tests/core/service/test_orchestrator.py index 908e085..802af65 100644 --- a/builders/server/tests/service/test_orchestrator.py +++ b/builders/server/tests/core/service/test_orchestrator.py @@ -2,8 +2,8 @@ from unittest.mock import MagicMock, patch import pytest -from runtime.config import DependencyInfo -from service.orchestrator import run_build +from core.runtime.config import DependencyInfo +from core.service.orchestrator import run_build from .conftest import V010, _cfg @@ -14,8 +14,8 @@ # --- single root dataset works --- -@patch("service.orchestrator.execute_job") -@patch("service.scheduler.registry") +@patch("core.service.orchestrator.execute_job") +@patch("core.service.scheduler.registry") def test_single_root(mock_registry, mock_execute) -> None: """Root with no deps -> schedule + execute 1 job successfully.""" mock_registry.get_config.return_value = _cfg(name="root") @@ -31,8 +31,8 @@ def test_single_root(mock_registry, mock_execute) -> None: # --- level-by-level execution order --- -@patch("service.orchestrator.execute_job") -@patch("service.scheduler.registry") +@patch("core.service.orchestrator.execute_job") +@patch("core.service.scheduler.registry") def test_level_order_execution(mock_registry, mock_execute) -> None: """A -> B -> C: C built first, then B, then A.""" configs = { @@ -54,8 +54,8 @@ def test_level_order_execution(mock_registry, mock_execute) -> None: # --- failure at level N prevents level N+1 --- -@patch("service.orchestrator.execute_job") -@patch("service.scheduler.registry") +@patch("core.service.orchestrator.execute_job") +@patch("core.service.scheduler.registry") def test_failure_stops_subsequent_levels(mock_registry, mock_execute) -> None: """If B fails at level 1, A at level 2 never executes.""" configs = { @@ -86,8 +86,8 @@ def mock_exec(job, cancelled): # --- NoValidTimestampsError propagates --- -@patch("service.orchestrator.execute_job") -@patch("service.scheduler.registry") +@patch("core.service.orchestrator.execute_job") +@patch("core.service.scheduler.registry") def test_no_valid_timestamps_propagates(mock_registry, mock_execute) -> None: """Worker failure with no-valid-timestamps error propagates as RuntimeError.""" mock_registry.get_config.return_value = _cfg(name="ds") @@ -102,8 +102,8 @@ def test_no_valid_timestamps_propagates(mock_registry, mock_execute) -> None: # --- diamond graph executes correctly --- -@patch("service.orchestrator.execute_job") -@patch("service.scheduler.registry") +@patch("core.service.orchestrator.execute_job") +@patch("core.service.scheduler.registry") def test_diamond_graph(mock_registry, mock_execute) -> None: """Diamond A -> {B, C}, B -> D, C -> D: D once at level 0, {B,C} at 1, A at 2.""" configs = { @@ -140,8 +140,8 @@ def test_diamond_graph(mock_registry, mock_execute) -> None: # --- cancelled event is set on failure --- -@patch("service.orchestrator.execute_job") -@patch("service.scheduler.registry") +@patch("core.service.orchestrator.execute_job") +@patch("core.service.scheduler.registry") def test_cancelled_event_set_on_failure(mock_registry, mock_execute) -> None: """When a job fails, the cancelled event passed to execute_job is set.""" mock_registry.get_config.return_value = _cfg(name="ds") diff --git a/builders/server/tests/service/test_scheduler.py b/builders/server/tests/core/service/test_scheduler.py similarity index 95% rename from builders/server/tests/service/test_scheduler.py rename to builders/server/tests/core/service/test_scheduler.py index 2514f54..2a2a005 100644 --- a/builders/server/tests/service/test_scheduler.py +++ b/builders/server/tests/core/service/test_scheduler.py @@ -2,8 +2,8 @@ from unittest.mock import patch import pytest -from runtime.config import DependencyInfo -from service.scheduler import collect_graph, schedule_build +from core.runtime.config import DependencyInfo +from core.service.scheduler import collect_graph, schedule_build from .conftest import V010, _cfg @@ -13,7 +13,7 @@ # --- single root, no deps --- -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_single_root_no_deps(mock_registry) -> None: """Root dataset with no dependencies produces 1 node, no edges.""" mock_registry.get_config.return_value = _cfg(name="root") @@ -29,7 +29,7 @@ def test_single_root_no_deps(mock_registry) -> None: # --- linear chain --- -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_linear_chain(mock_registry) -> None: """A -> B -> C produces 3 nodes with correct edges and identical ranges.""" configs = { @@ -63,7 +63,7 @@ def test_linear_chain(mock_registry) -> None: # --- diamond dependency --- -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_diamond_deduplication(mock_registry) -> None: """Diamond: A -> {B, C}, B -> D, C -> D. D appears once in ranges.""" configs = { @@ -102,7 +102,7 @@ def test_diamond_deduplication(mock_registry) -> None: # --- lookback expansion --- -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_lookback_expands_dep_range(mock_registry) -> None: """Lookback on a dependency widens its build range backwards.""" configs = { @@ -133,7 +133,7 @@ def test_lookback_expands_dep_range(mock_registry) -> None: ) -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_lookback_propagates_through_chain(mock_registry) -> None: """Lookback expansion propagates: A (lookback=5d on B) -> B -> C. C's range should be expanded by A's lookback on B.""" @@ -177,7 +177,7 @@ def test_lookback_propagates_through_chain(mock_registry) -> None: # --- diamond with different lookbacks --- -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_diamond_different_lookbacks_union(mock_registry) -> None: """Diamond where B and C both depend on D with different lookbacks. D's range should be the union of both expanded ranges.""" @@ -219,7 +219,7 @@ def test_diamond_different_lookbacks_union(mock_registry) -> None: # --- start-date clamping --- -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_start_date_clamping(mock_registry) -> None: """Start before dataset start-date gets clamped forward.""" mock_registry.get_config.return_value = _cfg( @@ -235,7 +235,7 @@ def test_start_date_clamping(mock_registry) -> None: ) -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_end_before_start_date_raises(mock_registry) -> None: """End before dataset start-date raises ValueError.""" mock_registry.get_config.return_value = _cfg( @@ -246,7 +246,7 @@ def test_end_before_start_date_raises(mock_registry) -> None: collect_graph("ds", V010, datetime(2024, 5, 1), datetime(2024, 5, 15)) -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_lookback_clamped_by_dep_start_date(mock_registry) -> None: """Lookback expansion that pushes start before dep's start-date gets clamped.""" configs = { @@ -273,7 +273,7 @@ def test_lookback_clamped_by_dep_start_date(mock_registry) -> None: # --- diamond re-expansion propagation --- -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_diamond_reexpansion_propagates_to_grandchildren(mock_registry) -> None: """When a diamond dep's range expands on second visit, the expansion propagates to its children. @@ -333,7 +333,7 @@ def _job_names_by_level(plan) -> list[set[str]]: return [{j.dataset_name for j in level} for level in plan.levels] -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_schedule_single_root(mock_registry) -> None: """Root with no deps -> 1 level, 1 job.""" mock_registry.get_config.return_value = _cfg(name="root") @@ -348,7 +348,7 @@ def test_schedule_single_root(mock_registry) -> None: assert job.end == datetime(2024, 1, 5) -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_schedule_linear_chain_order(mock_registry) -> None: """A -> B -> C produces 3 levels: C first, A last.""" configs = { @@ -373,7 +373,7 @@ def test_schedule_linear_chain_order(mock_registry) -> None: assert names[2] == {"A"} # level 2, requested dataset -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_schedule_diamond_levels(mock_registry) -> None: """Diamond produces 3 levels: D at 0, {B, C} at 1, A at 2.""" configs = { @@ -405,7 +405,7 @@ def test_schedule_diamond_levels(mock_registry) -> None: assert names[2] == {"A"} -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_schedule_lookback_ranges_in_jobs(mock_registry) -> None: """Lookback-expanded ranges are reflected in the JobDescriptor start/end.""" configs = { @@ -436,7 +436,7 @@ def test_schedule_lookback_ranges_in_jobs(mock_registry) -> None: assert parent_job.end == datetime(2024, 1, 20) -@patch("service.scheduler.registry") +@patch("core.service.scheduler.registry") def test_schedule_jobs_within_level_are_independent(mock_registry) -> None: """Jobs within the same level have no dependency edges between them.""" configs = { diff --git a/builders/server/tests/service/test_worker.py b/builders/server/tests/core/service/test_worker.py similarity index 86% rename from builders/server/tests/service/test_worker.py rename to builders/server/tests/core/service/test_worker.py index 122ff25..5b986a7 100644 --- a/builders/server/tests/service/test_worker.py +++ b/builders/server/tests/core/service/test_worker.py @@ -3,9 +3,9 @@ from unittest.mock import patch import pytest -from runtime.config import DependencyInfo, SchemaType -from service.models import JobDescriptor -from service.worker import execute_job +from core.runtime.config import DependencyInfo, SchemaType +from core.service.models import JobDescriptor +from core.service.worker import execute_job from .conftest import V010, _cfg @@ -26,8 +26,8 @@ def _never_cancelled() -> threading.Event: # --- all timestamps exist -> success, no insert --- -@patch("service.worker.db.datasets") -@patch("service.worker.registry") +@patch("core.db.datasets") +@patch("core.service.worker.registry") def test_all_timestamps_exist_skips_build(mock_registry, mock_db) -> None: """When all timestamps already exist, no builder runs and no insert.""" mock_registry.get_config.return_value = _cfg(name="ds") @@ -46,10 +46,10 @@ def test_all_timestamps_exist_skips_build(mock_registry, mock_db) -> None: # --- missing timestamps built, validated, and inserted --- -@patch("service.worker.validator") -@patch("service.worker.runner") -@patch("service.worker.db.datasets") -@patch("service.worker.registry") +@patch("core.service.worker.validator") +@patch("core.service.worker.runner") +@patch("core.db.datasets") +@patch("core.service.worker.registry") def test_missing_timestamps_built_and_inserted( mock_registry, mock_db, mock_runner, mock_validator ) -> None: @@ -77,9 +77,9 @@ def test_missing_timestamps_built_and_inserted( # --- builder failure mid-range -> no rows inserted --- -@patch("service.worker.runner") -@patch("service.worker.db.datasets") -@patch("service.worker.registry") +@patch("core.service.worker.runner") +@patch("core.db.datasets") +@patch("core.service.worker.registry") def test_builder_failure_no_partial_insert(mock_registry, mock_db, mock_runner) -> None: """If builder fails on timestamp 3 of 5, no rows are inserted.""" mock_registry.get_config.return_value = _cfg(name="ds") @@ -107,9 +107,9 @@ def fail_on_third(*args, **kwargs): # --- cancelled event stops early --- -@patch("service.worker.runner") -@patch("service.worker.db.datasets") -@patch("service.worker.registry") +@patch("core.service.worker.runner") +@patch("core.db.datasets") +@patch("core.service.worker.registry") def test_cancelled_event_stops_early(mock_registry, mock_db, mock_runner) -> None: """When cancelled is set, worker stops before building remaining timestamps.""" mock_registry.get_config.return_value = _cfg(name="ds") @@ -139,9 +139,9 @@ def cancel_after_first(*args, **kwargs): # --- lookback dep data uses get_rows_range --- -@patch("service.worker.runner") -@patch("service.worker.db.datasets") -@patch("service.worker.registry") +@patch("core.service.worker.runner") +@patch("core.db.datasets") +@patch("core.service.worker.registry") def test_lookback_dep_uses_get_rows_range(mock_registry, mock_db, mock_runner) -> None: """Dependency with lookback fetches data via get_rows_range.""" mock_registry.get_config.return_value = _cfg( @@ -165,9 +165,9 @@ def test_lookback_dep_uses_get_rows_range(mock_registry, mock_db, mock_runner) - # --- no-lookback dep data uses get_rows_timestamps --- -@patch("service.worker.runner") -@patch("service.worker.db.datasets") -@patch("service.worker.registry") +@patch("core.service.worker.runner") +@patch("core.db.datasets") +@patch("core.service.worker.registry") def test_no_lookback_dep_uses_get_rows_timestamps( mock_registry, mock_db, mock_runner ) -> None: @@ -192,8 +192,8 @@ def test_no_lookback_dep_uses_get_rows_timestamps( # --- missing dep data raises RuntimeError --- -@patch("service.worker.db.datasets") -@patch("service.worker.registry") +@patch("core.db.datasets") +@patch("core.service.worker.registry") def test_missing_dep_data_returns_failure(mock_registry, mock_db) -> None: """When dependency data is missing, worker returns failure.""" mock_registry.get_config.return_value = _cfg( @@ -216,12 +216,12 @@ def test_missing_dep_data_returns_failure(mock_registry, mock_db) -> None: # --- no valid timestamps raises NoValidTimestampsError --- -@patch("service.worker.db.datasets") -@patch("service.worker.registry") +@patch("core.db.datasets") +@patch("core.service.worker.registry") def test_no_valid_timestamps_raises(mock_registry, mock_db) -> None: """When no valid calendar timestamps exist, NoValidTimestampsError propagates.""" - from calendars.registry import CALENDARS_MAP - from service.builder import NoValidTimestampsError + from core.calendars.registry import CALENDARS_MAP + from core.service.builder import NoValidTimestampsError mock_registry.get_config.return_value = _cfg( name="ds", diff --git a/builders/server/tests/core/utils/__init__.py b/builders/server/tests/core/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/builders/server/tests/utils/test_retry.py b/builders/server/tests/core/utils/test_retry.py similarity index 89% rename from builders/server/tests/utils/test_retry.py rename to builders/server/tests/core/utils/test_retry.py index e23198a..cef6f48 100644 --- a/builders/server/tests/utils/test_retry.py +++ b/builders/server/tests/core/utils/test_retry.py @@ -1,7 +1,7 @@ from unittest.mock import MagicMock, patch import pytest -from utils.retry import retry_with_backoff +from core.utils.retry import retry_with_backoff def test_succeeds_first_try() -> None: @@ -14,7 +14,7 @@ def test_succeeds_first_try() -> None: assert fn.call_count == 1 -@patch("utils.retry.time.sleep") +@patch("core.utils.retry.time.sleep") def test_succeeds_after_retries(_mock_sleep: MagicMock) -> None: """Retries until fn succeeds.""" fn = MagicMock(side_effect=[ValueError("fail"), ValueError("fail"), "ok"]) @@ -25,7 +25,7 @@ def test_succeeds_after_retries(_mock_sleep: MagicMock) -> None: assert fn.call_count == 3 -@patch("utils.retry.time.sleep") +@patch("core.utils.retry.time.sleep") def test_exhausts_retries(_mock_sleep: MagicMock) -> None: """Raises last exception after all retries exhausted.""" fn = MagicMock(side_effect=ValueError("persistent failure")) @@ -35,7 +35,7 @@ def test_exhausts_retries(_mock_sleep: MagicMock) -> None: assert fn.call_count == 3 -@patch("utils.retry.time.sleep") +@patch("core.utils.retry.time.sleep") def test_exponential_delay(mock_sleep: MagicMock) -> None: """Sleep delays follow exponential backoff pattern.""" fn = MagicMock(side_effect=[RuntimeError] * 5 + ["ok"]) @@ -46,8 +46,8 @@ def test_exponential_delay(mock_sleep: MagicMock) -> None: assert delays == [2.0, 4.0, 8.0, 16.0, 32.0] -@patch("utils.retry.logger") -@patch("utils.retry.time.sleep") +@patch("core.utils.retry.logger") +@patch("core.utils.retry.time.sleep") def test_logs_retry_attempts(_mock_sleep: MagicMock, mock_logger: MagicMock) -> None: """Warning logged on each retry attempt.""" fn = MagicMock(side_effect=[RuntimeError("oops"), "ok"]) diff --git a/builders/server/tests/utils/test_semver.py b/builders/server/tests/core/utils/test_semver.py similarity index 97% rename from builders/server/tests/utils/test_semver.py rename to builders/server/tests/core/utils/test_semver.py index 9805abe..5316ebe 100644 --- a/builders/server/tests/utils/test_semver.py +++ b/builders/server/tests/core/utils/test_semver.py @@ -1,5 +1,5 @@ import pytest -from utils.semver import SemVer +from core.utils.semver import SemVer def test_parse_basic(): diff --git a/builders/server/tests/integration/conftest.py b/builders/server/tests/integration/conftest.py index 67d6a0a..800ee81 100644 --- a/builders/server/tests/integration/conftest.py +++ b/builders/server/tests/integration/conftest.py @@ -67,21 +67,21 @@ def clean_db(db_conn): @pytest.fixture(autouse=True) def patch_db_conn(db_conn, monkeypatch): """Redirect all production DB calls to the test database.""" - import db.connection - import db.datasets + import core.db.connection + import core.db.datasets @contextmanager def _test_conn(): yield db_conn - monkeypatch.setattr(db.connection, "get_conn", _test_conn) - monkeypatch.setattr(db.datasets, "get_conn", _test_conn) + monkeypatch.setattr(core.db.connection, "get_conn", _test_conn) + monkeypatch.setattr(core.db.datasets, "get_conn", _test_conn) @pytest.fixture(autouse=True) def real_scripts_dir(monkeypatch): """Point SCRIPTS_DIR to the real builders/scripts/ directory.""" - from runtime import config, loader, registry + from core.runtime import config, loader, registry monkeypatch.setattr(config, "SCRIPTS_DIR", REAL_SCRIPTS_DIR) monkeypatch.setattr(loader, "SCRIPTS_DIR", REAL_SCRIPTS_DIR) @@ -91,7 +91,7 @@ def real_scripts_dir(monkeypatch): @pytest.fixture(autouse=True) def integration_retry_settings(monkeypatch): """Use short retries in integration tests to keep suite runtime bounded.""" - from runtime import runner + from core.runtime import runner monkeypatch.setattr(runner, "RETRY_MAX_RETRIES", 3) monkeypatch.setattr(runner, "RETRY_INITIAL_DELAY", 0.01) @@ -117,7 +117,7 @@ def write_temp_builder(tmp_path, monkeypatch): scripts_dir = tmp_path / "scripts" scripts_dir.mkdir() - from runtime import config, loader, registry + from core.runtime import config, loader, registry monkeypatch.setattr(config, "SCRIPTS_DIR", scripts_dir) monkeypatch.setattr(loader, "SCRIPTS_DIR", scripts_dir)