diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18021dce..4aa6a34e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,12 +171,65 @@ jobs: - name: Run tests (parallel) run: > pytest --tb=short -x -m "not glaas and not live_glaas" + --ignore=tests/backends/osmo --ignore=tests/execution/runtime/test_sitecustomize_perf.py - name: Run runtime perf guards (serial) if: matrix.python-version == '3.12' run: pytest --tb=short -x tests/execution/runtime/test_sitecustomize_perf.py -n 0 + test-osmo-e2e: + runs-on: ubuntu-latest + timeout-minutes: 90 + needs: [build-rust-binaries] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Download Rust binaries + uses: actions/download-artifact@v4 + with: + name: roar-rust-binaries-linux-x86_64 + path: rust/target/release/ + + - name: Make tracer and proxy executables runnable + run: | + chmod +x \ + rust/target/release/roar-tracer \ + rust/target/release/roar-proxy \ + rust/target/release/roar-tracer-ebpf \ + rust/target/release/roard \ + rust/target/release/roar-tracer-preload + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install maturin + pip install -e .[dev] + pip install "ray[default]==2.44.1" + + - name: Build native hash extension + run: ./scripts/build_hash_native.sh + + - name: Verify native hash extension import + run: python -c "import roar._hash_native as m; print(m.__name__)" + + - name: Run OSMO e2e tests (serial) + env: + OSMO_DOCKERHUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + OSMO_DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + OSMO_PRELOAD_PULL_RETRIES: "5" + OSMO_QUICK_START_CHART_VERSION: "1.0.1" + OSMO_TEST_PYTHON_IMAGE: public.ecr.aws/docker/library/python:3.11-slim + run: pytest --tb=short -x -m "osmo_e2e" tests/backends/osmo -n 0 + test-macos: needs: [build-rust-binaries-macos] strategy: @@ -252,6 +305,7 @@ jobs: - name: Run tests (parallel) run: > pytest --tb=short -x -m "not glaas and not live_glaas and not ebpf and not large_pipeline" + --ignore=tests/backends/osmo --ignore=tests/execution/runtime/test_sitecustomize_perf.py test-tracer-privileged: diff --git a/.gitignore b/.gitignore index 6c637cbe..104140d6 100644 --- a/.gitignore +++ b/.gitignore @@ -221,6 +221,7 @@ CLAUDE.md # roar roar/bin/* .roar/ +.tmp-osmo-e2e/ .ralph # Ray benchmark results diff --git a/pyproject.toml b/pyproject.toml index 024ad38e..7e074946 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ "cryptography>=42.0.0", "dependency-injector>=4.40.0", "msgpack>=1.0.0", + "PyYAML>=6.0", "sqlalchemy>=2.0.0", "pysqlite3-binary>=0.5.0; sys_platform == 'linux' and platform_machine == 'x86_64'", # Fallback when stdlib sqlite3 unavailable "pydantic>=2.0.0", @@ -74,6 +75,7 @@ roar-worker = "roar.execution.runtime.worker_bootstrap:main" [project.entry-points."roar.execution_backends"] ray = "roar.backends.ray.plugin:register" +osmo = "roar.backends.osmo.plugin:register" [tool.maturin] manifest-path = "rust/crates/artifact-hash-py/Cargo.toml" @@ -103,6 +105,7 @@ markers = [ "happy_path: Happy path tests for core functionality", "diagnostic: Opt-in diagnostics or aspirational performance budgets outside the default gate", "large_pipeline: Stress-style pipeline coverage with larger DAG fixtures", + "osmo_e2e: OSMO end-to-end tests requiring a Docker Compose managed KIND harness", "ray_e2e: Ray end-to-end tests requiring a running Docker cluster", "ray_contract: User-facing Ray contract tests using `roar run ray job submit ...`", "ray_diagnostic: Diagnostic Ray tests that intentionally inspect internal runtime details", diff --git a/roar/backends/osmo/__init__.py b/roar/backends/osmo/__init__.py new file mode 100644 index 00000000..ce457de5 --- /dev/null +++ b/roar/backends/osmo/__init__.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from .export import OsmoLineageBundleExport, export_osmo_lineage_bundle +from .host_execution import OsmoAttachOptions, attach_osmo_workflow, execute_osmo_workflow_submit +from .lineage import discover_downloaded_lineage_bundles, reconstitute_osmo_lineage_bundles +from .plugin import OSMO_EXECUTION_BACKEND, register +from .runtime_bundle import OsmoRuntimeBundle, build_osmo_runtime_bundle +from .workflow import ( + PreparedOsmoWorkflow, + prepare_osmo_workflow_for_lineage, + resolve_roar_install_requirement, +) + +__all__ = [ + "OSMO_EXECUTION_BACKEND", + "OsmoAttachOptions", + "OsmoLineageBundleExport", + "OsmoRuntimeBundle", + "PreparedOsmoWorkflow", + "attach_osmo_workflow", + "build_osmo_runtime_bundle", + "discover_downloaded_lineage_bundles", + "execute_osmo_workflow_submit", + "export_osmo_lineage_bundle", + "prepare_osmo_workflow_for_lineage", + "reconstitute_osmo_lineage_bundles", + "register", + "resolve_roar_install_requirement", +] diff --git a/roar/backends/osmo/config.py b/roar/backends/osmo/config.py new file mode 100644 index 00000000..86f7df20 --- /dev/null +++ b/roar/backends/osmo/config.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from roar.execution.framework.contract import BackendConfigAdapter, ConfigurableKeySpec + + +class OsmoBackendConfig(BaseModel): + """OSMO backend configuration.""" + + model_config = ConfigDict( + strict=False, + validate_assignment=True, + extra="ignore", + revalidate_instances="never", + ) + + enabled: bool = True + auto_prepare_submissions: bool = True + force_json_output: bool = True + wait_for_completion: bool = False + download_declared_outputs: bool = False + download_directory: str = ".roar/osmo/downloads" + ingest_lineage_bundles: bool = False + lineage_bundle_dataset_name: str = "roar-lineage" + lineage_bundle_filename: str = "roar-fragments.json" + runtime_install_requirement: str = "" + runtime_install_local_path: str = "" + runtime_install_remote_path: str = "/tmp/roar-osmo-install.whl" + query_timeout_seconds: int = Field(default=12 * 60, ge=1) + poll_interval_seconds: float = Field(default=5.0, gt=0.0) + + +OSMO_CONFIGURABLE_KEYS = { + "osmo.enabled": ConfigurableKeySpec( + value_type=bool, + default=True, + description="Enable automatic OSMO workflow submit handling in roar run", + ), + "osmo.auto_prepare_submissions": ConfigurableKeySpec( + value_type=bool, + default=True, + description="Rewrite local OSMO workflow submits through a temporary Roar-instrumented workflow", + ), + "osmo.force_json_output": ConfigurableKeySpec( + value_type=bool, + default=True, + description="Append --format-type json to OSMO workflow submit commands when missing", + ), + "osmo.wait_for_completion": ConfigurableKeySpec( + value_type=bool, + default=False, + description="Poll submitted OSMO workflows to a terminal state before completing roar run", + ), + "osmo.download_declared_outputs": ConfigurableKeySpec( + value_type=bool, + default=False, + description="Download declared dataset outputs after successful waited OSMO workflow completion", + ), + "osmo.download_directory": ConfigurableKeySpec( + value_type=str, + default=".roar/osmo/downloads", + description="Directory for downloaded OSMO dataset outputs, relative to the repo root when not absolute", + ), + "osmo.ingest_lineage_bundles": ConfigurableKeySpec( + value_type=bool, + default=False, + description="Ingest downloaded OSMO output bundles named by osmo.lineage_bundle_filename into the local Roar DB", + ), + "osmo.lineage_bundle_dataset_name": ConfigurableKeySpec( + value_type=str, + default="roar-lineage", + description="Dataset name Roar uses by default when downloading returned OSMO lineage bundles", + ), + "osmo.lineage_bundle_filename": ConfigurableKeySpec( + value_type=str, + default="roar-fragments.json", + description="Filename Roar treats as a downloaded OSMO lineage bundle when ingest_lineage_bundles is enabled", + ), + "osmo.runtime_install_requirement": ConfigurableKeySpec( + value_type=str, + default="", + description="Pinned requirement, wheel URL, or install target used by OSMO runtime wrappers when bootstrapping Roar remotely; packaged roar-cli wheels are expected to include tracer binaries", + ), + "osmo.runtime_install_local_path": ConfigurableKeySpec( + value_type=str, + default="", + description="Optional local wheel or artifact path injected into prepared OSMO workflows and installed by the wrapper; roar-cli wheels should be built with packaged binaries", + ), + "osmo.runtime_install_remote_path": ConfigurableKeySpec( + value_type=str, + default="/tmp/roar-osmo-install.whl", + description="Remote path inside OSMO tasks used for an injected runtime install artifact", + ), + "osmo.query_timeout_seconds": ConfigurableKeySpec( + value_type=int, + default=12 * 60, + description="Maximum time to wait for a submitted OSMO workflow to reach a terminal state", + ), + "osmo.poll_interval_seconds": ConfigurableKeySpec( + value_type=float, + default=5.0, + description="Polling interval in seconds when waiting for OSMO workflow completion", + ), +} + +OSMO_INIT_TEMPLATE = """\ +[osmo] +# Enable OSMO workflow-submit recognition in roar run +enabled = true +# Rewrite local workflow submits through a temporary Roar-instrumented workflow +auto_prepare_submissions = true +# Append --format-type json when missing so Roar can capture workflow metadata +force_json_output = true +# Optionally wait for submitted workflows to reach a terminal state +wait_for_completion = false +# Optionally download declared dataset outputs after a successful waited run +download_declared_outputs = false +# Optionally ingest downloaded lineage bundles back into the local Roar DB +ingest_lineage_bundles = false +# Standard dataset name Roar expects for returned lineage bundles +lineage_bundle_dataset_name = "roar-lineage" +# Optional pinned requirement or wheel URL installed by the injected OSMO wrapper +# For roar-cli, use a packaged wheel or index source that includes bundled tracer binaries +runtime_install_requirement = "" +# Optional local wheel path injected into the workflow and installed by the wrapper +# For roar-cli, prefer a wheel built through scripts/build_wheel_with_bins.sh +runtime_install_local_path = "" +# Remote install-artifact path used inside OSMO tasks when runtime_install_local_path is set +runtime_install_remote_path = "/tmp/roar-osmo-install.whl" +""" + + +def normalize_osmo_backend_config(section: Mapping[str, Any] | None) -> dict[str, Any]: + return OsmoBackendConfig.model_validate(dict(section or {})).model_dump() + + +def load_osmo_backend_config(start_dir: str | None = None) -> dict[str, Any]: + try: + from roar.integrations.config import load_config + + config = load_config(start_dir=start_dir) + except Exception: + return dict(OSMO_BACKEND_CONFIG.default_values) + + section = config.get("osmo", {}) + if not isinstance(section, Mapping): + return dict(OSMO_BACKEND_CONFIG.default_values) + return normalize_osmo_backend_config(section) + + +OSMO_BACKEND_CONFIG = BackendConfigAdapter( + section_name="osmo", + default_values=OsmoBackendConfig().model_dump(), + configurable_keys=OSMO_CONFIGURABLE_KEYS, + init_template=OSMO_INIT_TEMPLATE, + normalize_section=normalize_osmo_backend_config, +) + + +__all__ = [ + "OSMO_BACKEND_CONFIG", + "OSMO_CONFIGURABLE_KEYS", + "OSMO_INIT_TEMPLATE", + "OsmoBackendConfig", + "load_osmo_backend_config", + "normalize_osmo_backend_config", +] diff --git a/roar/backends/osmo/export.py b/roar/backends/osmo/export.py new file mode 100644 index 00000000..19172a79 --- /dev/null +++ b/roar/backends/osmo/export.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from roar.db.context import create_database_context +from roar.execution.fragments.models import ArtifactRef, ExecutionFragment + + +@dataclass(frozen=True) +class OsmoLineageBundleExport: + output_path: str + exported_job_uid: str + fragment_count: int + task_id: str + task_name: str + + +def export_osmo_lineage_bundle( + *, + roar_dir: Path, + output_path: Path, + job_uid: str | None = None, + task_id: str | None = None, + task_name: str | None = None, + backend_name: str = "osmo", +) -> OsmoLineageBundleExport: + project_dir = str(roar_dir.parent.resolve()) + + with create_database_context(roar_dir) as db_ctx: + selected_job = _select_export_job(db_ctx, job_uid=job_uid) + if selected_job is None: + raise ValueError("no local Roar jobs are available to export") + + selected_job_id = int(selected_job["id"]) + selected_job_uid = str(selected_job.get("job_uid") or "").strip() + resolved_task_id = str(task_id or selected_job_uid).strip() + if not resolved_task_id: + raise ValueError("task_id is required when exporting a job without a persisted job_uid") + + resolved_task_name = _resolve_task_name(selected_job, task_name) + reads = [ + _build_artifact_ref(item, project_dir=project_dir) + for item in db_ctx.jobs.get_inputs(selected_job_id) + ] + writes = [ + _build_artifact_ref(item, project_dir=project_dir) + for item in db_ctx.jobs.get_outputs(selected_job_id) + ] + + started_at = float(selected_job.get("timestamp") or 0.0) + duration = max(0.0, float(selected_job.get("duration_seconds") or 0.0)) + fragment = ExecutionFragment( + job_uid=selected_job_uid or resolved_task_id, + parent_job_uid="", + task_id=resolved_task_id, + worker_id="", + node_id="", + actor_id=None, + task_name=resolved_task_name, + started_at=started_at, + ended_at=started_at + duration, + exit_code=int(selected_job.get("exit_code") or 0), + backend=str(backend_name or "osmo").strip() or "osmo", + reads=reads, + writes=writes, + backend_metadata={ + "execution_role": "task", + "source_job_uid": selected_job_uid or None, + "source_execution_backend": str(selected_job.get("execution_backend") or "").strip() + or None, + "source_execution_role": str(selected_job.get("execution_role") or "").strip() or None, + "source_job_type": str(selected_job.get("job_type") or "").strip() or None, + }, + ) + + payload = { + "fragments": [fragment.to_dict()], + "metadata": { + "exported_job_uid": selected_job_uid or None, + "task_id": resolved_task_id, + "task_name": resolved_task_name, + }, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + return OsmoLineageBundleExport( + output_path=str(output_path), + exported_job_uid=selected_job_uid, + fragment_count=1, + task_id=resolved_task_id, + task_name=resolved_task_name, + ) + + +def _select_export_job(db_ctx, *, job_uid: str | None) -> dict[str, Any] | None: + if job_uid: + selected = db_ctx.jobs.get_by_uid(job_uid) + if selected is None: + raise ValueError(f"local Roar job not found for job UID {job_uid!r}") + return selected + + jobs = db_ctx.jobs.get_recent(limit=1) + if not jobs: + return None + return jobs[0] + + +def _resolve_task_name(job: dict[str, Any], requested_task_name: str | None) -> str: + explicit = str(requested_task_name or "").strip() + if explicit: + return explicit + + script = str(job.get("script") or "").strip() + if script: + return script + + command = str(job.get("command") or "").strip() + if not command: + return "osmo-task" + + first_token = command.split(" ", 1)[0].strip() + return first_token or "osmo-task" + + +def _build_artifact_ref(payload: dict[str, Any], *, project_dir: str) -> ArtifactRef: + hash_value, hash_algorithm = _select_primary_hash(payload.get("hashes")) + path = _normalize_bundle_path(str(payload.get("path") or ""), project_dir=project_dir) + return ArtifactRef( + path=path, + hash=hash_value, + hash_algorithm=hash_algorithm, + size=int(payload.get("size") or 0), + capture_method="python", + ) + + +def _select_primary_hash(hashes: Any) -> tuple[str | None, str]: + if isinstance(hashes, list): + blake3_row = next( + ( + item + for item in hashes + if isinstance(item, dict) and str(item.get("algorithm") or "").strip() == "blake3" + ), + None, + ) + if isinstance(blake3_row, dict): + digest = str(blake3_row.get("digest") or "").strip() + if digest: + return digest, "blake3" + + for item in hashes: + if not isinstance(item, dict): + continue + algorithm = str(item.get("algorithm") or "").strip() + digest = str(item.get("digest") or "").strip() + if algorithm and digest: + return digest, algorithm + + return None, "blake3" + + +def _normalize_bundle_path(path: str, *, project_dir: str) -> str: + normalized = str(path or "").strip() + if not normalized or "://" in normalized: + return normalized + + candidate = Path(normalized) + project_root = Path(project_dir) + if not candidate.is_absolute(): + candidate = (project_root / candidate).resolve(strict=False) + + try: + relative = candidate.resolve(strict=False).relative_to(project_root) + except ValueError: + return str(candidate) + + return "${ROAR_PROJECT_DIR}/" + relative.as_posix() + + +__all__ = [ + "OsmoLineageBundleExport", + "export_osmo_lineage_bundle", +] diff --git a/roar/backends/osmo/host_execution.py b/roar/backends/osmo/host_execution.py new file mode 100644 index 00000000..6723ef1d --- /dev/null +++ b/roar/backends/osmo/host_execution.py @@ -0,0 +1,1646 @@ +from __future__ import annotations + +import json +import os +import re +import shlex +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml # type: ignore[import-untyped] + +from roar.backends.osmo.config import load_osmo_backend_config +from roar.backends.osmo.lineage import ( + OsmoLineageReconstitutionResult, + discover_downloaded_lineage_bundles, + reconstitute_osmo_lineage_bundles, +) +from roar.backends.osmo.workflow import ( + prepare_osmo_workflow_for_lineage, + resolve_roar_install_requirement, +) +from roar.core.bootstrap import bootstrap +from roar.core.models.run import RunContext, RunResult +from roar.core.operation_metadata import build_operation_metadata_json +from roar.db.context import create_database_context +from roar.db.hashing import hash_files_blake3 +from roar.execution.recording import LocalJobRecorder, LocalRecordedArtifact, StalenessAnalyzer +from roar.execution.runtime.host_execution import ExecutionSetupError + +_TERMINAL_WORKFLOW_STATUSES = { + "CANCELLED", + "CANCELED", + "COMPLETED", + "FAILED", + "TERMINATED", + "TIMED_OUT", + "TIMEOUT", +} +_OSMO_TEMPLATE_SENTINEL_RE = re.compile(r"__ROAR_OSMO_TEMPLATE_([A-Za-z0-9_.-]+)__") + + +@dataclass(frozen=True) +class OsmoWorkflowWaitResult: + status: str | None = None + payload: dict[str, Any] | None = None + timed_out: bool = False + error: str | None = None + + +@dataclass(frozen=True) +class OsmoSubmitCommandContext: + repo_root: str | None = None + workflow_spec_argument: str | None = None + workflow_spec_path: str | None = None + prepared_workflow_argument: str | None = None + prepared_workflow_path: str | None = None + prepared_wrapped_tasks: list[str] | None = None + prepared_runtime_install_requirement: str | None = None + prepared_runtime_install_local_path: str | None = None + prepared_runtime_install_remote_path: str | None = None + pool: str | None = None + format_type: str | None = None + set_strings: dict[str, str] | None = None + set_files: dict[str, str] | None = None + dataset_hints: list[str] | None = None + task_name_hints: list[str] | None = None + + +@dataclass(frozen=True) +class OsmoDeclaredDatasetOutput: + dataset_name: str + declared_path: str | None = None + task_name: str | None = None + + +@dataclass(frozen=True) +class OsmoOutputDownloadResult: + artifacts: list[LocalRecordedArtifact] = field(default_factory=list) + datasets: list[dict[str, Any]] = field(default_factory=list) + error: str | None = None + + +@dataclass(frozen=True) +class OsmoWorkflowDiagnosticsResult: + artifacts: list[LocalRecordedArtifact] = field(default_factory=list) + query_artifact_path: str | None = None + task_logs: list[dict[str, Any]] = field(default_factory=list) + error: str | None = None + + +@dataclass(frozen=True) +class OsmoAttachOptions: + workflow_spec_argument: str | None = None + workflow_spec_path: str | None = None + set_strings: dict[str, str] | None = None + dataset_names: list[str] | None = None + task_names: list[str] | None = None + wait_for_completion: bool | None = None + download_declared_outputs: bool | None = None + ingest_lineage_bundles: bool | None = None + + +def execute_osmo_workflow_submit(ctx: RunContext) -> RunResult: + """Execute an OSMO workflow submit locally and record it as a Roar job.""" + bootstrap(ctx.roar_dir) + started_at = time.time() + config = load_osmo_backend_config(start_dir=ctx.repo_root) + submit_context = _extract_submit_command_context(ctx.command, ctx.repo_root) + submit_context = _merge_configured_osmo_context_hints( + submit_context, + config=config, + include_lineage_dataset_hint=bool(config.get("ingest_lineage_bundles", False)), + ) + submit_command, submit_context, prepared_workflow_path = _prepare_submit_command( + command=ctx.command, + submit_context=submit_context, + config=config, + ) + + try: + try: + completed = subprocess.run( + submit_command, + cwd=ctx.repo_root, + capture_output=True, + text=True, + check=False, + ) + finally: + if prepared_workflow_path is not None: + prepared_workflow_path.unlink(missing_ok=True) + except FileNotFoundError as exc: + raise ExecutionSetupError( + "Error: osmo CLI not found. Install the OSMO CLI or adjust PATH." + ) from exc + + _emit_captured_output(completed.stdout, sys.stdout) + _emit_captured_output(completed.stderr, sys.stderr) + + workflow_id = _extract_workflow_id(_parse_json_response(completed.stdout)) + wait_result: OsmoWorkflowWaitResult | None = None + final_exit_code = completed.returncode + if completed.returncode == 0 and bool(config.get("wait_for_completion", False)): + wait_result = _wait_for_workflow_completion( + command=ctx.command, + repo_root=ctx.repo_root, + workflow_id=workflow_id, + timeout_seconds=int(config.get("query_timeout_seconds", 12 * 60)), + poll_interval_seconds=float(config.get("poll_interval_seconds", 5.0)), + ) + final_exit_code = _resolve_final_exit_code(completed.returncode, wait_result) + + download_result: OsmoOutputDownloadResult | None = None + if bool(config.get("download_declared_outputs", False)): + download_result = _download_declared_outputs( + osmo_binary=ctx.command[0], + repo_root=ctx.repo_root, + roar_dir=ctx.roar_dir, + submit_context=submit_context, + workflow_id=workflow_id, + wait_result=wait_result, + download_directory=str(config.get("download_directory", ".roar/osmo/downloads")), + ) + if download_result.error and final_exit_code == 0: + final_exit_code = 1 + + diagnostics_result = _capture_workflow_diagnostics( + osmo_binary=ctx.command[0], + repo_root=ctx.repo_root, + roar_dir=ctx.roar_dir, + submit_context=submit_context, + workflow_id=workflow_id, + wait_result=wait_result, + ) + + duration = max(0.0, time.time() - started_at) + input_artifacts = _build_submit_input_artifacts(submit_context) + initial_payload = _build_osmo_submit_payload( + ctx=ctx, + submit_context=submit_context, + started_at=started_at, + duration=duration, + completed=completed, + final_exit_code=final_exit_code, + wait_enabled=bool(config.get("wait_for_completion", False)), + wait_result=wait_result, + download_result=download_result, + diagnostics_result=diagnostics_result, + lineage_result=None, + ) + metadata = build_operation_metadata_json("osmo_submit", initial_payload) + non_receipt_output_artifacts = [ + *(download_result.artifacts if download_result is not None else []), + *diagnostics_result.artifacts, + ] + + with create_database_context(ctx.roar_dir) as db_ctx: + session_id = db_ctx.sessions.get_or_create_active() + recorder = LocalJobRecorder() + job_id, job_uid = recorder.record( + db_ctx, + command=shlex.join(ctx.command), + timestamp=started_at, + metadata=metadata, + execution_backend=ctx.execution_backend, + execution_role=ctx.execution_role, + job_type=ctx.job_type or "run", + input_artifacts=input_artifacts, + output_artifacts=non_receipt_output_artifacts, + duration_seconds=duration, + exit_code=final_exit_code, + session_id=session_id, + ) + job = db_ctx.jobs.get(job_id) + resolved_session_id = ( + int(job["session_id"]) if job and job.get("session_id") else session_id + ) + submit_step_number = int(job["step_number"]) if job and job.get("step_number") else 1 + db_ctx.commit() + + lineage_result = _maybe_reconstitute_downloaded_lineage( + config=config, + download_result=download_result, + repo_root=ctx.repo_root, + roar_dir=ctx.roar_dir, + job_uid=job_uid, + session_id=resolved_session_id, + submit_step_number=submit_step_number, + ) + + final_payload = _build_osmo_submit_payload( + ctx=ctx, + submit_context=submit_context, + started_at=started_at, + duration=duration, + completed=completed, + final_exit_code=final_exit_code, + wait_enabled=bool(config.get("wait_for_completion", False)), + wait_result=wait_result, + download_result=download_result, + diagnostics_result=diagnostics_result, + lineage_result=lineage_result, + ) + receipt_artifact = _write_osmo_submit_receipt( + roar_dir=ctx.roar_dir, + payload=final_payload, + ) + + with create_database_context(ctx.roar_dir) as db_ctx: + _update_recorded_osmo_submit( + db_ctx=db_ctx, + job_id=job_id, + metadata=build_operation_metadata_json("osmo_submit", final_payload), + receipt_artifact=receipt_artifact, + ) + stale_upstream, stale_downstream = StalenessAnalyzer().analyze( + db_ctx, resolved_session_id, job_id + ) + inputs = db_ctx.jobs.get_inputs(job_id) + outputs = db_ctx.jobs.get_outputs(job_id) + + return RunResult( + exit_code=final_exit_code, + job_id=job_id, + job_uid=job_uid, + duration=duration, + inputs=inputs, + outputs=outputs, + interrupted=False, + is_build=ctx.job_type == "build", + stale_upstream=stale_upstream, + stale_downstream=stale_downstream, + ) + + +def attach_osmo_workflow( + *, + roar_dir: Path, + repo_root: str, + workflow_id: str, + options: OsmoAttachOptions | None = None, + osmo_binary: str = "osmo", +) -> RunResult: + """Attach local Roar lineage to an existing OSMO workflow.""" + bootstrap(roar_dir) + started_at = time.time() + config = load_osmo_backend_config(start_dir=repo_root) + effective_options = options or OsmoAttachOptions() + attach_context = _build_attach_context( + repo_root=repo_root, + workflow_spec_argument=effective_options.workflow_spec_argument, + workflow_spec_path=effective_options.workflow_spec_path, + set_strings=effective_options.set_strings, + dataset_names=effective_options.dataset_names, + task_names=effective_options.task_names, + ) + wait_enabled = ( + bool(config.get("wait_for_completion", False)) + if effective_options.wait_for_completion is None + else bool(effective_options.wait_for_completion) + ) + download_enabled = ( + bool(config.get("download_declared_outputs", False)) + if effective_options.download_declared_outputs is None + else bool(effective_options.download_declared_outputs) + ) + ingest_lineage_enabled = ( + bool(config.get("ingest_lineage_bundles", False)) + if effective_options.ingest_lineage_bundles is None + else bool(effective_options.ingest_lineage_bundles) + ) + attach_context = _merge_configured_osmo_context_hints( + attach_context, + config=config, + include_lineage_dataset_hint=ingest_lineage_enabled, + ) + + wait_result = ( + _wait_for_workflow_completion( + command=[osmo_binary], + repo_root=repo_root, + workflow_id=workflow_id, + timeout_seconds=int(config.get("query_timeout_seconds", 12 * 60)), + poll_interval_seconds=float(config.get("poll_interval_seconds", 5.0)), + ) + if wait_enabled + else _query_workflow_state( + osmo_binary=osmo_binary, + repo_root=repo_root, + workflow_id=workflow_id, + ) + ) + final_exit_code = _resolve_attach_exit_code(wait_result) + + download_result: OsmoOutputDownloadResult | None = None + if download_enabled: + download_result = _download_declared_outputs( + osmo_binary=osmo_binary, + repo_root=repo_root, + roar_dir=roar_dir, + submit_context=attach_context, + workflow_id=workflow_id, + wait_result=wait_result, + download_directory=str(config.get("download_directory", ".roar/osmo/downloads")), + ) + if download_result.error and final_exit_code == 0: + final_exit_code = 1 + + diagnostics_result = _capture_workflow_diagnostics( + osmo_binary=osmo_binary, + repo_root=repo_root, + roar_dir=roar_dir, + submit_context=attach_context, + workflow_id=workflow_id, + wait_result=wait_result, + ) + + duration = max(0.0, time.time() - started_at) + input_artifacts = _build_osmo_input_artifacts( + attach_context, + metadata_key="osmo_attach_input", + ) + initial_payload = _build_osmo_attach_payload( + osmo_binary=osmo_binary, + workflow_id=workflow_id, + attach_context=attach_context, + started_at=started_at, + duration=duration, + final_exit_code=final_exit_code, + wait_enabled=wait_enabled, + wait_result=wait_result, + download_enabled=download_enabled, + download_result=download_result, + diagnostics_result=diagnostics_result, + lineage_result=None, + git_commit=_resolve_git_commit(repo_root), + ) + metadata = build_operation_metadata_json("osmo_attach", initial_payload) + non_receipt_output_artifacts = [ + *(download_result.artifacts if download_result is not None else []), + *diagnostics_result.artifacts, + ] + command = [osmo_binary, "workflow", "attach", workflow_id] + + with create_database_context(roar_dir) as db_ctx: + session_id = db_ctx.sessions.get_or_create_active() + recorder = LocalJobRecorder() + job_id, job_uid = recorder.record( + db_ctx, + command=shlex.join(command), + timestamp=started_at, + metadata=metadata, + execution_backend="osmo", + execution_role="attach", + job_type="run", + input_artifacts=input_artifacts, + output_artifacts=non_receipt_output_artifacts, + duration_seconds=duration, + exit_code=final_exit_code, + session_id=session_id, + ) + job = db_ctx.jobs.get(job_id) + resolved_session_id = ( + int(job["session_id"]) if job and job.get("session_id") else session_id + ) + attach_step_number = int(job["step_number"]) if job and job.get("step_number") else 1 + db_ctx.commit() + + lineage_result = _maybe_reconstitute_downloaded_lineage( + config={**config, "ingest_lineage_bundles": ingest_lineage_enabled}, + download_result=download_result, + repo_root=repo_root, + roar_dir=roar_dir, + job_uid=job_uid, + session_id=resolved_session_id, + submit_step_number=attach_step_number, + ) + if lineage_result is not None and lineage_result.error and final_exit_code == 0: + final_exit_code = 1 + + final_payload = _build_osmo_attach_payload( + osmo_binary=osmo_binary, + workflow_id=workflow_id, + attach_context=attach_context, + started_at=started_at, + duration=duration, + final_exit_code=final_exit_code, + wait_enabled=wait_enabled, + wait_result=wait_result, + download_enabled=download_enabled, + download_result=download_result, + diagnostics_result=diagnostics_result, + lineage_result=lineage_result, + git_commit=_resolve_git_commit(repo_root), + ) + receipt_artifact = _write_osmo_attach_receipt( + roar_dir=roar_dir, + payload=final_payload, + ) + + with create_database_context(roar_dir) as db_ctx: + _update_recorded_osmo_submit( + db_ctx=db_ctx, + job_id=job_id, + metadata=build_operation_metadata_json("osmo_attach", final_payload), + receipt_artifact=receipt_artifact, + ) + stale_upstream, stale_downstream = StalenessAnalyzer().analyze( + db_ctx, resolved_session_id, job_id + ) + inputs = db_ctx.jobs.get_inputs(job_id) + outputs = db_ctx.jobs.get_outputs(job_id) + + return RunResult( + exit_code=final_exit_code, + job_id=job_id, + job_uid=job_uid, + duration=duration, + inputs=inputs, + outputs=outputs, + interrupted=False, + is_build=False, + stale_upstream=stale_upstream, + stale_downstream=stale_downstream, + ) + + +def _emit_captured_output(text: str, stream: Any) -> None: + if not text: + return + stream.write(text) + if not text.endswith("\n"): + stream.write("\n") + stream.flush() + + +def _build_osmo_submit_payload( + *, + ctx: RunContext, + submit_context: OsmoSubmitCommandContext, + started_at: float, + duration: float, + completed: subprocess.CompletedProcess[str], + final_exit_code: int, + wait_enabled: bool, + wait_result: OsmoWorkflowWaitResult | None, + download_result: OsmoOutputDownloadResult | None, + diagnostics_result: OsmoWorkflowDiagnosticsResult, + lineage_result: OsmoLineageReconstitutionResult | None, +) -> dict[str, Any]: + parsed_response = _parse_json_response(completed.stdout) + workflow_id = _extract_workflow_id(parsed_response) + + payload: dict[str, Any] = { + "command": list(ctx.command), + "command_string": shlex.join(ctx.command), + "workflow_id": workflow_id, + "response_format": "json" if parsed_response is not None else "text", + "return_code": final_exit_code, + "submit_return_code": completed.returncode, + "duration_seconds": duration, + "timestamp": started_at, + "git_commit": _resolve_git_commit(ctx.repo_root), + "wait_for_completion": wait_enabled, + } + submit_payload = _build_submit_context_payload(submit_context) + if submit_payload: + payload["submit"] = submit_payload + if parsed_response is not None: + payload["response"] = parsed_response + else: + payload["stdout"] = _truncate_output(completed.stdout) + if completed.stderr.strip(): + payload["stderr"] = _truncate_output(completed.stderr) + if wait_result is not None: + payload["workflow_status"] = wait_result.status + if wait_result.payload is not None: + payload["workflow_query"] = wait_result.payload + if wait_result.timed_out: + payload["workflow_query_timed_out"] = True + if wait_result.error: + payload["workflow_query_error"] = wait_result.error + if download_result is not None: + payload["download_declared_outputs"] = True + if download_result.datasets: + payload["downloaded_outputs"] = list(download_result.datasets) + if download_result.error: + payload["download_error"] = download_result.error + if ( + diagnostics_result.query_artifact_path + or diagnostics_result.task_logs + or diagnostics_result.error + ): + diagnostics_payload: dict[str, Any] = {} + if diagnostics_result.query_artifact_path: + diagnostics_payload["query_artifact_path"] = diagnostics_result.query_artifact_path + if diagnostics_result.task_logs: + diagnostics_payload["task_logs"] = list(diagnostics_result.task_logs) + if diagnostics_result.error: + diagnostics_payload["error"] = diagnostics_result.error + payload["workflow_diagnostics"] = diagnostics_payload + if lineage_result is not None: + lineage_payload: dict[str, Any] = { + "bundle_count": len(lineage_result.bundles), + "fragments_processed": lineage_result.fragments_processed, + "jobs_merged": lineage_result.jobs_merged, + "artifacts_merged": lineage_result.artifacts_merged, + } + if lineage_result.bundles: + lineage_payload["bundles"] = list(lineage_result.bundles) + if lineage_result.error: + lineage_payload["error"] = lineage_result.error + payload["lineage_reconstitution"] = lineage_payload + + return payload + + +def _build_osmo_attach_payload( + *, + osmo_binary: str, + workflow_id: str, + attach_context: OsmoSubmitCommandContext, + started_at: float, + duration: float, + final_exit_code: int, + wait_enabled: bool, + wait_result: OsmoWorkflowWaitResult | None, + download_enabled: bool, + download_result: OsmoOutputDownloadResult | None, + diagnostics_result: OsmoWorkflowDiagnosticsResult, + lineage_result: OsmoLineageReconstitutionResult | None, + git_commit: str | None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "command": [osmo_binary, "workflow", "attach", workflow_id], + "command_string": shlex.join([osmo_binary, "workflow", "attach", workflow_id]), + "workflow_id": workflow_id, + "return_code": final_exit_code, + "duration_seconds": duration, + "timestamp": started_at, + "git_commit": git_commit, + "wait_for_completion": wait_enabled, + } + attach_payload = _build_submit_context_payload(attach_context) + if attach_payload: + payload["attach"] = attach_payload + if wait_result is not None: + payload["workflow_status"] = wait_result.status + if wait_result.payload is not None: + payload["workflow_query"] = wait_result.payload + if wait_result.timed_out: + payload["workflow_query_timed_out"] = True + if wait_result.error: + payload["workflow_query_error"] = wait_result.error + if download_enabled: + payload["download_declared_outputs"] = True + if download_result is not None: + if download_result.datasets: + payload["downloaded_outputs"] = list(download_result.datasets) + if download_result.error: + payload["download_error"] = download_result.error + if ( + diagnostics_result.query_artifact_path + or diagnostics_result.task_logs + or diagnostics_result.error + ): + diagnostics_payload: dict[str, Any] = {} + if diagnostics_result.query_artifact_path: + diagnostics_payload["query_artifact_path"] = diagnostics_result.query_artifact_path + if diagnostics_result.task_logs: + diagnostics_payload["task_logs"] = list(diagnostics_result.task_logs) + if diagnostics_result.error: + diagnostics_payload["error"] = diagnostics_result.error + payload["workflow_diagnostics"] = diagnostics_payload + if lineage_result is not None: + lineage_payload: dict[str, Any] = { + "bundle_count": len(lineage_result.bundles), + "fragments_processed": lineage_result.fragments_processed, + "jobs_merged": lineage_result.jobs_merged, + "artifacts_merged": lineage_result.artifacts_merged, + } + if lineage_result.bundles: + lineage_payload["bundles"] = list(lineage_result.bundles) + if lineage_result.error: + lineage_payload["error"] = lineage_result.error + payload["lineage_reconstitution"] = lineage_payload + return payload + + +def _parse_json_response(stdout: str) -> dict[str, Any] | None: + text = stdout.strip() + if not text: + return None + try: + parsed = json.loads(text) + except json.JSONDecodeError: + return None + return parsed if isinstance(parsed, dict) else None + + +def _extract_submit_command_context(command: list[str], repo_root: str) -> OsmoSubmitCommandContext: + workflow_spec_argument = None + workflow_spec_path = None + if len(command) >= 4 and not command[3].startswith("-"): + workflow_spec_argument = command[3] + workflow_spec_path = _resolve_local_path(workflow_spec_argument, repo_root) + + pool = None + format_type = None + set_strings: dict[str, str] = {} + set_files: dict[str, str] = {} + + i = 4 + while i < len(command): + arg = command[i] + if arg == "--pool" and i + 1 < len(command): + pool = command[i + 1] + i += 2 + continue + if arg.startswith("--pool="): + pool = arg.split("=", 1)[1] + i += 1 + continue + if arg == "--format-type" and i + 1 < len(command): + format_type = command[i + 1] + i += 2 + continue + if arg.startswith("--format-type="): + format_type = arg.split("=", 1)[1] + i += 1 + continue + if arg == "--set-string" and i + 1 < len(command): + _assign_submit_mapping(set_strings, command[i + 1]) + i += 2 + continue + if arg.startswith("--set-string="): + _assign_submit_mapping(set_strings, arg.split("=", 1)[1]) + i += 1 + continue + if arg == "--set-file" and i + 1 < len(command): + _assign_submit_mapping(set_files, command[i + 1]) + i += 2 + continue + if arg.startswith("--set-file="): + _assign_submit_mapping(set_files, arg.split("=", 1)[1]) + i += 1 + continue + i += 1 + + return OsmoSubmitCommandContext( + repo_root=repo_root, + workflow_spec_argument=workflow_spec_argument, + workflow_spec_path=workflow_spec_path, + pool=pool, + format_type=format_type, + set_strings=set_strings or None, + set_files=set_files or None, + ) + + +def _assign_submit_mapping(target: dict[str, str], value: str) -> None: + if "=" not in value: + return + key, mapped_value = value.split("=", 1) + key = key.strip() + mapped_value = mapped_value.strip() + if key and mapped_value: + target[key] = mapped_value + + +def _build_attach_context( + *, + repo_root: str, + workflow_spec_argument: str | None, + workflow_spec_path: str | None, + set_strings: dict[str, str] | None, + dataset_names: list[str] | None = None, + task_names: list[str] | None = None, +) -> OsmoSubmitCommandContext: + resolved_spec_path = workflow_spec_path + if resolved_spec_path: + resolved_spec_path = _resolve_local_path(resolved_spec_path, repo_root) + return OsmoSubmitCommandContext( + repo_root=repo_root, + workflow_spec_argument=workflow_spec_argument, + workflow_spec_path=resolved_spec_path, + set_strings=dict(set_strings) if set_strings else None, + dataset_hints=[item for item in (dataset_names or []) if str(item).strip()] or None, + task_name_hints=[item for item in (task_names or []) if str(item).strip()] or None, + ) + + +def _merge_configured_osmo_context_hints( + context: OsmoSubmitCommandContext, + *, + config: dict[str, Any], + include_lineage_dataset_hint: bool, +) -> OsmoSubmitCommandContext: + dataset_hints = [ + str(item).strip() for item in (context.dataset_hints or []) if str(item).strip() + ] + if include_lineage_dataset_hint: + lineage_dataset_name = str(config.get("lineage_bundle_dataset_name", "")).strip() + if lineage_dataset_name and lineage_dataset_name not in dataset_hints: + dataset_hints.append(lineage_dataset_name) + + if dataset_hints == list(context.dataset_hints or []): + return context + + return OsmoSubmitCommandContext( + repo_root=context.repo_root, + workflow_spec_argument=context.workflow_spec_argument, + workflow_spec_path=context.workflow_spec_path, + prepared_workflow_argument=context.prepared_workflow_argument, + prepared_workflow_path=context.prepared_workflow_path, + prepared_wrapped_tasks=list(context.prepared_wrapped_tasks) + if context.prepared_wrapped_tasks + else None, + prepared_runtime_install_requirement=context.prepared_runtime_install_requirement, + prepared_runtime_install_local_path=context.prepared_runtime_install_local_path, + prepared_runtime_install_remote_path=context.prepared_runtime_install_remote_path, + pool=context.pool, + format_type=context.format_type, + set_strings=dict(context.set_strings) if context.set_strings else None, + set_files=dict(context.set_files) if context.set_files else None, + dataset_hints=dataset_hints or None, + task_name_hints=list(context.task_name_hints) if context.task_name_hints else None, + ) + + +def _build_submit_context_payload(context: OsmoSubmitCommandContext) -> dict[str, Any]: + payload: dict[str, Any] = {} + if context.workflow_spec_argument or context.workflow_spec_path: + payload["workflow_spec"] = { + "argument": context.workflow_spec_argument, + "path": context.workflow_spec_path, + } + if context.prepared_workflow_argument or context.prepared_workflow_path: + prepared_payload: dict[str, Any] = {} + if context.prepared_workflow_argument or context.prepared_workflow_path: + prepared_payload["workflow_spec"] = { + "argument": context.prepared_workflow_argument, + "path": context.prepared_workflow_path, + } + if context.prepared_wrapped_tasks: + prepared_payload["wrapped_tasks"] = list(context.prepared_wrapped_tasks) + if context.prepared_runtime_install_requirement: + prepared_payload["runtime_install_requirement"] = ( + context.prepared_runtime_install_requirement + ) + if context.prepared_runtime_install_local_path: + prepared_payload["runtime_install_local_path"] = ( + context.prepared_runtime_install_local_path + ) + if context.prepared_runtime_install_remote_path: + prepared_payload["runtime_install_remote_path"] = ( + context.prepared_runtime_install_remote_path + ) + payload["prepared_workflow"] = prepared_payload + if context.pool: + payload["pool"] = context.pool + if context.format_type: + payload["format_type"] = context.format_type + if context.set_strings: + payload["set_strings"] = dict(context.set_strings) + if context.set_files: + payload["set_files"] = dict(context.set_files) + if context.dataset_hints: + payload["dataset_hints"] = list(context.dataset_hints) + if context.task_name_hints: + payload["task_name_hints"] = list(context.task_name_hints) + return payload + + +def _prepare_submit_command( + *, + command: list[str], + submit_context: OsmoSubmitCommandContext, + config: dict[str, Any], +) -> tuple[list[str], OsmoSubmitCommandContext, Path | None]: + if not bool(config.get("auto_prepare_submissions", True)): + return list(command), submit_context, None + if not submit_context.workflow_spec_path: + return list(command), submit_context, None + + workflow_spec_path = Path(submit_context.workflow_spec_path) + if not workflow_spec_path.is_file(): + return list(command), submit_context, None + + temp_output_path = _create_prepared_workflow_temp_path(workflow_spec_path) + runtime_install_local_path = _resolve_local_path( + str(config.get("runtime_install_local_path") or ""), + submit_context.repo_root, + ) + runtime_install_remote_path = ( + str(config.get("runtime_install_remote_path", "/tmp/roar-osmo-install.whl")).strip() + or "/tmp/roar-osmo-install.whl" + ) + runtime_install_requirement = ( + None + if runtime_install_local_path + else resolve_roar_install_requirement( + _coerce_optional_text(config.get("runtime_install_requirement")) + ) + ) + + try: + prepared = prepare_osmo_workflow_for_lineage( + input_path=workflow_spec_path, + output_path=temp_output_path, + lineage_dataset_name=str( + config.get("lineage_bundle_dataset_name", "roar-lineage") + ).strip() + or "roar-lineage", + lineage_bundle_filename=str( + config.get("lineage_bundle_filename", "roar-fragments.json") + ).strip() + or "roar-fragments.json", + inject_runtime_wrapper=True, + runtime_install_requirement=runtime_install_requirement, + runtime_install_local_path=runtime_install_local_path, + runtime_install_remote_path=runtime_install_remote_path, + task_names=None, + default_to_all_tasks=True, + ) + except ValueError as exc: + temp_output_path.unlink(missing_ok=True) + raise ExecutionSetupError( + f"Error preparing OSMO workflow for Roar instrumentation: {exc}" + ) from exc + + rewritten_command = list(command) + rewritten_argument = str(temp_output_path) + if len(rewritten_command) >= 4: + rewritten_command[3] = rewritten_argument + + rewritten_context = OsmoSubmitCommandContext( + repo_root=submit_context.repo_root, + workflow_spec_argument=submit_context.workflow_spec_argument, + workflow_spec_path=submit_context.workflow_spec_path, + prepared_workflow_argument=rewritten_argument, + prepared_workflow_path=str(temp_output_path), + prepared_wrapped_tasks=list(prepared.wrapped_tasks), + prepared_runtime_install_requirement=runtime_install_requirement, + prepared_runtime_install_local_path=runtime_install_local_path, + prepared_runtime_install_remote_path=( + runtime_install_remote_path if runtime_install_local_path else None + ), + pool=submit_context.pool, + format_type=submit_context.format_type, + set_strings=dict(submit_context.set_strings) if submit_context.set_strings else None, + set_files=dict(submit_context.set_files) if submit_context.set_files else None, + dataset_hints=list(submit_context.dataset_hints) if submit_context.dataset_hints else None, + task_name_hints=list(submit_context.task_name_hints) + if submit_context.task_name_hints + else None, + ) + return rewritten_command, rewritten_context, temp_output_path + + +def _create_prepared_workflow_temp_path(workflow_spec_path: Path) -> Path: + suffix = workflow_spec_path.suffix or ".yaml" + file_descriptor, temp_path = tempfile.mkstemp( + prefix=f".{workflow_spec_path.stem}.roar-osmo-", + suffix=suffix, + dir=workflow_spec_path.parent, + text=True, + ) + os.close(file_descriptor) + Path(temp_path).unlink(missing_ok=True) + return Path(temp_path) + + +def _coerce_optional_text(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _build_osmo_input_artifacts( + context: OsmoSubmitCommandContext, + *, + metadata_key: str, +) -> list[LocalRecordedArtifact]: + artifact_specs: list[tuple[Path, dict[str, Any]]] = [] + + if context.workflow_spec_path: + workflow_path = Path(context.workflow_spec_path) + if workflow_path.is_file(): + artifact_specs.append( + ( + workflow_path, + { + metadata_key: { + "role": "workflow_spec", + "argument": context.workflow_spec_argument, + } + }, + ) + ) + + for key, raw_value in (context.set_files or {}).items(): + resolved = _resolve_local_path(raw_value, context.repo_root) + if resolved is None: + continue + file_path = Path(resolved) + if not file_path.is_file(): + continue + artifact_specs.append( + ( + file_path, + { + metadata_key: { + "role": "set_file", + "key": key, + "argument": raw_value, + } + }, + ) + ) + + deduped_paths: dict[str, tuple[Path, dict[str, Any]]] = {} + for path, metadata in artifact_specs: + deduped_paths[str(path)] = (path, metadata) + + hashes = hash_files_blake3([path for path, _metadata in deduped_paths.values()]) + artifacts: list[LocalRecordedArtifact] = [] + for path_str, (path, metadata) in deduped_paths.items(): + digest = hashes.get(path_str) + if not digest: + continue + artifacts.append( + LocalRecordedArtifact( + path=path_str, + hashes={"blake3": digest}, + size=path.stat().st_size, + metadata=json.dumps(metadata), + ) + ) + return artifacts + + +def _build_submit_input_artifacts( + context: OsmoSubmitCommandContext, +) -> list[LocalRecordedArtifact]: + return _build_osmo_input_artifacts(context, metadata_key="osmo_submit_input") + + +def _resolve_local_path(value: str, base_dir: str | None) -> str | None: + text = str(value or "").strip() + if not text or "://" in text: + return None + path = Path(text) + if not path.is_absolute(): + if not base_dir: + return None + path = Path(base_dir) / path + return str(path.resolve()) + + +def _maybe_reconstitute_downloaded_lineage( + *, + config: dict[str, Any], + download_result: OsmoOutputDownloadResult | None, + repo_root: str, + roar_dir: Path, + job_uid: str, + session_id: int | None, + submit_step_number: int, +) -> OsmoLineageReconstitutionResult | None: + if not bool(config.get("ingest_lineage_bundles", False)): + return None + if not bool(config.get("download_declared_outputs", False)): + message = ( + "osmo.ingest_lineage_bundles requires osmo.download_declared_outputs = true " + "so Roar can inspect downloaded output datasets for lineage bundles." + ) + _emit_captured_output(f"[roar] {message}", sys.stderr) + return OsmoLineageReconstitutionResult(error=message) + if download_result is None or download_result.error: + return OsmoLineageReconstitutionResult( + error=download_result.error + if download_result is not None + else "no downloaded outputs available" + ) + + bundle_filename = str(config.get("lineage_bundle_filename", "roar-fragments.json")) + try: + bundles = discover_downloaded_lineage_bundles( + download_result.datasets, + bundle_filename=bundle_filename, + ) + except ValueError as exc: + message = str(exc) + _emit_captured_output(f"[roar] {message}", sys.stderr) + return OsmoLineageReconstitutionResult(error=message) + + if not bundles: + return OsmoLineageReconstitutionResult() + + result = reconstitute_osmo_lineage_bundles( + bundles=bundles, + project_dir=repo_root, + roar_db_path=roar_dir / "roar.db", + driver_job_uid=job_uid, + session_id=session_id, + step_number=submit_step_number, + ) + if result.error: + _emit_captured_output(f"[roar] {result.error}", sys.stderr) + elif result.fragments_processed: + _emit_captured_output( + "[roar] OSMO lineage reconstituted: " + f"{result.jobs_merged} jobs, {result.artifacts_merged} artifacts", + sys.stderr, + ) + return result + + +def _download_declared_outputs( + *, + osmo_binary: str, + repo_root: str, + roar_dir: Path, + submit_context: OsmoSubmitCommandContext, + workflow_id: str | None, + wait_result: OsmoWorkflowWaitResult | None, + download_directory: str, +) -> OsmoOutputDownloadResult: + if wait_result is None: + message = ( + "osmo.download_declared_outputs requires a successful workflow query " + "so Roar can resolve terminal workflow completion before downloading outputs." + ) + _emit_captured_output(f"[roar] {message}", sys.stderr) + return OsmoOutputDownloadResult(error=message) + if wait_result is None or wait_result.status != "COMPLETED": + return OsmoOutputDownloadResult(error="workflow did not complete successfully") + + declared_outputs = _resolve_declared_dataset_outputs(submit_context) + if not declared_outputs: + return OsmoOutputDownloadResult() + + base_dir = Path(download_directory) + if not base_dir.is_absolute(): + base_dir = Path(repo_root) / base_dir + workflow_dir = base_dir / (_sanitize_receipt_component(str(workflow_id or "")) or "submit") + workflow_dir.mkdir(parents=True, exist_ok=True) + + artifacts: list[LocalRecordedArtifact] = [] + datasets: list[dict[str, Any]] = [] + for declared in declared_outputs: + dataset_ref = f"{declared.dataset_name}:latest" + dataset_dir = workflow_dir / ( + _sanitize_receipt_component(declared.dataset_name) or "dataset" + ) + dataset_dir.mkdir(parents=True, exist_ok=True) + result = subprocess.run( + [ + osmo_binary, + "dataset", + "download", + dataset_ref, + str(dataset_dir), + ], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + details = _format_query_failure(result) + message = f"failed to download declared dataset output {dataset_ref}: {details}" + _emit_captured_output(f"[roar] {message}", sys.stderr) + return OsmoOutputDownloadResult(artifacts=artifacts, datasets=datasets, error=message) + + dataset_files = [path for path in dataset_dir.rglob("*") if path.is_file()] + hashes = hash_files_blake3(dataset_files) + for file_path in dataset_files: + digest = hashes.get(str(file_path)) + if not digest: + continue + artifacts.append( + LocalRecordedArtifact( + path=str(file_path), + hashes={"blake3": digest}, + size=file_path.stat().st_size, + source_type="osmo_dataset", + source_url=dataset_ref, + metadata=json.dumps( + { + "osmo_dataset_download": { + "dataset_name": declared.dataset_name, + "dataset_ref": dataset_ref, + "declared_path": declared.declared_path, + "task_name": declared.task_name, + } + } + ), + ) + ) + + datasets.append( + { + "dataset_name": declared.dataset_name, + "dataset_ref": dataset_ref, + "local_directory": str(dataset_dir), + "file_count": len(dataset_files), + "declared_path": declared.declared_path, + "task_name": declared.task_name, + } + ) + + return OsmoOutputDownloadResult(artifacts=artifacts, datasets=datasets) + + +def _capture_workflow_diagnostics( + *, + osmo_binary: str, + repo_root: str, + roar_dir: Path, + submit_context: OsmoSubmitCommandContext, + workflow_id: str | None, + wait_result: OsmoWorkflowWaitResult | None, +) -> OsmoWorkflowDiagnosticsResult: + if not workflow_id or wait_result is None or wait_result.payload is None: + return OsmoWorkflowDiagnosticsResult() + + diagnostics_dir = ( + roar_dir / "osmo" / "diagnostics" / (_sanitize_receipt_component(workflow_id) or "submit") + ) + diagnostics_dir.mkdir(parents=True, exist_ok=True) + + artifacts: list[LocalRecordedArtifact] = [] + task_logs: list[dict[str, Any]] = [] + query_status = _sanitize_receipt_component(str(wait_result.status or "")) or "latest" + query_path = diagnostics_dir / f"query-{query_status}.json" + query_json = json.dumps(wait_result.payload, indent=2, sort_keys=True) + query_path.write_text(f"{query_json}\n", encoding="utf-8") + artifacts.append( + _build_file_artifact( + query_path, + source_type="osmo_workflow_query", + metadata={ + "osmo_workflow_query": { + "workflow_id": workflow_id, + "workflow_status": wait_result.status, + } + }, + ) + ) + + error: str | None = None + if ( + wait_result.status + and wait_result.status != "COMPLETED" + and _is_terminal_workflow_status(wait_result.status) + ): + for task_name in _resolve_workflow_task_names(submit_context): + result = subprocess.run( + [osmo_binary, "workflow", "logs", workflow_id, "--task", task_name], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + log_path = ( + diagnostics_dir + / "tasks" + / f"{_sanitize_receipt_component(task_name) or 'task'}.log" + ) + log_path.parent.mkdir(parents=True, exist_ok=True) + log_text = result.stdout + if result.stderr: + if log_text and not log_text.endswith("\n"): + log_text += "\n" + log_text += result.stderr + log_path.write_text(log_text, encoding="utf-8") + artifacts.append( + _build_file_artifact( + log_path, + source_type="osmo_workflow_log", + metadata={ + "osmo_workflow_log": { + "workflow_id": workflow_id, + "workflow_status": wait_result.status, + "task_name": task_name, + "return_code": result.returncode, + } + }, + ) + ) + task_logs.append( + { + "task_name": task_name, + "path": str(log_path), + "return_code": result.returncode, + } + ) + if result.returncode != 0 and error is None: + error = f"failed to capture logs for task {task_name}" + + return OsmoWorkflowDiagnosticsResult( + artifacts=artifacts, + query_artifact_path=str(query_path), + task_logs=task_logs, + error=error, + ) + + +def _resolve_declared_dataset_outputs( + context: OsmoSubmitCommandContext, +) -> list[OsmoDeclaredDatasetOutput]: + outputs: list[OsmoDeclaredDatasetOutput] = [] + loaded = _load_workflow_spec_data(context) + if loaded is not None: + parsed, replacements = loaded + workflow = parsed.get("workflow", {}) + if isinstance(workflow, dict): + tasks = workflow.get("tasks", []) + if isinstance(tasks, list): + for task in tasks: + if not isinstance(task, dict): + continue + task_name = str(task.get("name") or "").strip() or None + task_outputs = task.get("outputs", []) + if not isinstance(task_outputs, list): + continue + for item in task_outputs: + if not isinstance(item, dict): + continue + dataset = item.get("dataset") + if not isinstance(dataset, dict): + continue + dataset_name = _render_submit_template(dataset.get("name"), replacements) + if not dataset_name: + continue + declared_path = _render_submit_template(dataset.get("path"), replacements) + outputs.append( + OsmoDeclaredDatasetOutput( + dataset_name=dataset_name, + declared_path=declared_path, + task_name=task_name, + ) + ) + + merged = _merge_declared_dataset_output_hints(outputs, context.dataset_hints) + return merged + + +def _merge_declared_dataset_output_hints( + outputs: list[OsmoDeclaredDatasetOutput], + hints: list[str] | None, +) -> list[OsmoDeclaredDatasetOutput]: + merged: list[OsmoDeclaredDatasetOutput] = list(outputs) + seen_dataset_names = { + str(item.dataset_name).strip() for item in outputs if str(item.dataset_name).strip() + } + for dataset_name in hints or []: + normalized = str(dataset_name or "").strip() + if not normalized or normalized in seen_dataset_names: + continue + merged.append(OsmoDeclaredDatasetOutput(dataset_name=normalized)) + seen_dataset_names.add(normalized) + return merged + + +def _resolve_workflow_task_names(context: OsmoSubmitCommandContext) -> list[str]: + task_names: list[str] = [] + loaded = _load_workflow_spec_data(context) + if loaded is not None: + parsed, replacements = loaded + workflow = parsed.get("workflow", {}) + if isinstance(workflow, dict): + tasks = workflow.get("tasks", []) + if isinstance(tasks, list): + for task in tasks: + if not isinstance(task, dict): + continue + task_name = _render_submit_template(task.get("name"), replacements) + if task_name: + task_names.append(task_name) + + return _merge_workflow_task_name_hints(task_names, context.task_name_hints) + + +def _merge_workflow_task_name_hints( + task_names: list[str], + hints: list[str] | None, +) -> list[str]: + merged: list[str] = [] + seen: set[str] = set() + + for task_name in [*task_names, *(hints or [])]: + normalized = str(task_name or "").strip() + if not normalized or normalized in seen: + continue + merged.append(normalized) + seen.add(normalized) + + return merged + + +def _load_workflow_spec_data( + context: OsmoSubmitCommandContext, +) -> tuple[dict[str, Any], dict[str, str]] | None: + workflow_spec_path = str(context.workflow_spec_path or "").strip() + if not workflow_spec_path: + return None + + raw_text = Path(workflow_spec_path).read_text(encoding="utf-8") + normalized_text = re.sub( + r":\s*{{\s*([A-Za-z0-9_.-]+)\s*}}(\s*(#.*)?)$", + r': "__ROAR_OSMO_TEMPLATE_\1__"\2', + raw_text, + flags=re.MULTILINE, + ) + parsed = yaml.safe_load(normalized_text) + if not isinstance(parsed, dict): + return None + + default_values = parsed.get("default-values", {}) + if not isinstance(default_values, dict): + default_values = {} + replacements: dict[str, str] = { + str(key): str(value) + for key, value in default_values.items() + if value is not None and not isinstance(value, (dict, list)) + } + for key, value in (context.set_strings or {}).items(): + replacements[str(key)] = str(value) + return parsed, replacements + + +def _render_submit_template(value: Any, replacements: dict[str, str]) -> str | None: + if value is None: + return None + text = str(value) + rendered = _OSMO_TEMPLATE_SENTINEL_RE.sub( + lambda match: replacements.get(match.group(1), match.group(0)), + text, + ) + rendered = re.sub( + r"{{\s*([A-Za-z0-9_.-]+)\s*}}", + lambda match: replacements.get(match.group(1), match.group(0)), + rendered, + ).strip() + if "{{" in rendered or "}}" in rendered: + return None + if _OSMO_TEMPLATE_SENTINEL_RE.search(rendered): + return None + return rendered or None + + +def _build_file_artifact( + path: Path, + *, + source_type: str | None = None, + source_url: str | None = None, + metadata: dict[str, Any] | None = None, +) -> LocalRecordedArtifact: + hashes = hash_files_blake3([path]) + return LocalRecordedArtifact( + path=str(path), + hashes={"blake3": hashes[str(path)]}, + size=path.stat().st_size, + source_type=source_type, + source_url=source_url, + metadata=json.dumps(metadata) if metadata is not None else None, + ) + + +def _update_recorded_osmo_submit( + *, + db_ctx: Any, + job_id: int, + metadata: str | None, + receipt_artifact: LocalRecordedArtifact, +) -> None: + if metadata is not None: + db_ctx.jobs.update_metadata(job_id, metadata) + + artifact_id, _created = db_ctx.artifacts.register( + hashes=receipt_artifact.hashes, + size=receipt_artifact.size, + path=receipt_artifact.path, + source_type=receipt_artifact.source_type, + source_url=receipt_artifact.source_url, + metadata=receipt_artifact.metadata, + ) + db_ctx.jobs.add_output(job_id, artifact_id, receipt_artifact.path) + + +def _extract_workflow_id(payload: dict[str, Any] | None) -> str | None: + if payload is None: + return None + for key in ("name", "workflow_id", "id", "workflowUuid", "workflow_uuid"): + value = payload.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _resolve_git_commit(repo_root: str) -> str | None: + try: + return subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + stderr=subprocess.DEVNULL, + text=True, + ).strip() + except (FileNotFoundError, subprocess.CalledProcessError): + return None + + +def _query_workflow_state( + *, + osmo_binary: str, + repo_root: str, + workflow_id: str | None, +) -> OsmoWorkflowWaitResult: + if not workflow_id: + return OsmoWorkflowWaitResult(error="missing workflow id") + + result = subprocess.run( + [osmo_binary, "workflow", "query", workflow_id, "--format-type", "json"], + cwd=repo_root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return OsmoWorkflowWaitResult(error=_format_query_failure(result)) + + payload = _parse_json_response(result.stdout) + if payload is None: + return OsmoWorkflowWaitResult(error="OSMO workflow query did not return JSON output") + + status = str(payload.get("status") or "").strip().upper() or None + return OsmoWorkflowWaitResult(status=status, payload=payload) + + +def _wait_for_workflow_completion( + *, + command: list[str], + repo_root: str, + workflow_id: str | None, + timeout_seconds: int, + poll_interval_seconds: float, +) -> OsmoWorkflowWaitResult: + if not workflow_id: + _emit_captured_output( + "[roar] OSMO submit did not return a workflow id; cannot wait for completion.", + sys.stderr, + ) + return OsmoWorkflowWaitResult(error="missing workflow id") + + _emit_captured_output( + f"[roar] waiting for OSMO workflow {workflow_id} to reach a terminal state...", + sys.stderr, + ) + deadline = time.monotonic() + timeout_seconds + last_error: str | None = None + + while time.monotonic() < deadline: + query_result = _query_workflow_state( + osmo_binary=command[0], + repo_root=repo_root, + workflow_id=workflow_id, + ) + if query_result.error: + last_error = query_result.error + elif _is_terminal_workflow_status(str(query_result.status or "")): + _emit_captured_output( + f"[roar] OSMO workflow {workflow_id} finished with status {query_result.status}.", + sys.stderr, + ) + return query_result + time.sleep(poll_interval_seconds) + + message = f"[roar] timed out waiting for OSMO workflow {workflow_id} after {timeout_seconds}s." + if last_error: + message = f"{message} last_error={last_error}" + _emit_captured_output(message, sys.stderr) + return OsmoWorkflowWaitResult(timed_out=True, error=last_error) + + +def _resolve_final_exit_code(submit_return_code: int, wait_result: OsmoWorkflowWaitResult) -> int: + if submit_return_code != 0: + return submit_return_code + if wait_result.timed_out or wait_result.error: + return 1 + if wait_result.status and wait_result.status != "COMPLETED": + return 1 + return 0 + + +def _resolve_attach_exit_code(wait_result: OsmoWorkflowWaitResult) -> int: + if wait_result.timed_out or wait_result.error: + return 1 + return 0 + + +def _format_query_failure(result: subprocess.CompletedProcess[str]) -> str: + stderr = _truncate_output(result.stderr) + stdout = _truncate_output(result.stdout) + details = stderr or stdout or "unknown error" + return f"query rc={result.returncode}: {details}" + + +def _is_terminal_workflow_status(status: str) -> bool: + normalized = str(status or "").strip().upper() + return normalized in _TERMINAL_WORKFLOW_STATUSES or normalized.startswith("FAILED") + + +def _truncate_output(text: str, limit: int = 2000) -> str: + stripped = text.strip() + if len(stripped) <= limit: + return stripped + return f"{stripped[:limit]}..." + + +def _write_osmo_submit_receipt( + *, + roar_dir: Path, + payload: dict[str, Any], +) -> LocalRecordedArtifact: + return _write_osmo_workflow_receipt( + roar_dir=roar_dir, + payload=payload, + receipt_dir_name="submissions", + payload_key="osmo_submit", + ) + + +def _write_osmo_attach_receipt( + *, + roar_dir: Path, + payload: dict[str, Any], +) -> LocalRecordedArtifact: + return _write_osmo_workflow_receipt( + roar_dir=roar_dir, + payload=payload, + receipt_dir_name="attachments", + payload_key="osmo_attach", + ) + + +def _write_osmo_workflow_receipt( + *, + roar_dir: Path, + payload: dict[str, Any], + receipt_dir_name: str, + payload_key: str, +) -> LocalRecordedArtifact: + receipt_dir = roar_dir / "osmo" / receipt_dir_name + receipt_dir.mkdir(parents=True, exist_ok=True) + workflow_id = str(payload.get("workflow_id") or "").strip() + status = str(payload.get("workflow_status") or "").strip() + filename_parts = [ + _sanitize_receipt_component(workflow_id) or "submit", + ] + if status: + filename_parts.append(_sanitize_receipt_component(status)) + receipt_path = receipt_dir / f"{'-'.join(filename_parts)}.json" + receipt_json = json.dumps( + {payload_key: payload}, + indent=2, + sort_keys=True, + ) + receipt_path.write_text(f"{receipt_json}\n", encoding="utf-8") + hashes = hash_files_blake3([receipt_path]) + return LocalRecordedArtifact( + path=str(receipt_path), + hashes={"blake3": hashes[str(receipt_path)]}, + size=receipt_path.stat().st_size, + source_type="osmo_workflow_receipt", + metadata=json.dumps( + { + "osmo_workflow_receipt": { + "workflow_id": workflow_id or None, + "workflow_status": status or None, + "receipt_dir": receipt_dir_name, + } + } + ), + ) + + +def _sanitize_receipt_component(value: str) -> str: + normalized = re.sub(r"[^A-Za-z0-9._-]+", "-", value.strip()) + normalized = normalized.strip("-._") + return normalized[:120] + + +__all__ = [ + "OsmoAttachOptions", + "attach_osmo_workflow", + "execute_osmo_workflow_submit", +] diff --git a/roar/backends/osmo/lineage.py b/roar/backends/osmo/lineage.py new file mode 100644 index 00000000..af7d8814 --- /dev/null +++ b/roar/backends/osmo/lineage.py @@ -0,0 +1,315 @@ +from __future__ import annotations + +import json +import sqlite3 +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from roar.execution.fragments.lineage import FragmentLineageBackend, merge_execution_fragments +from roar.execution.fragments.models import ExecutionFragment, derive_fragment_identity + + +@dataclass(frozen=True) +class OsmoLineageBundle: + path: str + dataset_name: str | None = None + declared_path: str | None = None + task_name: str | None = None + + +@dataclass(frozen=True) +class OsmoLineageReconstitutionResult: + jobs_merged: int = 0 + artifacts_merged: int = 0 + fragments_processed: int = 0 + bundles: list[dict[str, Any]] = field(default_factory=list) + error: str | None = None + + +OSMO_FRAGMENT_LINEAGE_BACKEND = FragmentLineageBackend( + job_type="osmo_task", + command_for_fragment=lambda fragment: _osmo_fragment_command(fragment.task_name), + script_for_fragment=lambda fragment: fragment.task_name or None, + execution_role_from_fragment=lambda fragment, fallback_parent_job_uid: _osmo_fragment_role( + fragment, + fallback_parent_job_uid=fallback_parent_job_uid, + ), + metadata_from_fragment=lambda fragment, fallback_parent_job_uid: _osmo_fragment_metadata( + fragment, + fallback_parent_job_uid=fallback_parent_job_uid, + ), + task_identity_from_metadata=lambda parent_job_uid, job_uid, metadata: derive_fragment_identity( + "osmo", + parent_job_uid, + str(metadata.get("osmo_task_id") or metadata.get("task_id") or ""), + job_uid, + ), +) + + +def reconstitute_osmo_lineage_bundles( + *, + bundles: list[OsmoLineageBundle], + project_dir: str, + roar_db_path: Path, + driver_job_uid: str, + session_id: int | None, + step_number: int, +) -> OsmoLineageReconstitutionResult: + normalized_bundles = [bundle for bundle in bundles if str(bundle.path or "").strip()] + if not normalized_bundles: + return OsmoLineageReconstitutionResult() + + fragments: list[ExecutionFragment] = [] + bundle_rows: list[dict[str, Any]] = [] + for bundle in normalized_bundles: + try: + bundle_fragments = _load_bundle_fragments(Path(bundle.path), project_dir=project_dir) + except Exception as exc: + return OsmoLineageReconstitutionResult( + bundles=_bundle_payloads(normalized_bundles), + error=f"failed to load lineage bundle {bundle.path}: {exc}", + ) + fragments.extend(bundle_fragments) + bundle_rows.append( + { + "path": bundle.path, + "dataset_name": bundle.dataset_name, + "declared_path": bundle.declared_path, + "task_name": bundle.task_name, + "fragments": len(bundle_fragments), + } + ) + + if not fragments: + return OsmoLineageReconstitutionResult(bundles=bundle_rows) + + jobs_before, artifacts_before = _count_local_rows(roar_db_path) + try: + merge_execution_fragments( + fragments=fragments, + project_dir=project_dir, + backend=OSMO_FRAGMENT_LINEAGE_BACKEND, + driver_job_uid=driver_job_uid, + session_id=session_id, + step_number=step_number, + ) + except Exception as exc: + return OsmoLineageReconstitutionResult( + bundles=bundle_rows, + fragments_processed=len(fragments), + error=f"failed to merge lineage bundles: {exc}", + ) + + jobs_after, artifacts_after = _count_local_rows(roar_db_path) + return OsmoLineageReconstitutionResult( + jobs_merged=max(0, jobs_after - jobs_before), + artifacts_merged=max(0, artifacts_after - artifacts_before), + fragments_processed=len(fragments), + bundles=bundle_rows, + ) + + +def discover_downloaded_lineage_bundles( + datasets: list[dict[str, Any]], + *, + bundle_filename: str, +) -> list[OsmoLineageBundle]: + target_name = str(bundle_filename or "").strip() + if not target_name: + return [] + + bundles: list[OsmoLineageBundle] = [] + for dataset in datasets: + local_directory = str(dataset.get("local_directory") or "").strip() + if not local_directory: + continue + + declared_path = str(dataset.get("declared_path") or "").strip() or None + candidate_path: Path | None = None + if declared_path and Path(declared_path).name == target_name: + resolved = Path(local_directory) / declared_path + if resolved.is_file(): + candidate_path = resolved + + if candidate_path is None: + matches = sorted(Path(local_directory).rglob(target_name)) + if len(matches) == 1: + candidate_path = matches[0] + elif len(matches) > 1: + raise ValueError( + f"multiple lineage bundle candidates found in {local_directory} for {target_name}" + ) + + if candidate_path is None or not candidate_path.is_file(): + continue + + bundles.append( + OsmoLineageBundle( + path=str(candidate_path), + dataset_name=str(dataset.get("dataset_name") or "").strip() or None, + declared_path=declared_path, + task_name=str(dataset.get("task_name") or "").strip() or None, + ) + ) + + return bundles + + +def _load_bundle_fragments(bundle_path: Path, *, project_dir: str) -> list[ExecutionFragment]: + payload = json.loads(bundle_path.read_text(encoding="utf-8")) + raw_fragments = _extract_raw_fragments(payload) + + fragments: list[ExecutionFragment] = [] + for item in raw_fragments: + normalized = dict(item) + normalized.setdefault("backend", "osmo") + normalized.setdefault("job_uid", "") + normalized.setdefault("parent_job_uid", "") + normalized.setdefault("task_id", "") + normalized.setdefault("worker_id", "") + normalized.setdefault("node_id", "") + normalized.setdefault("actor_id", None) + normalized.setdefault("task_name", "") + normalized.setdefault("started_at", 0.0) + normalized.setdefault("ended_at", 0.0) + normalized.setdefault("exit_code", 0) + normalized["reads"] = _normalize_ref_payloads( + normalized.get("reads"), + project_dir=project_dir, + ) + normalized["writes"] = _normalize_ref_payloads( + normalized.get("writes"), + project_dir=project_dir, + ) + fragments.append(ExecutionFragment.from_dict(normalized)) + return fragments + + +def _extract_raw_fragments(payload: Any) -> list[dict[str, Any]]: + if isinstance(payload, list): + return [item for item in payload if isinstance(item, dict)] + if not isinstance(payload, dict): + raise ValueError("lineage bundle payload must be a JSON object or list") + + direct = payload.get("fragments") + if isinstance(direct, list): + return [item for item in direct if isinstance(item, dict)] + + execution_fragments = payload.get("execution_fragments") + if isinstance(execution_fragments, list): + return [item for item in execution_fragments if isinstance(item, dict)] + + data = payload.get("data") + if isinstance(data, dict): + nested = data.get("fragments") + if isinstance(nested, list): + return [item for item in nested if isinstance(item, dict)] + + raise ValueError("lineage bundle JSON is missing a fragments list") + + +def _normalize_ref_payloads(refs: Any, *, project_dir: str) -> list[dict[str, Any]]: + if not isinstance(refs, list): + return [] + + normalized: list[dict[str, Any]] = [] + for ref in refs: + if not isinstance(ref, dict): + continue + item = dict(ref) + path = str(item.get("path") or "").strip() + if path: + item["path"] = _normalize_fragment_path(path, project_dir=project_dir) + item.setdefault("hash", None) + item.setdefault("hash_algorithm", "blake3") + item["size"] = int(item.get("size") or 0) + item["capture_method"] = str(item.get("capture_method") or "python") + normalized.append(item) + return normalized + + +def _normalize_fragment_path(path: str, *, project_dir: str) -> str: + text = str(path or "").strip() + if not text or "://" in text: + return text + if text.startswith("${ROAR_PROJECT_DIR}/"): + text = text[len("${ROAR_PROJECT_DIR}/") :] + elif text.startswith("$ROAR_PROJECT_DIR/"): + text = text[len("$ROAR_PROJECT_DIR/") :] + + candidate = Path(text) + if candidate.is_absolute(): + return str(candidate) + return str((Path(project_dir) / candidate).resolve()) + + +def _count_local_rows(roar_db_path: Path) -> tuple[int, int]: + if not roar_db_path.exists(): + return 0, 0 + + conn = sqlite3.connect(str(roar_db_path)) + try: + jobs = int(conn.execute("SELECT COUNT(*) FROM jobs").fetchone()[0]) + artifacts = int(conn.execute("SELECT COUNT(*) FROM artifacts").fetchone()[0]) + return jobs, artifacts + finally: + conn.close() + + +def _bundle_payloads(bundles: list[OsmoLineageBundle]) -> list[dict[str, Any]]: + return [ + { + "path": bundle.path, + "dataset_name": bundle.dataset_name, + "declared_path": bundle.declared_path, + "task_name": bundle.task_name, + } + for bundle in bundles + ] + + +def _osmo_fragment_command(task_name: str) -> str: + normalized = str(task_name or "").strip() + return f"osmo_task:{normalized}" if normalized else "osmo_task" + + +def _osmo_fragment_role( + fragment: ExecutionFragment, + *, + fallback_parent_job_uid: str | None, +) -> str: + del fallback_parent_job_uid + execution_role = str(fragment.backend_metadata.get("execution_role") or "").strip() + return execution_role or "task" + + +def _osmo_fragment_metadata( + fragment: ExecutionFragment, + *, + fallback_parent_job_uid: str | None, +) -> dict[str, Any]: + metadata: dict[str, Any] = { + "osmo_task_id": fragment.task_id, + "osmo_worker_id": fragment.worker_id, + "osmo_node_id": fragment.node_id, + } + if fragment.actor_id: + metadata["osmo_actor_id"] = fragment.actor_id + if fallback_parent_job_uid and not fragment.parent_job_uid: + metadata["parent_job_uid"] = fallback_parent_job_uid + + backend_metadata = dict(fragment.backend_metadata or {}) + if backend_metadata: + metadata["backend_metadata"] = backend_metadata + return metadata + + +__all__ = [ + "OSMO_FRAGMENT_LINEAGE_BACKEND", + "OsmoLineageBundle", + "OsmoLineageReconstitutionResult", + "discover_downloaded_lineage_bundles", + "reconstitute_osmo_lineage_bundles", +] diff --git a/roar/backends/osmo/plugin.py b/roar/backends/osmo/plugin.py new file mode 100644 index 00000000..8fef1ebe --- /dev/null +++ b/roar/backends/osmo/plugin.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from roar.backends.osmo.config import OSMO_BACKEND_CONFIG +from roar.backends.osmo.host_execution import execute_osmo_workflow_submit +from roar.backends.osmo.submit import ( + matches_osmo_workflow_submit_command, + plan_osmo_workflow_submit_command, +) +from roar.execution.framework.contract import ( + ExecutionBackend, + ExecutionPolicyAdapter, + HostExecutionAdapter, +) +from roar.execution.framework.registry import register_execution_backend + +OSMO_EXECUTION_BACKEND = ExecutionBackend( + name="osmo", + priority=90, + matches_command=matches_osmo_workflow_submit_command, + plan_command=plan_osmo_workflow_submit_command, + host_execution=HostExecutionAdapter(execute=execute_osmo_workflow_submit), + policy=ExecutionPolicyAdapter( + submit_roles=("submit",), + ), + config=OSMO_BACKEND_CONFIG, +) + + +def register() -> ExecutionBackend: + register_execution_backend(OSMO_EXECUTION_BACKEND) + return OSMO_EXECUTION_BACKEND + + +__all__ = [ + "OSMO_EXECUTION_BACKEND", + "register", +] diff --git a/roar/backends/osmo/runtime_bundle.py b/roar/backends/osmo/runtime_bundle.py new file mode 100644 index 00000000..ed39c694 --- /dev/null +++ b/roar/backends/osmo/runtime_bundle.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import os +import sys +import tarfile +from dataclasses import dataclass +from pathlib import Path + +from roar.execution.runtime import tracer_backends + +_RUNTIME_PYTHON_PREFIX = "python" +_RUNTIME_SITE_PACKAGES_PREFIX = f"{_RUNTIME_PYTHON_PREFIX}/site-packages" +_RUNTIME_BIN_PREFIX = "bin" + + +@dataclass(frozen=True) +class OsmoRuntimeBundle: + output_path: str + roar_package_dir: str + python_roots: list[str] + ptrace_tracer_path: str + + +def build_osmo_runtime_bundle( + *, + output_path: Path, + roar_package_dir: Path | None = None, + python_roots: list[Path] | None = None, + ptrace_tracer_path: Path | None = None, +) -> OsmoRuntimeBundle: + resolved_roar_package_dir = (roar_package_dir or _default_roar_package_dir()).resolve() + if not resolved_roar_package_dir.is_dir(): + raise ValueError(f"Roar package directory not found: {resolved_roar_package_dir}") + + resolved_python_roots = [path.resolve() for path in (python_roots or _default_python_roots())] + if not resolved_python_roots: + raise ValueError("no Python runtime roots were found to stage for the OSMO runtime bundle") + for path in resolved_python_roots: + if not path.is_dir(): + raise ValueError(f"Python runtime root is not a directory: {path}") + + resolved_ptrace_tracer = (ptrace_tracer_path or _default_ptrace_tracer_path()).resolve() + if not resolved_ptrace_tracer.is_file(): + raise ValueError(f"ptrace tracer binary not found: {resolved_ptrace_tracer}") + + output_path.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(output_path, mode="w:gz") as archive: + _add_directory( + archive, + source_dir=resolved_roar_package_dir, + archive_prefix=f"{_RUNTIME_PYTHON_PREFIX}/roar", + ) + for root in resolved_python_roots: + _add_directory( + archive, + source_dir=root, + archive_prefix=_RUNTIME_SITE_PACKAGES_PREFIX, + ) + _add_file( + archive, + source_path=resolved_ptrace_tracer, + archive_path=f"{_RUNTIME_BIN_PREFIX}/roar-tracer", + ) + + return OsmoRuntimeBundle( + output_path=str(output_path), + roar_package_dir=str(resolved_roar_package_dir), + python_roots=[str(path) for path in resolved_python_roots], + ptrace_tracer_path=str(resolved_ptrace_tracer), + ) + + +def _default_roar_package_dir() -> Path: + import roar + + return Path(roar.__file__).resolve().parent + + +def _default_python_roots() -> list[Path]: + roots: list[Path] = [] + seen: set[Path] = set() + for entry in sys.path: + candidate = Path(entry or ".").resolve() + if candidate in seen or not candidate.is_dir(): + continue + if candidate.name not in {"site-packages", "dist-packages"}: + continue + roots.append(candidate) + seen.add(candidate) + return roots + + +def _default_ptrace_tracer_path() -> Path: + import roar + + package_dir = Path(roar.__file__).resolve().parent + tracer_path = tracer_backends.find_ptrace_tracer(package_dir) + if not tracer_path: + raise ValueError( + "ptrace tracer binary could not be resolved from the local Roar environment" + ) + return Path(tracer_path) + + +def _add_directory(archive: tarfile.TarFile, *, source_dir: Path, archive_prefix: str) -> None: + source_root = source_dir.resolve() + for path in sorted(source_root.rglob("*")): + if _should_skip_runtime_path(path): + continue + relative = path.relative_to(source_root) + archive_path = str(Path(archive_prefix, relative.as_posix())) + archive.add(path, arcname=archive_path, recursive=False) + + +def _add_file(archive: tarfile.TarFile, *, source_path: Path, archive_path: str) -> None: + archive.add(source_path, arcname=archive_path, recursive=False) + + +def _should_skip_runtime_path(path: Path) -> bool: + name = path.name + if name == "__pycache__": + return True + if path.is_file() and (name.endswith(".pyc") or name.endswith(".pyo")): + return True + return path.is_symlink() and not os.path.exists(path) + + +__all__ = [ + "OsmoRuntimeBundle", + "build_osmo_runtime_bundle", +] diff --git a/roar/backends/osmo/submit.py b/roar/backends/osmo/submit.py new file mode 100644 index 00000000..b3e08a9a --- /dev/null +++ b/roar/backends/osmo/submit.py @@ -0,0 +1,65 @@ +"""OSMO submit planning through the execution backend framework.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from roar.backends.osmo.config import load_osmo_backend_config +from roar.execution.framework.contract import ExecutionCommandPlan + + +def matches_osmo_workflow_submit_command(command: list[str]) -> bool: + if len(command) < 3: + return False + if not _osmo_backend_enabled(): + return False + + binary = Path(command[0]).name.lower() + noun = command[1].lower() + verb = command[2].lower() + return binary == "osmo" and noun == "workflow" and verb == "submit" + + +def plan_osmo_workflow_submit_command(command: list[str]) -> ExecutionCommandPlan: + if not matches_osmo_workflow_submit_command(command): + return ExecutionCommandPlan(backend_name="osmo", command=list(command)) + + normalized_command = _normalize_osmo_submit_command(command) + return ExecutionCommandPlan( + backend_name="osmo", + command=normalized_command, + execution_role="submit", + ) + + +def _osmo_backend_enabled() -> bool: + start_dir = os.environ.get("ROAR_PROJECT_DIR") or os.getcwd() + return bool(load_osmo_backend_config(start_dir=start_dir).get("enabled", True)) + + +def _normalize_osmo_submit_command(command: list[str]) -> list[str]: + start_dir = os.environ.get("ROAR_PROJECT_DIR") or os.getcwd() + config = load_osmo_backend_config(start_dir=start_dir) + if not bool(config.get("force_json_output", True)): + return list(command) + if _find_format_type_arg(command) is not None: + return list(command) + return [*command, "--format-type", "json"] + + +def _find_format_type_arg(command: list[str]) -> tuple[int, int | None] | None: + for index, arg in enumerate(command): + if arg == "--format-type": + if index + 1 < len(command): + return index, index + 1 + return index, None + if arg.startswith("--format-type="): + return index, None + return None + + +__all__ = [ + "matches_osmo_workflow_submit_command", + "plan_osmo_workflow_submit_command", +] diff --git a/roar/backends/osmo/workflow.py b/roar/backends/osmo/workflow.py new file mode 100644 index 00000000..dad85a46 --- /dev/null +++ b/roar/backends/osmo/workflow.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import importlib.metadata as importlib_metadata +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml # type: ignore[import-untyped] + +_OSMO_TEMPLATE_SENTINEL_RE = re.compile(r"__ROAR_OSMO_TEMPLATE_([A-Za-z0-9_.-]+)__") + + +class _LiteralBlockString(str): + """Marker for YAML literal-block string rendering.""" + + +def _represent_literal_block_string( + dumper: yaml.SafeDumper, + data: _LiteralBlockString, +) -> yaml.nodes.ScalarNode: + return dumper.represent_scalar("tag:yaml.org,2002:str", str(data), style="|") + + +yaml.SafeDumper.add_representer(_LiteralBlockString, _represent_literal_block_string) + + +@dataclass(frozen=True) +class PreparedOsmoWorkflow: + input_path: str + output_path: str + selected_tasks: list[str] + modified_tasks: list[str] + wrapped_tasks: list[str] + lineage_dataset_name: str + lineage_bundle_filename: str + wrapper_script_path: str | None = None + runtime_bundle_local_path: str | None = None + runtime_bundle_remote_path: str | None = None + runtime_install_requirement: str | None = None + runtime_install_local_path: str | None = None + runtime_install_remote_path: str | None = None + + +def prepare_osmo_workflow_for_lineage( + *, + input_path: Path, + output_path: Path, + lineage_dataset_name: str, + lineage_bundle_filename: str, + task_names: list[str] | None = None, + default_to_all_tasks: bool = False, + inject_runtime_wrapper: bool = False, + wrapper_script_path: str = "/tmp/roar-osmo-wrapper.sh", + runtime_bundle_local_path: str | None = None, + runtime_bundle_remote_path: str = "/tmp/roar-osmo-runtime.tar.gz", + runtime_install_requirement: str | None = None, + runtime_install_local_path: str | None = None, + runtime_install_remote_path: str = "/tmp/roar-osmo-install.whl", +) -> PreparedOsmoWorkflow: + if runtime_bundle_local_path and not inject_runtime_wrapper: + raise ValueError("runtime bundle staging requires --inject-runtime-wrapper") + if runtime_install_requirement and not inject_runtime_wrapper: + raise ValueError("runtime install requires --inject-runtime-wrapper") + if runtime_install_local_path and not inject_runtime_wrapper: + raise ValueError("runtime install artifact requires --inject-runtime-wrapper") + if runtime_install_requirement and runtime_install_local_path: + raise ValueError("runtime install requirement and artifact are mutually exclusive") + + payload = _load_workflow_payload(input_path) + workflow = payload.get("workflow") + if not isinstance(workflow, dict): + raise ValueError(f"{input_path} is missing a workflow mapping") + + tasks = workflow.get("tasks") + if not isinstance(tasks, list) or not tasks: + raise ValueError(f"{input_path} is missing workflow.tasks") + + selected_tasks = _select_target_tasks(tasks, task_names, default_to_all_tasks) + modified_tasks: list[str] = [] + wrapped_tasks: list[str] = [] + for task in tasks: + if not isinstance(task, dict): + continue + task_name = str(task.get("name") or "").strip() + if task_name not in selected_tasks: + continue + task_modified = _ensure_lineage_dataset_output( + task, + lineage_dataset_name=lineage_dataset_name, + lineage_bundle_filename=lineage_bundle_filename, + ) + if inject_runtime_wrapper and _inject_runtime_wrapper( + task, + task_name=task_name, + lineage_bundle_filename=lineage_bundle_filename, + wrapper_script_path=wrapper_script_path, + runtime_bundle_remote_path=runtime_bundle_remote_path + if runtime_bundle_local_path + else None, + runtime_install_source=( + runtime_install_remote_path + if runtime_install_local_path + else runtime_install_requirement + ), + ): + task_modified = True + wrapped_tasks.append(task_name) + if runtime_bundle_local_path: + _ensure_runtime_bundle_file( + task, + runtime_bundle_local_path=runtime_bundle_local_path, + runtime_bundle_remote_path=runtime_bundle_remote_path, + ) + task_modified = True + if runtime_install_local_path: + _ensure_runtime_install_file( + task, + runtime_install_local_path=runtime_install_local_path, + runtime_install_remote_path=runtime_install_remote_path, + ) + task_modified = True + if task_modified: + modified_tasks.append(task_name) + + rendered = _render_workflow_payload(payload) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(rendered, encoding="utf-8") + + return PreparedOsmoWorkflow( + input_path=str(input_path), + output_path=str(output_path), + selected_tasks=selected_tasks, + modified_tasks=modified_tasks, + wrapped_tasks=wrapped_tasks, + lineage_dataset_name=lineage_dataset_name, + lineage_bundle_filename=lineage_bundle_filename, + wrapper_script_path=wrapper_script_path if inject_runtime_wrapper else None, + runtime_bundle_local_path=runtime_bundle_local_path, + runtime_bundle_remote_path=runtime_bundle_remote_path + if runtime_bundle_local_path + else None, + runtime_install_requirement=runtime_install_requirement, + runtime_install_local_path=runtime_install_local_path, + runtime_install_remote_path=runtime_install_remote_path + if runtime_install_local_path + else None, + ) + + +def _load_workflow_payload(input_path: Path) -> dict[str, Any]: + raw_text = input_path.read_text(encoding="utf-8") + normalized_text = re.sub( + r":\s*{{\s*([A-Za-z0-9_.-]+)\s*}}(\s*(#.*)?)$", + r': "__ROAR_OSMO_TEMPLATE_\1__"\2', + raw_text, + flags=re.MULTILINE, + ) + payload = yaml.safe_load(normalized_text) + if not isinstance(payload, dict): + raise ValueError(f"{input_path} did not parse to a workflow mapping") + return payload + + +def _select_target_tasks( + tasks: list[Any], + requested_task_names: list[str] | None, + default_to_all_tasks: bool, +) -> list[str]: + available_task_names = [ + str(task.get("name") or "").strip() + for task in tasks + if isinstance(task, dict) and str(task.get("name") or "").strip() + ] + if not available_task_names: + raise ValueError("workflow.tasks does not contain any named tasks") + + if requested_task_names: + normalized = [str(item).strip() for item in requested_task_names if str(item).strip()] + if not normalized: + raise ValueError("at least one non-empty task name is required") + missing = [name for name in normalized if name not in available_task_names] + if missing: + raise ValueError(f"workflow is missing requested task(s): {', '.join(missing)}") + # Preserve first-seen order and deduplicate. + ordered: list[str] = [] + seen: set[str] = set() + for name in normalized: + if name in seen: + continue + ordered.append(name) + seen.add(name) + return ordered + + if len(available_task_names) > 1 and not default_to_all_tasks: + raise ValueError( + "workflow has multiple tasks; specify --task for the task(s) that should emit Roar lineage" + ) + + return available_task_names + + +def _ensure_lineage_dataset_output( + task: dict[str, Any], + *, + lineage_dataset_name: str, + lineage_bundle_filename: str, +) -> bool: + outputs = task.get("outputs") + if not isinstance(outputs, list): + outputs = [] + task["outputs"] = outputs + + for item in outputs: + if not isinstance(item, dict): + continue + dataset = item.get("dataset") + if not isinstance(dataset, dict): + continue + existing_name = str(dataset.get("name") or "").strip() + existing_path = str(dataset.get("path") or "").strip() + if existing_name == lineage_dataset_name and existing_path == lineage_bundle_filename: + return False + + outputs.append( + { + "dataset": { + "name": lineage_dataset_name, + "path": lineage_bundle_filename, + } + } + ) + return True + + +def _inject_runtime_wrapper( + task: dict[str, Any], + *, + task_name: str, + lineage_bundle_filename: str, + wrapper_script_path: str, + runtime_bundle_remote_path: str | None, + runtime_install_source: str | None, +) -> bool: + command = task.get("command") + if not isinstance(command, list) or not command: + raise ValueError( + f"workflow task {task_name!r} must define command as a non-empty list to inject the Roar runtime wrapper" + ) + args = task.get("args") + if args is None: + normalized_args: list[Any] = [] + elif isinstance(args, list): + normalized_args = list(args) + else: + raise ValueError( + f"workflow task {task_name!r} must define args as a list when injecting the Roar runtime wrapper" + ) + + if command == ["bash", wrapper_script_path]: + _ensure_wrapper_file( + task, + wrapper_script_path=wrapper_script_path, + runtime_bundle_remote_path=runtime_bundle_remote_path, + runtime_install_source=runtime_install_source, + ) + return False + + _ensure_wrapper_file( + task, + wrapper_script_path=wrapper_script_path, + runtime_bundle_remote_path=runtime_bundle_remote_path, + runtime_install_source=runtime_install_source, + ) + task["command"] = ["bash", wrapper_script_path] + task["args"] = [ + task_name, + "{{output}}/" + lineage_bundle_filename, + *[str(item) for item in command], + *[str(item) for item in normalized_args], + ] + return True + + +def _ensure_wrapper_file( + task: dict[str, Any], + *, + wrapper_script_path: str, + runtime_bundle_remote_path: str | None, + runtime_install_source: str | None, +) -> None: + files = task.get("files") + if not isinstance(files, list): + files = [] + task["files"] = files + + for item in files: + if not isinstance(item, dict): + continue + if str(item.get("path") or "").strip() != wrapper_script_path: + continue + item["contents"] = _runtime_wrapper_contents( + runtime_bundle_remote_path, + runtime_install_source, + ) + item.pop("localpath", None) + return + + files.append( + { + "path": wrapper_script_path, + "contents": _runtime_wrapper_contents( + runtime_bundle_remote_path, + runtime_install_source, + ), + } + ) + + +def _ensure_runtime_bundle_file( + task: dict[str, Any], + *, + runtime_bundle_local_path: str, + runtime_bundle_remote_path: str, +) -> None: + files = task.get("files") + if not isinstance(files, list): + files = [] + task["files"] = files + + for item in files: + if not isinstance(item, dict): + continue + if str(item.get("path") or "").strip() != runtime_bundle_remote_path: + continue + item["localpath"] = runtime_bundle_local_path + item.pop("contents", None) + return + + files.append( + { + "localpath": runtime_bundle_local_path, + "path": runtime_bundle_remote_path, + } + ) + + +def _ensure_runtime_install_file( + task: dict[str, Any], + *, + runtime_install_local_path: str, + runtime_install_remote_path: str, +) -> None: + files = task.get("files") + if not isinstance(files, list): + files = [] + task["files"] = files + + for item in files: + if not isinstance(item, dict): + continue + if str(item.get("path") or "").strip() != runtime_install_remote_path: + continue + item["localpath"] = runtime_install_local_path + item.pop("contents", None) + return + + files.append( + { + "localpath": runtime_install_local_path, + "path": runtime_install_remote_path, + } + ) + + +def _runtime_wrapper_contents( + runtime_bundle_remote_path: str | None, + runtime_install_source: str | None, +) -> str: + runtime_setup = "" + if runtime_bundle_remote_path: + runtime_setup = f""" +runtime_bundle="{runtime_bundle_remote_path}" +runtime_root="/tmp/roar-osmo-runtime" +if [ -f "$runtime_bundle" ]; then + rm -rf "$runtime_root" + mkdir -p "$runtime_root" + tar -xzf "$runtime_bundle" -C "$runtime_root" + export PYTHONPATH="$runtime_root/python:$runtime_root/python/site-packages:${{PYTHONPATH:-}}" + export PATH="$runtime_root/bin:${{PATH:-}}" +fi +""" + runtime_install = "" + if runtime_install_source: + escaped_requirement = ( + runtime_install_source.replace("\\", "\\\\").replace('"', '\\"').replace("$", "\\$") + ) + runtime_install = f""" +install_root="/tmp/roar-osmo-python" +rm -rf "$install_root" +mkdir -p "$install_root" +if ! "$python_bin" -m pip --version >/dev/null 2>&1; then + pip_bootstrap_root="/tmp/roar-osmo-pip" + export HOME="$pip_bootstrap_root/home" + export PYTHONUSERBASE="$pip_bootstrap_root/userbase" + mkdir -p "$HOME" "$PYTHONUSERBASE" + if ! "$python_bin" -m ensurepip --user >/dev/null 2>&1; then + echo "pip is not available and ensurepip bootstrap failed for Roar OSMO wrapper" >&2 + exit 127 + fi + python_minor_version="$("$python_bin" - <<'PY' +import sys + +print(f"python{{sys.version_info[0]}}.{{sys.version_info[1]}}") +PY +)" + export PYTHONPATH="$PYTHONUSERBASE/lib/$python_minor_version/site-packages:${{PYTHONPATH:-}}" + pip_command="$PYTHONUSERBASE/bin/pip3" + if [ ! -x "$pip_command" ]; then + echo "ensurepip bootstrap did not produce an executable pip3 for Roar OSMO wrapper" >&2 + exit 127 + fi + "$pip_command" install --disable-pip-version-check --no-input --target "$install_root" "{escaped_requirement}" +else + "$python_bin" -m pip install --disable-pip-version-check --no-input --target "$install_root" "{escaped_requirement}" +fi +export PYTHONPATH="$install_root:${{PYTHONPATH:-}}" +""" + + return _LiteralBlockString( + f"""#!/usr/bin/env bash +set -euo pipefail + +task_name="$1" +bundle_path="$2" +shift 2 + +export ROAR_JOB_INSTRUMENTED=1 +export ROAR_EXECUTION_BACKEND=osmo +export ROAR_PROJECT_DIR="${{ROAR_PROJECT_DIR:-$PWD}}" +{runtime_setup} +python_bin="$(command -v python3 || command -v python)" +if [ -z "$python_bin" ]; then + echo "python interpreter not found for Roar OSMO wrapper" >&2 + exit 127 +fi +{runtime_install} +"$python_bin" - <<'PY' +from pathlib import Path + +import roar +from roar.execution.runtime import tracer_backends + +package_dir = Path(roar.__file__).resolve().parent +if tracer_backends.find_ptrace_tracer(package_dir) is None: + raise SystemExit( + "installed roar-cli distribution does not expose roar-tracer; use a packaged wheel with bundled binaries" + ) +PY +set +e +"$python_bin" -m roar run --tracer ptrace --no-tracer-fallback "$@" +command_status=$? +mkdir -p "$(dirname "$bundle_path")" +"$python_bin" -m roar osmo export-lineage-bundle "$bundle_path" --task-id "$task_name" --task-name "$task_name" +export_status=$? +set -e + +if [ "$command_status" -ne 0 ]; then + exit "$command_status" +fi + +exit "$export_status" +""" + ) + + +def _render_workflow_payload(payload: dict[str, Any]) -> str: + rendered = yaml.safe_dump(payload, sort_keys=False) + rendered = re.sub( + r"(['\"])__ROAR_OSMO_TEMPLATE_([A-Za-z0-9_.-]+)__\1", + lambda match: "{{ " + match.group(2) + " }}", + rendered, + ) + rendered = _OSMO_TEMPLATE_SENTINEL_RE.sub( + lambda match: "{{ " + match.group(1) + " }}", + rendered, + ) + return rendered + + +def resolve_roar_install_requirement(explicit_requirement: str | None = None) -> str: + if explicit_requirement and explicit_requirement.strip(): + return explicit_requirement.strip() + + override = os.environ.get("ROAR_CLUSTER_PIP_REQ", "").strip() + if override: + return override + + try: + version = importlib_metadata.version("roar-cli") + return f"roar-cli=={version}" + except importlib_metadata.PackageNotFoundError: + pass + except Exception: + pass + + return "roar-cli" + + +__all__ = [ + "PreparedOsmoWorkflow", + "prepare_osmo_workflow_for_lineage", + "resolve_roar_install_requirement", +] diff --git a/roar/cli/__init__.py b/roar/cli/__init__.py index e4c6e748..d7e7cfb8 100644 --- a/roar/cli/__init__.py +++ b/roar/cli/__init__.py @@ -39,6 +39,7 @@ "label": ("roar.cli.commands.label", "label", "Manage local labels"), "lineage": ("roar.cli.commands.lineage", "lineage", "Inspect lineage for a tracked artifact"), "log": ("roar.cli.commands.log", "log", "List jobs in the active session"), + "osmo": ("roar.cli.commands.osmo", "osmo", "Manage OSMO workflow attachment"), "pop": ("roar.cli.commands.pop", "pop", "Remove the last local step"), "proxy": ("roar.cli.commands.proxy", "proxy", "Manage S3 proxy for lineage tracking"), "put": ("roar.cli.commands.put", "put", "Publish artifacts and register lineage"), diff --git a/roar/cli/commands/osmo.py b/roar/cli/commands/osmo.py new file mode 100644 index 00000000..46e8cd6b --- /dev/null +++ b/roar/cli/commands/osmo.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import os +from collections.abc import Mapping +from pathlib import Path + +import click + +from ...backends.osmo import ( + OsmoAttachOptions, + attach_osmo_workflow, + build_osmo_runtime_bundle, + export_osmo_lineage_bundle, + prepare_osmo_workflow_for_lineage, + resolve_roar_install_requirement, +) +from ...backends.osmo.config import normalize_osmo_backend_config +from ..context import RoarContext +from ..decorators import require_init + + +@click.group("osmo", invoke_without_command=True) +@click.pass_context +def osmo(ctx: click.Context) -> None: + """Manage OSMO workflow-native lineage operations.""" + if ctx.invoked_subcommand is None: + click.echo(ctx.get_help()) + + +@osmo.command("attach") +@click.argument("workflow_id") +@click.option( + "--workflow-spec", + type=click.Path(path_type=Path, dir_okay=False, exists=True), + default=None, + help="Optional local workflow spec used to resolve task names and declared outputs", +) +@click.option( + "--set-string", + "set_strings", + multiple=True, + help="Template replacement in KEY=VALUE form for workflow-spec placeholders", +) +@click.option( + "--dataset", + "datasets", + multiple=True, + help="Declared output dataset name to download when no local workflow spec is available", +) +@click.option( + "--task", + "tasks", + multiple=True, + help="Workflow task name to use for diagnostics/log capture when no local workflow spec is available", +) +@click.option( + "--wait/--no-wait", + default=None, + help="Override osmo.wait_for_completion for this attach", +) +@click.option( + "--download-declared-outputs/--no-download-declared-outputs", + default=None, + help="Override osmo.download_declared_outputs for this attach", +) +@click.option( + "--ingest-lineage-bundles/--no-ingest-lineage-bundles", + default=None, + help="Override osmo.ingest_lineage_bundles for this attach", +) +@click.option( + "--osmo-binary", + default="osmo", + show_default=True, + help="OSMO CLI binary or path to invoke for workflow query/download operations", +) +@click.pass_obj +@require_init +def osmo_attach( + ctx: RoarContext, + workflow_id: str, + workflow_spec: Path | None, + set_strings: tuple[str, ...], + datasets: tuple[str, ...], + tasks: tuple[str, ...], + wait: bool | None, + download_declared_outputs: bool | None, + ingest_lineage_bundles: bool | None, + osmo_binary: str, +) -> None: + """Attach local Roar lineage ingestion to an existing OSMO workflow.""" + repo_root = str(ctx.repo_root or ctx.roar_dir.parent) + resolved_set_strings = _parse_set_strings(set_strings) + + result = attach_osmo_workflow( + roar_dir=ctx.roar_dir, + repo_root=repo_root, + workflow_id=workflow_id, + options=OsmoAttachOptions( + workflow_spec_argument=str(workflow_spec) if workflow_spec is not None else None, + workflow_spec_path=str(workflow_spec) if workflow_spec is not None else None, + set_strings=resolved_set_strings or None, + dataset_names=[item for item in datasets if item.strip()] or None, + task_names=[item for item in tasks if item.strip()] or None, + wait_for_completion=wait, + download_declared_outputs=download_declared_outputs, + ingest_lineage_bundles=ingest_lineage_bundles, + ), + osmo_binary=osmo_binary, + ) + if result.exit_code != 0: + raise SystemExit(result.exit_code) + + +@osmo.command("prepare-workflow") +@click.argument( + "input_path", + type=click.Path(path_type=Path, dir_okay=False, exists=True), +) +@click.argument( + "output_path", + type=click.Path(path_type=Path, dir_okay=False), +) +@click.option( + "--task", + "tasks", + multiple=True, + help="Workflow task name to augment with the standard Roar lineage dataset output", +) +@click.option( + "--inject-runtime-wrapper/--no-inject-runtime-wrapper", + default=False, + help="Rewrite selected tasks to run through a Roar wrapper that exports the standard lineage bundle", +) +@click.option( + "--wrapper-script-path", + default="/tmp/roar-osmo-wrapper.sh", + show_default=True, + help="Path written into selected OSMO tasks for the generated Roar wrapper script", +) +@click.option( + "--stage-roar-runtime/--no-stage-roar-runtime", + default=False, + help="Build and attach a local Roar runtime bundle for the generated wrapper to unpack remotely", +) +@click.option( + "--install-roar-runtime/--no-install-roar-runtime", + default=False, + help="Install roar-cli inside the generated wrapper at task startup instead of relying on a prebuilt image", +) +@click.option( + "--runtime-install-requirement", + default=None, + help="Pinned requirement, wheel URL, or package reference installed by the generated wrapper; use a packaged roar-cli distribution with bundled tracer binaries", +) +@click.option( + "--runtime-install-local-path", + type=click.Path(path_type=Path, dir_okay=False, exists=True), + default=None, + help="Local wheel or artifact path injected into the prepared workflow and installed by the wrapper; a roar-cli wheel should include bundled tracer binaries", +) +@click.option( + "--runtime-install-remote-path", + default="/tmp/roar-osmo-install.whl", + show_default=True, + help="Remote path used inside selected OSMO tasks for an injected runtime install artifact", +) +@click.option( + "--runtime-bundle-path", + type=click.Path(path_type=Path, dir_okay=False), + default=None, + help="Optional output path for the staged Roar runtime bundle tarball", +) +@click.option( + "--runtime-python-root", + "runtime_python_roots", + type=click.Path(path_type=Path, file_okay=False, exists=True), + multiple=True, + help="Override Python runtime root(s) copied into the staged Roar runtime bundle", +) +@click.option( + "--runtime-roar-package-dir", + type=click.Path(path_type=Path, file_okay=False, exists=True), + default=None, + help="Override the local roar package directory copied into the staged runtime bundle", +) +@click.option( + "--runtime-tracer", + type=click.Path(path_type=Path, dir_okay=False, exists=True), + default=None, + help="Override the ptrace tracer binary copied into the staged runtime bundle", +) +@click.option( + "--runtime-bundle-remote-path", + default="/tmp/roar-osmo-runtime.tar.gz", + show_default=True, + help="Remote path used inside selected OSMO tasks for the staged runtime bundle tarball", +) +@click.pass_obj +@require_init +def osmo_prepare_workflow( + ctx: RoarContext, + input_path: Path, + output_path: Path, + tasks: tuple[str, ...], + inject_runtime_wrapper: bool, + wrapper_script_path: str, + stage_roar_runtime: bool, + install_roar_runtime: bool, + runtime_install_requirement: str | None, + runtime_install_local_path: Path | None, + runtime_install_remote_path: str, + runtime_bundle_path: Path | None, + runtime_python_roots: tuple[Path, ...], + runtime_roar_package_dir: Path | None, + runtime_tracer: Path | None, + runtime_bundle_remote_path: str, +) -> None: + """Write an OSMO workflow spec augmented with Roar's lineage dataset output.""" + if stage_roar_runtime and not inject_runtime_wrapper: + raise click.ClickException("--stage-roar-runtime requires --inject-runtime-wrapper") + if install_roar_runtime and not inject_runtime_wrapper: + raise click.ClickException("--install-roar-runtime requires --inject-runtime-wrapper") + if stage_roar_runtime and install_roar_runtime: + raise click.ClickException( + "--stage-roar-runtime and --install-roar-runtime are mutually exclusive" + ) + if runtime_install_local_path is not None and not install_roar_runtime: + raise click.ClickException("--runtime-install-local-path requires --install-roar-runtime") + if runtime_install_local_path is not None and runtime_install_requirement is not None: + raise click.ClickException( + "--runtime-install-local-path and --runtime-install-requirement are mutually exclusive" + ) + + config_section = ctx.config.get("osmo", {}) + if not isinstance(config_section, Mapping): + config_section = None + config = normalize_osmo_backend_config(config_section) + resolved_runtime_bundle_path: Path | None = None + runtime_bundle_local_path: str | None = None + if stage_roar_runtime: + resolved_runtime_bundle_path = runtime_bundle_path or ( + output_path.parent / "roar-osmo-runtime.tar.gz" + ) + try: + build_osmo_runtime_bundle( + output_path=resolved_runtime_bundle_path, + roar_package_dir=runtime_roar_package_dir, + python_roots=list(runtime_python_roots) or None, + ptrace_tracer_path=runtime_tracer, + ) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + + runtime_bundle_local_path = os.path.relpath( + resolved_runtime_bundle_path, + start=output_path.parent, + ) + + resolved_runtime_install_requirement: str | None = None + resolved_runtime_install_local_path: str | None = None + if install_roar_runtime: + if runtime_install_local_path is not None: + resolved_runtime_install_local_path = os.path.relpath( + runtime_install_local_path, + start=output_path.parent, + ) + else: + resolved_runtime_install_requirement = resolve_roar_install_requirement( + runtime_install_requirement + ) + + try: + prepared = prepare_osmo_workflow_for_lineage( + input_path=input_path, + output_path=output_path, + lineage_dataset_name=str(config["lineage_bundle_dataset_name"]), + lineage_bundle_filename=str(config["lineage_bundle_filename"]), + task_names=[item for item in tasks if item.strip()] or None, + inject_runtime_wrapper=inject_runtime_wrapper, + wrapper_script_path=wrapper_script_path, + runtime_bundle_local_path=runtime_bundle_local_path, + runtime_bundle_remote_path=runtime_bundle_remote_path, + runtime_install_requirement=resolved_runtime_install_requirement, + runtime_install_local_path=resolved_runtime_install_local_path, + runtime_install_remote_path=runtime_install_remote_path, + ) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + + selected_tasks = ", ".join(prepared.selected_tasks) + modified_tasks = ", ".join(prepared.modified_tasks) if prepared.modified_tasks else "none" + click.echo(f"Prepared OSMO workflow: {prepared.output_path}") + click.echo(f"Selected tasks: {selected_tasks}") + click.echo(f"Modified tasks: {modified_tasks}") + if prepared.wrapped_tasks: + click.echo(f"Wrapped tasks: {', '.join(prepared.wrapped_tasks)}") + if resolved_runtime_bundle_path is not None: + click.echo(f"Staged runtime bundle: {resolved_runtime_bundle_path}") + if resolved_runtime_install_requirement is not None: + click.echo(f"Runtime install requirement: {resolved_runtime_install_requirement}") + if resolved_runtime_install_local_path is not None: + click.echo(f"Runtime install artifact: {resolved_runtime_install_local_path}") + + +@osmo.command("export-lineage-bundle") +@click.argument( + "output_path", + type=click.Path(path_type=Path, dir_okay=False), +) +@click.option("--job-uid", default=None, help="Specific local Roar job UID to export") +@click.option("--task-id", default=None, help="OSMO task identifier to place in the bundle") +@click.option("--task-name", default=None, help="OSMO task name to place in the bundle") +@click.pass_obj +@require_init +def osmo_export_lineage_bundle( + ctx: RoarContext, + output_path: Path, + job_uid: str | None, + task_id: str | None, + task_name: str | None, +) -> None: + """Export a local Roar job as an OSMO lineage bundle.""" + try: + exported = export_osmo_lineage_bundle( + roar_dir=ctx.roar_dir, + output_path=output_path, + job_uid=job_uid, + task_id=task_id, + task_name=task_name, + ) + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + + click.echo(f"Exported OSMO lineage bundle: {exported.output_path}") + click.echo(f"Job UID: {exported.exported_job_uid or ''}") + click.echo(f"Task ID: {exported.task_id}") + click.echo(f"Task name: {exported.task_name}") + + +def _parse_set_strings(values: tuple[str, ...]) -> dict[str, str]: + parsed: dict[str, str] = {} + for item in values: + if "=" not in item: + raise click.ClickException(f"Invalid --set-string value {item!r}; expected KEY=VALUE") + key, value = item.split("=", 1) + key = key.strip() + value = value.strip() + if not key or not value: + raise click.ClickException(f"Invalid --set-string value {item!r}; expected KEY=VALUE") + parsed[key] = value + return parsed + + +__all__ = [ + "osmo", + "osmo_attach", + "osmo_export_lineage_bundle", + "osmo_prepare_workflow", +] diff --git a/roar/execution/framework/registry.py b/roar/execution/framework/registry.py index 78d2dffa..a59c9c22 100644 --- a/roar/execution/framework/registry.py +++ b/roar/execution/framework/registry.py @@ -14,6 +14,7 @@ _ENTRYPOINT_GROUP = "roar.execution_backends" _BUILTIN_EXECUTION_BACKEND_MODULES = ( "roar.backends.ray.plugin", + "roar.backends.osmo.plugin", "roar.backends.local.plugin", ) _registered_execution_backends: list[ExecutionBackend] = [] diff --git a/roar/filters/files.py b/roar/filters/files.py index 7296976a..98e8bfb3 100644 --- a/roar/filters/files.py +++ b/roar/filters/files.py @@ -249,7 +249,7 @@ def classify(self, path: str, git_tracked: set[str] | None = None) -> tuple[str, stderr=subprocess.DEVNULL, ) return ("repo", None) - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): # In repo but not tracked - could be generated file return ("unmanaged", None) except ValueError: diff --git a/roar/integrations/git/provider.py b/roar/integrations/git/provider.py index ffa44a5d..a147c433 100644 --- a/roar/integrations/git/provider.py +++ b/roar/integrations/git/provider.py @@ -41,15 +41,17 @@ def get_repo_root(self, path: str | None = None) -> str | None: else: out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) return out.decode().strip() - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): return None def get_info(self, repo_root: str) -> VCSInfo: """Get comprehensive git repository information.""" info = VCSInfo() + if not self.is_available(): + return info # Current commit hash - with contextlib.suppress(subprocess.CalledProcessError): + with contextlib.suppress(subprocess.CalledProcessError, FileNotFoundError): info.commit = ( subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=repo_root, stderr=subprocess.DEVNULL @@ -59,7 +61,7 @@ def get_info(self, repo_root: str) -> VCSInfo: ) # Current branch - with contextlib.suppress(subprocess.CalledProcessError): + with contextlib.suppress(subprocess.CalledProcessError, FileNotFoundError): info.branch = ( subprocess.check_output( ["git", "rev-parse", "--abbrev-ref", "HEAD"], @@ -71,7 +73,7 @@ def get_info(self, repo_root: str) -> VCSInfo: ) # Remote URL (origin) - with contextlib.suppress(subprocess.CalledProcessError): + with contextlib.suppress(subprocess.CalledProcessError, FileNotFoundError): info.remote_url = ( subprocess.check_output( ["git", "remote", "get-url", "origin"], cwd=repo_root, stderr=subprocess.DEVNULL @@ -87,7 +89,7 @@ def get_info(self, repo_root: str) -> VCSInfo: info.uncommitted_changes = changes # Commit timestamp - with contextlib.suppress(subprocess.CalledProcessError): + with contextlib.suppress(subprocess.CalledProcessError, FileNotFoundError): info.commit_timestamp = ( subprocess.check_output( ["git", "show", "-s", "--format=%ci", "HEAD"], @@ -99,7 +101,7 @@ def get_info(self, repo_root: str) -> VCSInfo: ) # Commit message (first line) - with contextlib.suppress(subprocess.CalledProcessError): + with contextlib.suppress(subprocess.CalledProcessError, FileNotFoundError): info.commit_message = ( subprocess.check_output( ["git", "show", "-s", "--format=%s", "HEAD"], @@ -119,7 +121,7 @@ def get_status(self, repo_root: str) -> tuple[bool, list[str]]: lines = out.decode().splitlines() clean = len(lines) == 0 return clean, lines - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): return True, [] def classify_file(self, repo_root: str, path: str) -> str: @@ -158,7 +160,7 @@ def is_tracked(self, repo_root: str, path: str) -> bool: stderr=subprocess.DEVNULL, ) return True - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): return False def get_commit_hash(self, repo_root: str) -> str | None: @@ -171,7 +173,7 @@ def get_commit_hash(self, repo_root: str) -> str | None: .decode() .strip() ) - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): return None def get_branch(self, repo_root: str) -> str | None: @@ -186,7 +188,7 @@ def get_branch(self, repo_root: str) -> str | None: .decode() .strip() ) - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): return None def get_remote_url(self, repo_root: str, remote: str = "origin") -> str | None: @@ -199,7 +201,7 @@ def get_remote_url(self, repo_root: str, remote: str = "origin") -> str | None: .decode() .strip() ) - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): return None def create_tag( diff --git a/roar/integrations/glaas/transport.py b/roar/integrations/glaas/transport.py index ca2802fd..38ef9c18 100644 --- a/roar/integrations/glaas/transport.py +++ b/roar/integrations/glaas/transport.py @@ -196,7 +196,9 @@ def _perform_request( return None, str(e), None auth_mode = _get_cached_auth_mode(base_url) - auth_header = None if auth_mode == "anonymous" else auth_header_factory(method, path, body_bytes) + auth_header = ( + None if auth_mode == "anonymous" else auth_header_factory(method, path, body_bytes) + ) if auth_mode == "unknown" and auth_header: probe_auth_header = auth_header_factory("GET", _AUTH_PROBE_PATH, None) diff --git a/scripts/build_wheel_with_bins.sh b/scripts/build_wheel_with_bins.sh index 54d5531a..73ccaded 100755 --- a/scripts/build_wheel_with_bins.sh +++ b/scripts/build_wheel_with_bins.sh @@ -126,6 +126,55 @@ resolve_built_artifact() { return 1 } +build_python_wheel() { + if command -v uv >/dev/null 2>&1; then + echo "â–¶ Building wheel with uv..." + uv build --wheel --out-dir "$OUT_DIR" + return + fi + + local python_candidate + for python_candidate in "${PYTHON:-}" python3 python; do + if [[ -z "$python_candidate" ]]; then + continue + fi + if [[ "$python_candidate" == */* ]]; then + if [[ ! -x "$python_candidate" ]]; then + continue + fi + elif ! command -v "$python_candidate" >/dev/null 2>&1; then + continue + fi + if command -v maturin >/dev/null 2>&1; then + echo "â–¶ Building wheel with maturin..." + ( + cd "$ROOT_DIR" + maturin build \ + --release \ + --manifest-path rust/crates/artifact-hash-py/Cargo.toml \ + --interpreter "$python_candidate" \ + --out "$OUT_DIR" + ) + return + fi + if "$python_candidate" -m maturin --help >/dev/null 2>&1; then + echo "â–¶ Building wheel with $python_candidate -m maturin..." + ( + cd "$ROOT_DIR" + "$python_candidate" -m maturin build \ + --release \ + --manifest-path rust/crates/artifact-hash-py/Cargo.toml \ + --interpreter "$python_candidate" \ + --out "$OUT_DIR" + ) + return + fi + done + + echo "error: wheel build requires either uv or maturin available" >&2 + exit 1 +} + ensure_binary "roar-proxy" "roar-proxy" ensure_binary "roar-tracer" "roar-tracer" ensure_binary "roar-tracer-ebpf" "roar-tracer-ebpf" @@ -190,7 +239,7 @@ if [[ "$sync_preload_lib" -eq 1 ]]; then fi echo "â–¶ Building roar wheel into $OUT_DIR..." -uv build --wheel --out-dir "$OUT_DIR" +build_python_wheel echo "â–¶ Verifying wheel contents..." ( diff --git a/scripts/sync_packaged_rust_artifacts.py b/scripts/sync_packaged_rust_artifacts.py index 6b2e1682..e6d95d92 100644 --- a/scripts/sync_packaged_rust_artifacts.py +++ b/scripts/sync_packaged_rust_artifacts.py @@ -29,6 +29,7 @@ class SyncLayout: release_dir: Path package_bin_dir: Path artifacts: tuple[ArtifactSpec, ...] + fallback_release_dirs: tuple[Path, ...] = () portable_target: str | None = None @@ -46,11 +47,14 @@ def _default_layout() -> SyncLayout: root_dir = Path(__file__).resolve().parents[1] library_suffix = ".dylib" if sys.platform == "darwin" else ".so" common_sources = _common_tracer_sources(root_dir) - release_dir = root_dir / "rust" / "target" / "release" + host_release_dir = root_dir / "rust" / "target" / "release" + release_dir = host_release_dir + fallback_release_dirs: tuple[Path, ...] = () portable_target = None if sys.platform.startswith("linux"): release_dir = root_dir / "rust" / "target" / LINUX_PORTABLE_TARGET_DIR / "release" + fallback_release_dirs = (host_release_dir,) portable_target = LINUX_PORTABLE_TARGET artifacts = [ @@ -107,10 +111,31 @@ def _default_layout() -> SyncLayout: release_dir=release_dir, package_bin_dir=root_dir / "roar" / "bin", artifacts=tuple(artifacts), + fallback_release_dirs=fallback_release_dirs, portable_target=portable_target, ) +def _candidate_release_dirs(layout: SyncLayout) -> tuple[Path, ...]: + return (layout.release_dir, *layout.fallback_release_dirs) + + +def _find_release_binary(layout: SyncLayout, binary_name: str) -> Path | None: + for release_dir in _candidate_release_dirs(layout): + candidate = release_dir / binary_name + if candidate.exists(): + return candidate + return None + + +def _find_release_library(layout: SyncLayout, names: tuple[str, ...]) -> Path | None: + for release_dir in _candidate_release_dirs(layout): + candidate = _first_existing_path(release_dir, names) + if candidate is not None: + return candidate + return None + + def _iter_source_files(paths: tuple[Path, ...]) -> list[Path]: files: list[Path] = [] for source_path in paths: @@ -168,7 +193,7 @@ def sync_reason(layout: SyncLayout) -> str | None: for binary_name in artifact.binary_names: reason = _sync_reason_for_path( - release_path=layout.release_dir / binary_name, + release_path=_find_release_binary(layout, binary_name), package_path=layout.package_bin_dir / binary_name, latest_source_mtime=latest_source_mtime, missing_release_reason=f"release {binary_name} is missing", @@ -180,7 +205,7 @@ def sync_reason(layout: SyncLayout) -> str | None: return reason if artifact.library_names: - release_library = _first_existing_path(layout.release_dir, artifact.library_names) + release_library = _find_release_library(layout, artifact.library_names) package_library = _first_existing_path(layout.package_bin_dir, artifact.library_names) if release_library is None: return f"release library for {artifact.package_name} is missing" @@ -205,11 +230,12 @@ def _packages_needing_build(layout: SyncLayout) -> list[str]: latest_source_mtime = _latest_mtime(_iter_source_files(artifact.source_paths)) needs_build = False for binary_name in artifact.binary_names: - if _artifact_is_stale(layout.release_dir / binary_name, latest_source_mtime): + release_binary = _find_release_binary(layout, binary_name) + if release_binary is None or _artifact_is_stale(release_binary, latest_source_mtime): needs_build = True break if not needs_build and artifact.library_names: - release_library = _first_existing_path(layout.release_dir, artifact.library_names) + release_library = _find_release_library(layout, artifact.library_names) if release_library is None or _artifact_is_stale(release_library, latest_source_mtime): needs_build = True if needs_build and artifact.package_name not in packages: @@ -261,13 +287,13 @@ def sync_packaged_rust_artifacts(layout: SyncLayout) -> None: for artifact in layout.artifacts: for binary_name in artifact.binary_names: - release_path = layout.release_dir / binary_name - if not release_path.exists(): + release_path = _find_release_binary(layout, binary_name) + if release_path is None: raise SystemExit(f"release {binary_name} is missing after build") _sync_file(release_path, layout.package_bin_dir / binary_name) if artifact.library_names: - release_library = _first_existing_path(layout.release_dir, artifact.library_names) + release_library = _find_release_library(layout, artifact.library_names) if release_library is None: raise SystemExit( f"release library for {artifact.package_name} is missing after build" diff --git a/tests/backends/osmo/Dockerfile b/tests/backends/osmo/Dockerfile new file mode 100644 index 00000000..68a449d1 --- /dev/null +++ b/tests/backends/osmo/Dockerfile @@ -0,0 +1,37 @@ +ARG PYTHON_IMAGE=public.ecr.aws/docker/library/python:3.11-slim +FROM ${PYTHON_IMAGE} + +ARG DOCKER_VERSION=28.2.2 +ARG HELM_VERSION=v3.16.2 +ARG KIND_VERSION=v0.29.0 +ARG KUBECTL_VERSION=v1.32.2 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends bash ca-certificates curl git jq tar gzip \ + && rm -rf /var/lib/apt/lists/* + +RUN arch="$(dpkg --print-architecture)" \ + && case "$arch" in \ + amd64) docker_arch=x86_64; helm_arch=amd64; kind_arch=amd64; kubectl_arch=amd64 ;; \ + arm64) docker_arch=aarch64; helm_arch=arm64; kind_arch=arm64; kubectl_arch=arm64 ;; \ + *) echo "unsupported architecture: $arch" >&2; exit 1 ;; \ + esac \ + && curl -fsSL "https://download.docker.com/linux/static/stable/${docker_arch}/docker-${DOCKER_VERSION}.tgz" \ + | tar -xz --strip-components=1 -C /usr/local/bin docker/docker \ + && curl -fsSL -o /usr/local/bin/kubectl \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${kubectl_arch}/kubectl" \ + && chmod +x /usr/local/bin/kubectl \ + && curl -fsSL -o /usr/local/bin/kind \ + "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-${kind_arch}" \ + && chmod +x /usr/local/bin/kind \ + && curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-${helm_arch}.tar.gz" \ + | tar -xz -C /tmp \ + && mv "/tmp/linux-${helm_arch}/helm" /usr/local/bin/helm \ + && chmod +x /usr/local/bin/helm \ + && rm -rf /tmp/"linux-${helm_arch}" + +RUN curl -fsSL https://raw.githubusercontent.com/NVIDIA/OSMO/main/install.sh | bash + +WORKDIR /workspace/roar + +CMD ["bash", "-lc", "sleep infinity"] diff --git a/tests/backends/osmo/__init__.py b/tests/backends/osmo/__init__.py new file mode 100644 index 00000000..7bfbf2b4 --- /dev/null +++ b/tests/backends/osmo/__init__.py @@ -0,0 +1 @@ +"""OSMO end-to-end tests.""" diff --git a/tests/backends/osmo/conftest.py b/tests/backends/osmo/conftest.py new file mode 100644 index 00000000..c4d6d200 --- /dev/null +++ b/tests/backends/osmo/conftest.py @@ -0,0 +1,609 @@ +"""Pytest fixtures for the OSMO Docker Compose + KIND harness.""" + +from __future__ import annotations + +import contextlib +import functools +import hashlib +import json +import os +import shlex +import shutil +import socket +import subprocess +import sys +import textwrap +import time +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path + +import pytest + +COMPOSE_FILE = Path(__file__).resolve().parent / "docker-compose.yml" +REPO_ROOT = COMPOSE_FILE.parent.parent.parent.parent.resolve() +HOST_TMP_DIR = REPO_ROOT / ".tmp-osmo-e2e" +HOST_DOWNLOADS_DIR = HOST_TMP_DIR / "downloads" +HOST_PROJECTS_DIR = HOST_TMP_DIR / "projects" +HOST_WHEELS_DIR = HOST_TMP_DIR / "wheels" +CONTAINER_REPO_ROOT = Path("/workspace/roar") +CONTAINER_TMP_DIR = CONTAINER_REPO_ROOT / ".tmp-osmo-e2e" +CONTAINER_DOWNLOADS_DIR = CONTAINER_TMP_DIR / "downloads" +CONTAINER_PROJECTS_DIR = CONTAINER_TMP_DIR / "projects" +CONTAINER_WHEELS_DIR = CONTAINER_TMP_DIR / "wheels" +BASE_URL = "http://quick-start.osmo:38080" +BOOTSTRAP_TIMEOUT_SECONDS = 45 * 60 +QUERY_TIMEOUT_SECONDS = 12 * 60 +POLL_INTERVAL_SECONDS = 5 +PORT_FORWARD_TIMEOUT_SECONDS = 5 * 60 +LOCALSTACK_FORWARD_PORT = os.environ.get("OSMO_LOCALSTACK_PORT", "34566") +LOCALSTACK_HOST_OVERRIDE_URL = f"http://127.0.0.1:{LOCALSTACK_FORWARD_PORT}" +DEFAULT_OSMO_TEST_PYTHON_IMAGE = ( + f"public.ecr.aws/docker/library/python:{sys.version_info.major}.{sys.version_info.minor}-slim" +) +OSMO_TEST_PYTHON_IMAGE = os.environ.get( + "OSMO_TEST_PYTHON_IMAGE", + DEFAULT_OSMO_TEST_PYTHON_IMAGE, +) + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", + "osmo_e2e: OSMO end-to-end tests requiring a Docker Compose managed KIND harness", + ) + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + marker = pytest.mark.osmo_e2e + for item in items: + item.add_marker(marker) + if item.get_closest_marker("timeout") is None: + item.add_marker(pytest.mark.timeout(45 * 60)) + + +@functools.lru_cache(maxsize=1) +def _docker_accessible() -> bool: + try: + subprocess.run( + ["docker", "info"], + check=True, + capture_output=True, + timeout=5, + ) + return True + except (subprocess.SubprocessError, OSError): + return False + + +def run_docker(args: Sequence[str], **kwargs) -> subprocess.CompletedProcess[str]: + command = list(args) + if _docker_accessible(): + return subprocess.run(command, **kwargs) + return subprocess.run(["sg", "docker", "-c", shlex.join(command)], **kwargs) + + +def _compose_args(*args: str) -> list[str]: + return ["docker", "compose", "-f", str(COMPOSE_FILE), *args] + + +def _compose_env() -> dict[str, str]: + return { + **os.environ, + "OSMO_TEST_PYTHON_IMAGE": OSMO_TEST_PYTHON_IMAGE, + } + + +def _compose_exec_command( + service: str, + args: Sequence[str], + *, + env: Mapping[str, str] | None = None, +) -> list[str]: + command = ["docker", "compose", "-f", str(COMPOSE_FILE), "exec", "-T"] + if env: + for key, value in env.items(): + command.extend(["-e", f"{key}={value}"]) + command.append(service) + command.extend(args) + return command + + +def exec_on_service( + service: str, + args: Sequence[str], + *, + env: Mapping[str, str] | None = None, + timeout: float | None = None, +) -> subprocess.CompletedProcess[str]: + command = _compose_exec_command(service, args, env=env) + return run_docker(command, capture_output=True, text=True, check=False, timeout=timeout) + + +def osmo_exec( + args: Sequence[str], + *, + timeout: float | None = None, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + result = exec_on_service("osmo-tools", args, timeout=timeout) + if check and result.returncode != 0: + raise RuntimeError( + "OSMO command failed:\n" + f"command: {' '.join(args)}\n" + f"stdout:\n{textwrap.indent(result.stdout, ' ')}\n" + f"stderr:\n{textwrap.indent(result.stderr, ' ')}" + ) + return result + + +def roar_exec( + args: Sequence[str], + *, + cwd: str, + timeout: float | None = None, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + shell_command = f"cd {shlex.quote(cwd)} && python3 -m roar {shlex.join(list(args))}" + result = exec_on_service( + "osmo-tools", + ["bash", "-lc", shell_command], + timeout=timeout, + ) + if check and result.returncode != 0: + raise RuntimeError( + "Roar command failed:\n" + f"command: {' '.join(args)}\n" + f"cwd: {cwd}\n" + f"stdout:\n{textwrap.indent(result.stdout, ' ')}\n" + f"stderr:\n{textwrap.indent(result.stderr, ' ')}" + ) + return result + + +def allow_git_safe_directory(path: str | Path) -> None: + repo_path = str(path) + result = exec_on_service( + "osmo-tools", + [ + "bash", + "-lc", + f"cd /tmp && git config --global --add safe.directory {shlex.quote(repo_path)}", + ], + timeout=30, + ) + if result.returncode != 0: + raise RuntimeError( + "Failed to allow git safe.directory in osmo-tools:\n" + f"path: {repo_path}\n" + f"stdout:\n{textwrap.indent(result.stdout, ' ')}\n" + f"stderr:\n{textwrap.indent(result.stderr, ' ')}" + ) + + +def restore_host_path_ownership(path: str | Path) -> None: + repo_path = str(path) + result = exec_on_service( + "osmo-tools", + [ + "bash", + "-lc", + (f"chown -R {os.getuid()}:{os.getgid()} {shlex.quote(repo_path)}"), + ], + timeout=5 * 60, + ) + if result.returncode != 0: + raise RuntimeError( + "Failed to restore host path ownership from osmo-tools:\n" + f"path: {repo_path}\n" + f"stdout:\n{textwrap.indent(result.stdout, ' ')}\n" + f"stderr:\n{textwrap.indent(result.stderr, ' ')}" + ) + + +def patch_local_osmo_data_override_url(override_url: str) -> None: + script = f""" +cat <<'EOF' >/root/.config/osmo/config.yaml +auth: + data: + s3://osmo: + access_key: test + access_key_id: test + override_url: {override_url} + region: us-east-1 +EOF +""" + result = exec_on_service( + "osmo-tools", + ["bash", "-lc", script], + timeout=30, + ) + if result.returncode != 0: + raise RuntimeError( + "Failed to patch local OSMO data override URL.\n" + f"stdout:\n{textwrap.indent(result.stdout, ' ')}\n" + f"stderr:\n{textwrap.indent(result.stderr, ' ')}" + ) + + +def publish_runtime_artifact_service( + *, + service_name: str, + host_artifact_path: Path, + artifact_filename: str, +) -> str: + image_suffix = hashlib.sha256(host_artifact_path.read_bytes()).hexdigest()[:12] + image_tag = f"{service_name}:{image_suffix}" + build_dir = HOST_WHEELS_DIR / f"{service_name}-image" + shutil.rmtree(build_dir, ignore_errors=True) + build_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(host_artifact_path, build_dir / artifact_filename) + (build_dir / "Dockerfile").write_text( + f""" +FROM busybox:1.37.0 +COPY {artifact_filename} /srv/{artifact_filename} +CMD ["sh", "-c", "httpd -f -p 8080 -h /srv"] +""".strip() + + "\n", + encoding="utf-8", + ) + + build_result = run_docker( + [ + "docker", + "build", + "-t", + image_tag, + str(build_dir), + ], + check=False, + capture_output=True, + text=True, + timeout=20 * 60, + ) + if build_result.returncode != 0: + raise RuntimeError( + "Failed to build runtime artifact image.\n" + f"stdout:\n{textwrap.indent(build_result.stdout, ' ')}\n" + f"stderr:\n{textwrap.indent(build_result.stderr, ' ')}" + ) + + load_result = exec_on_service( + "osmo-tools", + ["bash", "-lc", f"kind load docker-image --name roar-osmo-e2e {shlex.quote(image_tag)}"], + timeout=20 * 60, + ) + if load_result.returncode != 0: + raise RuntimeError( + "Failed to load runtime artifact image into kind.\n" + f"stdout:\n{textwrap.indent(load_result.stdout, ' ')}\n" + f"stderr:\n{textwrap.indent(load_result.stderr, ' ')}" + ) + + script = f""" +set -euo pipefail +kubectl -n osmo delete deployment {shlex.quote(service_name)} --ignore-not-found >/dev/null +kubectl -n osmo delete service {shlex.quote(service_name)} --ignore-not-found >/dev/null +cat <<'YAML' | kubectl apply -f - +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {service_name} + namespace: osmo +spec: + replicas: 1 + selector: + matchLabels: + app: {service_name} + template: + metadata: + labels: + app: {service_name} + spec: + nodeSelector: + node_group: service + containers: + - name: http + image: {image_tag} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: {service_name} + namespace: osmo +spec: + selector: + app: {service_name} + ports: + - port: 80 + targetPort: 8080 +YAML +kubectl rollout status deployment/{shlex.quote(service_name)} -n osmo --timeout=5m >/dev/null +""" + result = exec_on_service( + "osmo-tools", + ["bash", "-lc", script], + timeout=10 * 60, + ) + if result.returncode != 0: + raise RuntimeError( + "Failed to publish runtime artifact service.\n" + f"stdout:\n{textwrap.indent(result.stdout, ' ')}\n" + f"stderr:\n{textwrap.indent(result.stderr, ' ')}" + ) + return f"http://{service_name}.osmo.svc.cluster.local/{artifact_filename}" + + +def container_repo_path(path: Path) -> Path: + return CONTAINER_REPO_ROOT / path.resolve().relative_to(REPO_ROOT) + + +def _remove_downloads_dir() -> None: + with contextlib.suppress(FileNotFoundError): + try: + shutil.rmtree(HOST_DOWNLOADS_DIR) + except PermissionError: + run_docker( + [ + "docker", + "run", + "--rm", + "-v", + f"{HOST_DOWNLOADS_DIR.parent}:/cleanup", + "alpine:3.19", + "sh", + "-c", + "rm -rf /cleanup/downloads", + ], + check=True, + capture_output=True, + text=True, + timeout=5 * 60, + ) + + +def _prepare_host_tmp_dirs() -> None: + HOST_PROJECTS_DIR.mkdir(parents=True, exist_ok=True) + HOST_WHEELS_DIR.mkdir(parents=True, exist_ok=True) + + +def _popen_docker( + args: Sequence[str], + *, + stdout, + stderr, +) -> subprocess.Popen[str]: + command = list(args) + if _docker_accessible(): + return subprocess.Popen(command, stdout=stdout, stderr=stderr, text=True) + return subprocess.Popen( + ["sg", "docker", "-c", shlex.join(command)], + stdout=stdout, + stderr=stderr, + text=True, + ) + + +@pytest.fixture(scope="session") +def osmo_harness() -> dict[str, str]: + if not shutil.which("docker"): + pytest.skip("docker is required for OSMO e2e tests") + + _remove_downloads_dir() + _prepare_host_tmp_dirs() + HOST_DOWNLOADS_DIR.mkdir(parents=True, exist_ok=True) + + run_docker( + _compose_args("up", "-d", "--build", "osmo-tools"), + check=True, + capture_output=True, + text=True, + env=_compose_env(), + timeout=30 * 60, + ) + + try: + bootstrap_env = { + key: value + for key in ( + "OSMO_DOCKERHUB_USERNAME", + "OSMO_DOCKERHUB_PASSWORD", + "OSMO_KAI_SCHEDULER_VERSION", + "OSMO_PRELOAD_DOCKERHUB_IMAGES", + "OSMO_PRELOAD_PULL_RETRIES", + "OSMO_QUICK_START_CHART_VERSION", + ) + if (value := os.environ.get(key)) + } + bootstrap_env["OSMO_TEST_PYTHON_IMAGE"] = OSMO_TEST_PYTHON_IMAGE + bootstrap = exec_on_service( + "osmo-tools", + ["bash", "tests/backends/osmo/scripts/bootstrap_osmo.sh"], + env=bootstrap_env or None, + timeout=BOOTSTRAP_TIMEOUT_SECONDS, + ) + if bootstrap.returncode != 0: + raise RuntimeError( + "OSMO bootstrap failed.\n" + f"stdout:\n{textwrap.indent(bootstrap.stdout, ' ')}\n" + f"stderr:\n{textwrap.indent(bootstrap.stderr, ' ')}" + ) + patch_local_osmo_data_override_url(LOCALSTACK_HOST_OVERRIDE_URL) + yield { + "base_url": BASE_URL, + "container_downloads_dir": str(CONTAINER_DOWNLOADS_DIR), + "container_projects_dir": str(CONTAINER_PROJECTS_DIR), + "container_wheels_dir": str(CONTAINER_WHEELS_DIR), + } + finally: + exec_on_service( + "osmo-tools", + ["bash", "tests/backends/osmo/scripts/destroy_osmo.sh"], + timeout=10 * 60, + ) + run_docker( + _compose_args("down", "-v"), + check=False, + capture_output=True, + text=True, + timeout=10 * 60, + ) + + +@pytest.fixture(scope="session") +def osmo_runtime_wheel(osmo_harness: dict[str, str]) -> dict[str, str]: + del osmo_harness + shutil.rmtree(HOST_WHEELS_DIR, ignore_errors=True) + HOST_WHEELS_DIR.mkdir(parents=True, exist_ok=True) + + result = subprocess.run( + ["./scripts/build_wheel_with_bins.sh", str(HOST_WHEELS_DIR)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + timeout=20 * 60, + ) + if result.returncode != 0: + raise RuntimeError( + "Failed to build roar wheel for OSMO e2e runtime install.\n" + f"stdout:\n{textwrap.indent(result.stdout, ' ')}\n" + f"stderr:\n{textwrap.indent(result.stderr, ' ')}" + ) + + wheels = sorted(HOST_WHEELS_DIR.glob("*.whl")) + if not wheels: + raise RuntimeError(f"wheel build did not produce a wheel in {HOST_WHEELS_DIR}") + wheel_path = wheels[-1] + container_wheel_path = CONTAINER_WHEELS_DIR / wheel_path.name + install_result = exec_on_service( + "osmo-tools", + [ + "bash", + "-lc", + ( + "python3 -m pip install --disable-pip-version-check --no-input " + f"--force-reinstall {shlex.quote(str(container_wheel_path))}" + ), + ], + timeout=15 * 60, + ) + if install_result.returncode != 0: + raise RuntimeError( + "Failed to install roar wheel into osmo-tools.\n" + f"stdout:\n{textwrap.indent(install_result.stdout, ' ')}\n" + f"stderr:\n{textwrap.indent(install_result.stderr, ' ')}" + ) + return { + "host_path": str(wheel_path), + "container_path": str(container_wheel_path), + "cluster_url": publish_runtime_artifact_service( + service_name="roar-runtime-wheel", + host_artifact_path=wheel_path, + artifact_filename=wheel_path.name, + ), + } + + +def wait_for_workflow_completion(workflow_id: str) -> dict[str, object]: + return wait_for_workflow_status(workflow_id, lambda status: status == "COMPLETED") + + +def wait_for_workflow_status( + workflow_id: str, + predicate: Callable[[str], bool], + *, + timeout_seconds: int = QUERY_TIMEOUT_SECONDS, +) -> dict[str, object]: + deadline = time.monotonic() + timeout_seconds + last_payload: dict[str, object] | None = None + + while time.monotonic() < deadline: + result = osmo_exec( + ["osmo", "workflow", "query", workflow_id, "--format-type", "json"], + timeout=120, + check=False, + ) + if result.returncode == 0: + payload = json.loads(result.stdout) + last_payload = payload + status = str(payload.get("status", "")) + if predicate(status): + return payload + if status.startswith("FAILED"): + raise AssertionError( + f"Workflow {workflow_id} failed with status {status}.\n" + f"payload={json.dumps(payload, indent=2)}" + ) + time.sleep(POLL_INTERVAL_SECONDS) + + raise AssertionError( + f"Timed out waiting for workflow {workflow_id}. " + f"last_payload={json.dumps(last_payload, indent=2) if last_payload else 'none'}" + ) + + +@contextlib.contextmanager +def osmo_port_forward( + workflow_id: str, + task: str, + *, + local_port: int, + task_port: int, + host: str = "127.0.0.1", +) -> dict[str, str]: + HOST_DOWNLOADS_DIR.mkdir(parents=True, exist_ok=True) + log_path = HOST_DOWNLOADS_DIR / f"port-forward-{workflow_id}-{task}-{local_port}.log" + command = _compose_exec_command( + "osmo-tools", + [ + "osmo", + "workflow", + "port-forward", + workflow_id, + task, + "--host", + host, + "--port", + f"{local_port}:{task_port}", + ], + ) + + with log_path.open("w+", encoding="utf-8") as log_file: + process = _popen_docker(command, stdout=log_file, stderr=subprocess.STDOUT) + try: + deadline = time.monotonic() + PORT_FORWARD_TIMEOUT_SECONDS + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError( + "OSMO port-forward exited early.\n" + f"log:\n{log_path.read_text(encoding='utf-8')}" + ) + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(1) + if sock.connect_ex((host, local_port)) == 0: + break + time.sleep(1) + else: + raise RuntimeError( + "Timed out waiting for OSMO port-forward.\n" + f"log:\n{log_path.read_text(encoding='utf-8')}" + ) + + yield { + "host": host, + "local_port": str(local_port), + "task_port": str(task_port), + "url": f"http://{host}:{local_port}", + "log_path": str(log_path), + } + finally: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) diff --git a/tests/backends/osmo/docker-compose.yml b/tests/backends/osmo/docker-compose.yml new file mode 100644 index 00000000..6a3234f3 --- /dev/null +++ b/tests/backends/osmo/docker-compose.yml @@ -0,0 +1,28 @@ +services: + osmo-tools: + build: + context: ../../../ + dockerfile: tests/backends/osmo/Dockerfile + args: + PYTHON_IMAGE: ${OSMO_TEST_PYTHON_IMAGE:-public.ecr.aws/docker/library/python:3.11-slim} + working_dir: /workspace/roar + environment: + AWS_DEFAULT_REGION: us-east-1 + AWS_ENDPOINT_URL_S3: http://127.0.0.1:34566 + AWS_S3_FORCE_PATH_STYLE: "true" + OSMO_BASE_URL: http://quick-start.osmo:38080 + OSMO_KIND_CLUSTER_NAME: roar-osmo-e2e + OSMO_LOCALSTACK_PORT: "34566" + OSMO_TEST_PYTHON_IMAGE: ${OSMO_TEST_PYTHON_IMAGE:-public.ecr.aws/docker/library/python:3.11-slim} + REPO_ROOT: /workspace/roar + network_mode: host + extra_hosts: + - "quick-start.osmo:host-gateway" + volumes: + - ../../../:/workspace/roar + - /var/run/docker.sock:/var/run/docker.sock + - osmo-home:/root + command: ["bash", "-lc", "sleep infinity"] + +volumes: + osmo-home: diff --git a/tests/backends/osmo/kind-osmo-cluster-config.yaml b/tests/backends/osmo/kind-osmo-cluster-config.yaml new file mode 100644 index 00000000..734fcdbb --- /dev/null +++ b/tests/backends/osmo/kind-osmo-cluster-config.yaml @@ -0,0 +1,49 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "node_group=ingress" + extraPortMappings: + - containerPort: 30080 + hostPort: 80 + protocol: TCP + - containerPort: 30080 + hostPort: 38080 + protocol: TCP + - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "node_group=kai-scheduler" + - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "node_group=data" + extraMounts: + - hostPath: /tmp/roar-osmo-localstack + containerPath: /var/lib/localstack + - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "node_group=service" + - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "node_group=compute" diff --git a/tests/backends/osmo/scripts/bootstrap_osmo.sh b/tests/backends/osmo/scripts/bootstrap_osmo.sh new file mode 100644 index 00000000..f04df5b4 --- /dev/null +++ b/tests/backends/osmo/scripts/bootstrap_osmo.sh @@ -0,0 +1,443 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cluster_name="${OSMO_KIND_CLUSTER_NAME:-roar-osmo-e2e}" +base_url="${OSMO_BASE_URL:-http://quick-start.osmo:38080}" +repo_root="${REPO_ROOT:-/workspace/roar}" +kind_config="${repo_root}/tests/backends/osmo/kind-osmo-cluster-config.yaml" +kai_scheduler_version="${OSMO_KAI_SCHEDULER_VERSION:-v0.12.10}" +quick_start_chart_version="${OSMO_QUICK_START_CHART_VERSION:-1.0.1}" +localstack_port="${OSMO_LOCALSTACK_PORT:-34566}" +localstack_forward_log="/tmp/osmo-localstack-port-forward.log" +localstack_forward_pid="/tmp/osmo-localstack-port-forward.pid" +localstack_cluster_url="http://localstack-s3.osmo:4566" +localstack_override_url="http://127.0.0.1:${localstack_port}" +dockerhub_username="${OSMO_DOCKERHUB_USERNAME:-}" +dockerhub_password="${OSMO_DOCKERHUB_PASSWORD:-}" +test_python_image="${OSMO_TEST_PYTHON_IMAGE:-public.ecr.aws/docker/library/python:3.11-slim}" +preload_images="${OSMO_PRELOAD_DOCKERHUB_IMAGES:-ghcr.io/nvidia/kai-scheduler/admission:${kai_scheduler_version} ghcr.io/nvidia/kai-scheduler/binder:${kai_scheduler_version} ghcr.io/nvidia/kai-scheduler/operator:${kai_scheduler_version} ghcr.io/nvidia/kai-scheduler/podgrouper:${kai_scheduler_version} ghcr.io/nvidia/kai-scheduler/podgroupcontroller:${kai_scheduler_version} ghcr.io/nvidia/kai-scheduler/queuecontroller:${kai_scheduler_version} ghcr.io/nvidia/kai-scheduler/scheduler:${kai_scheduler_version} postgres:15.1 redis:7.0 gresau/localstack-persist:latest busybox:1.37.0 alpine:3.18 alpine/k8s:1.28.4 alpine/curl:8.14.1 amazon/aws-cli:2.15.33 ${test_python_image}}" +preload_pull_retries="${OSMO_PRELOAD_PULL_RETRIES:-3}" +kind_nodes_csv="" + +cleanup_port_forward() { + if [[ -f "${localstack_forward_pid}" ]]; then + kill "$(cat "${localstack_forward_pid}")" >/dev/null 2>&1 || true + rm -f "${localstack_forward_pid}" + fi +} + +dockerhub_login() { + if [[ -z "${dockerhub_username}" || -z "${dockerhub_password}" ]]; then + return 0 + fi + + printf '%s' "${dockerhub_password}" | docker login \ + --username "${dockerhub_username}" \ + --password-stdin >/dev/null +} + +resolve_preload_pull_image() { + local image="$1" + case "${image}" in + public.ecr.aws/docker/library/*) + printf '%s\n' "${image}" + ;; + postgres:*|redis:*|busybox:*|alpine:*|python:*-slim) + printf 'public.ecr.aws/docker/library/%s\n' "${image}" + ;; + gresau/localstack-persist:*|alpine/k8s:*|alpine/curl:*|amazon/aws-cli:*) + printf 'mirror.gcr.io/%s\n' "${image}" + ;; + *) + printf '%s\n' "${image}" + ;; + esac +} + +docker_pull_with_retries() { + local image="$1" + local attempt=1 + while true; do + if docker pull "${image}" >/dev/null; then + return 0 + fi + if (( attempt >= preload_pull_retries )); then + echo "failed to pull ${image} after ${preload_pull_retries} attempts" >&2 + return 1 + fi + sleep $((attempt * 5)) + attempt=$((attempt + 1)) + done +} + +preload_dockerhub_images_into_kind() { + local image + local pull_image + local target_nodes_csv + if [[ -z "${kind_nodes_csv}" ]]; then + echo "kind node list is empty; cannot preload Docker Hub images" >&2 + exit 1 + fi + for image in ${preload_images}; do + target_nodes_csv="$(resolve_preload_nodes_csv "${image}")" + pull_image="$(resolve_preload_pull_image "${image}")" + docker_pull_with_retries "${pull_image}" + if [[ "${pull_image}" != "${image}" ]]; then + docker tag "${pull_image}" "${image}" + fi + kind load docker-image \ + --name "${cluster_name}" \ + --nodes "${target_nodes_csv}" \ + "${image}" >/dev/null + done +} + +resolve_preload_nodes_csv() { + local image="$1" + local selector="" + case "${image}" in + postgres:*|redis:*|gresau/localstack-persist:*) + selector="node_group=data" + ;; + alpine/k8s:*) + selector="node_group=service" + ;; + python:*-slim|public.ecr.aws/docker/library/python:*-slim) + selector="node_group=compute" + ;; + *) + ;; + esac + + if [[ -z "${selector}" ]]; then + printf '%s\n' "${kind_nodes_csv}" + return 0 + fi + + local selected_nodes_csv + selected_nodes_csv="$(kubectl get nodes -l "${selector}" -o jsonpath='{range .items[*]}{.metadata.name}{","}{end}')" + selected_nodes_csv="${selected_nodes_csv%,}" + if [[ -z "${selected_nodes_csv}" ]]; then + printf '%s\n' "${kind_nodes_csv}" + return 0 + fi + printf '%s\n' "${selected_nodes_csv}" +} + +wait_for_osmo_dev_login() { + local deadline=$((SECONDS + 600)) + while (( SECONDS < deadline )); do + if osmo login "${base_url}" --method=dev --username=testuser >/tmp/osmo-login.out 2>/tmp/osmo-login.err; then + return 0 + fi + sleep 5 + done + + echo "Timed out waiting for OSMO dev login" >&2 + cat /tmp/osmo-login.err >&2 || true + return 1 +} + +patch_local_osmo_data_override_url() { + cat </root/.config/osmo/config.yaml +auth: + data: + s3://osmo: + access_key: test + access_key_id: test + override_url: ${localstack_override_url} + region: us-east-1 +EOF +} + +patch_osmo_pod_templates_for_kind() { + local response + response="$( + curl \ + --max-time 30 \ + --retry 3 \ + --retry-delay 5 \ + --fail-with-body \ + -w 'HTTP_STATUS:%{http_code}' \ + -X PUT \ + -H "Content-Type: application/json" \ + -H "x-osmo-user: testuser" \ + "${base_url}/api/configs/pod_template" \ + -d '{ + "configs": { + "default_compute": { + "spec": { + "containers": [ + { + "name": "{{USER_CONTAINER_NAME}}", + "env": [ + { + "name": "AWS_ENDPOINT_URL_S3", + "value": "http://localstack-s3.osmo:4566" + }, + { + "name": "AWS_S3_FORCE_PATH_STYLE", + "value": "true" + }, + { + "name": "AWS_DEFAULT_REGION", + "value": "us-east-1" + }, + { + "name": "OSMO_LOGIN_DEV", + "value": "true" + }, + { + "name": "OSMO_SKIP_DATA_AUTH", + "value": "1" + } + ] + }, + { + "name": "osmo-ctrl", + "env": [ + { + "name": "AWS_ENDPOINT_URL_S3", + "value": "http://localstack-s3.osmo:4566" + }, + { + "name": "AWS_S3_FORCE_PATH_STYLE", + "value": "true" + }, + { + "name": "AWS_DEFAULT_REGION", + "value": "us-east-1" + }, + { + "name": "OSMO_LOGIN_DEV", + "value": "true" + }, + { + "name": "OSMO_SKIP_DATA_AUTH", + "value": "1" + } + ] + } + ], + "nodeSelector": { + "node_group": "compute" + } + } + }, + "default_user": { + "spec": { + "containers": [ + { + "name": "{{USER_CONTAINER_NAME}}", + "resources": { + "limits": { + "cpu": "{{USER_CPU}}", + "memory": "{{USER_MEMORY}}", + "ephemeral-storage": "{{USER_STORAGE}}" + }, + "requests": { + "cpu": "{{USER_CPU}}", + "memory": "{{USER_MEMORY}}", + "ephemeral-storage": "{{USER_STORAGE}}" + } + } + } + ] + } + } + }, + "description": "Adapt OSMO pod templates for KIND-based roar tests" + }' + )" + + local http_code body + http_code="$(printf '%s' "${response}" | grep -o 'HTTP_STATUS:[0-9]*' | cut -d: -f2)" + body="$(printf '%s' "${response}" | sed 's/HTTP_STATUS:[0-9]*$//')" + if [[ -z "${http_code}" || "${http_code}" -ge 400 ]]; then + echo "Failed to patch OSMO pod templates for KIND." >&2 + echo "HTTP status: ${http_code:-unknown}" >&2 + echo "Response body: ${body}" >&2 + return 1 + fi +} + +ensure_backend_operator_token() { + osmo user create backend-operator --roles osmo-backend --format-type json >/tmp/osmo-backend-user.out 2>/tmp/osmo-backend-user.err || true + osmo user update testuser --add-roles osmo-backend --format-type json >/tmp/osmo-backend-role.out 2>/tmp/osmo-backend-role.err || true + + local token_name="backend-operator-token-$(date +%s)" + local token_json + token_json="$( + osmo token set "${token_name}" \ + --user backend-operator \ + --roles osmo-backend \ + --format-type json + )" + local token + token="$( + printf '%s' "${token_json}" \ + | python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])' + )" + + kubectl patch secret backend-operator-token \ + -n osmo \ + --type merge \ + -p "{\"stringData\":{\"token\":\"${token}\"}}" >/dev/null +} + +patch_backend_operator_deployments() { + local patch_file="/tmp/osmo-remove-wait-token.json" + cat <<'JSON' >"${patch_file}" +[ + {"op":"remove","path":"/spec/template/spec/initContainers/1"} +] +JSON + + kubectl patch deployment osmo-osmo-backend-listener \ + -n osmo \ + --type json \ + --patch-file "${patch_file}" >/dev/null || true + kubectl patch deployment osmo-osmo-backend-worker \ + -n osmo \ + --type json \ + --patch-file "${patch_file}" >/dev/null || true + + kubectl rollout restart deployment/osmo-osmo-backend-listener -n osmo >/dev/null + kubectl rollout restart deployment/osmo-osmo-backend-worker -n osmo >/dev/null + kubectl rollout status deployment/osmo-osmo-backend-listener -n osmo --timeout=5m + kubectl rollout status deployment/osmo-osmo-backend-worker -n osmo --timeout=5m +} + +main() { + kind delete cluster --name "${cluster_name}" >/dev/null 2>&1 || true + docker ps -a --format '{{.Names}}' | grep "^${cluster_name}-" | xargs -r docker rm -f >/dev/null 2>&1 || true + cleanup_port_forward + + helm repo add osmo https://helm.ngc.nvidia.com/nvidia/osmo >/dev/null 2>&1 || true + helm repo update >/dev/null + + dockerhub_login + kind create cluster --name "${cluster_name}" --config "${kind_config}" + deadline=$((SECONDS + 30)) + while (( SECONDS < deadline )); do + kind_nodes_csv="$(kind get nodes --name "${cluster_name}" | paste -sd, -)" + if [[ -n "${kind_nodes_csv}" ]]; then + break + fi + sleep 1 + done + + kubectl wait --for=condition=Ready node --all --timeout=5m + preload_dockerhub_images_into_kind + + for node in $(docker ps --format '{{.Names}}' | grep "^${cluster_name}-"); do + docker exec "${node}" sysctl -w \ + fs.inotify.max_user_instances=8192 \ + fs.inotify.max_user_watches=524288 >/dev/null + done + + kubectl delete pod -n kube-system -l k8s-app=kube-proxy --ignore-not-found >/dev/null + kubectl rollout status daemonset/kube-proxy -n kube-system --timeout=5m + + helm upgrade --install kai-scheduler \ + oci://ghcr.io/nvidia/kai-scheduler/kai-scheduler \ + --version "${kai_scheduler_version}" \ + --create-namespace \ + -n kai-scheduler \ + --set global.nodeSelector.node_group=kai-scheduler \ + --set "scheduler.additionalArgs[0]=--default-staleness-grace-period=-1s" \ + --set "scheduler.additionalArgs[1]=--update-pod-eviction-condition=true" \ + --wait \ + --timeout 15m + + helm upgrade --install osmo osmo/quick-start \ + --version "${quick_start_chart_version}" \ + --namespace osmo \ + --create-namespace \ + --timeout 20m + + kubectl patch configmap quick-start \ + -n osmo \ + --type merge \ + -p '{"data":{"proxy-body-size":"32m"}}' >/dev/null + kubectl rollout restart deployment/quick-start -n osmo >/dev/null + kubectl rollout status deployment/quick-start -n osmo --timeout=5m + + kubectl rollout status deployment/localstack-s3 -n osmo --timeout=5m + wait_for_osmo_dev_login + ensure_backend_operator_token + patch_backend_operator_deployments + patch_osmo_pod_templates_for_kind + + nohup kubectl port-forward \ + --address 127.0.0.1 \ + -n osmo \ + service/localstack-s3 \ + "${localstack_port}:4566" >"${localstack_forward_log}" 2>&1 & + echo $! >"${localstack_forward_pid}" + + deadline=$((SECONDS + 120)) + while (( SECONDS < deadline )); do + if curl -fsS "${localstack_override_url}/_localstack/health" >/dev/null 2>&1; then + break + fi + sleep 2 + done + + if ! curl -fsS "${localstack_override_url}/_localstack/health" >/dev/null 2>&1; then + echo "Timed out waiting for LocalStack port-forward" >&2 + cat "${localstack_forward_log}" >&2 || true + exit 1 + fi + + deadline=$((SECONDS + 900)) + while (( SECONDS < deadline )); do + if wait_for_osmo_dev_login; then + if osmo pool list --format-type json >/tmp/osmo-pools.json 2>/tmp/osmo-pools.err; then + if python - <<'PY' +import json +from pathlib import Path + +payload = json.loads(Path("/tmp/osmo-pools.json").read_text()) +pools = { + pool.get("name"): pool + for node_set in payload.get("node_sets", []) + for pool in node_set.get("pools", []) +} +pool = pools.get("default") +raise SystemExit(0 if pool and pool.get("status") == "ONLINE" else 1) +PY + then + if osmo credential set quick-start-localstack \ + --type DATA \ + --payload \ + endpoint=s3://osmo \ + override_url="${localstack_cluster_url}" \ + region=us-east-1 \ + access_key_id=test \ + access_key=test >/tmp/osmo-credential.out 2>/tmp/osmo-credential.err \ + && osmo profile set bucket osmo >/tmp/osmo-profile.out 2>/tmp/osmo-profile.err + then + patch_local_osmo_data_override_url + exit 0 + fi + fi + fi + fi + sleep 5 + done + + echo "Timed out waiting for OSMO to become ready" >&2 + kubectl get pods -A >&2 || true + cat /tmp/osmo-login.err >&2 || true + cat /tmp/osmo-pools.err >&2 || true + cat "${localstack_forward_log}" >&2 || true + cat /tmp/osmo-credential.err >&2 || true + cat /tmp/osmo-profile.err >&2 || true + exit 1 +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/tests/backends/osmo/scripts/destroy_osmo.sh b/tests/backends/osmo/scripts/destroy_osmo.sh new file mode 100644 index 00000000..43df5ab1 --- /dev/null +++ b/tests/backends/osmo/scripts/destroy_osmo.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cluster_name="${OSMO_KIND_CLUSTER_NAME:-roar-osmo-e2e}" +localstack_forward_pid="/tmp/osmo-localstack-port-forward.pid" + +if [[ -f "${localstack_forward_pid}" ]]; then + kill "$(cat "${localstack_forward_pid}")" >/dev/null 2>&1 || true + rm -f "${localstack_forward_pid}" +fi + +kind delete cluster --name "${cluster_name}" >/dev/null 2>&1 || true diff --git a/tests/backends/osmo/test_osmo_ray_smoke.py b/tests/backends/osmo/test_osmo_ray_smoke.py new file mode 100644 index 00000000..d1c82db2 --- /dev/null +++ b/tests/backends/osmo/test_osmo_ray_smoke.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import textwrap +import time +import urllib.error +import urllib.request +import uuid + +import pytest + +from .conftest import ( + REPO_ROOT, + osmo_exec, + osmo_port_forward, + wait_for_workflow_status, +) + +pytestmark = [pytest.mark.e2e, pytest.mark.osmo_e2e] + +RAY_DASHBOARD_PORT = 18265 + + +def _host_ray_cli() -> str: + candidates = [ + REPO_ROOT / ".venv" / "bin" / "ray", + REPO_ROOT.parent / "roar" / ".venv" / "bin" / "ray", + ] + for candidate in candidates: + if candidate.exists(): + return str(candidate) + ray_cli = shutil.which("ray") + if ray_cli: + return ray_cli + pytest.skip("ray CLI is not available on the host") + + +def _host_ray_version(ray_cli: str) -> str: + result = subprocess.run( + [ray_cli, "--version"], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + match = re.search(r"version\s+([0-9.]+)", result.stdout) + if not match: + raise AssertionError(f"could not parse Ray version from: {result.stdout!r}") + return match.group(1) + + +def _wait_for_dashboard(url: str) -> dict[str, object]: + deadline = time.monotonic() + 5 * 60 + last_error = "" + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(f"{url}/api/version", timeout=10) as response: + payload = json.load(response) + if payload.get("ray_version"): + return payload + except (OSError, urllib.error.URLError, json.JSONDecodeError) as exc: + last_error = str(exc) + time.sleep(2) + raise AssertionError(f"timed out waiting for Ray dashboard at {url}: {last_error}") + + +def _wait_for_task_log_marker(workflow_id: str, task: str, marker: str) -> str: + deadline = time.monotonic() + 10 * 60 + last_output = "" + while time.monotonic() < deadline: + try: + result = osmo_exec( + ["osmo", "workflow", "logs", workflow_id, "--task", task], + timeout=10, + check=False, + ) + output = f"{result.stdout}\n{result.stderr}" + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout if isinstance(exc.stdout, str) else (exc.stdout or b"").decode() + stderr = exc.stderr if isinstance(exc.stderr, str) else (exc.stderr or b"").decode() + output = f"{stdout}\n{stderr}" + if marker in output: + return output + if output.strip(): + last_output = output + time.sleep(5) + raise AssertionError( + f"timed out waiting for {marker!r} in logs for {workflow_id}/{task}.\n" + f"last_output:\n{textwrap.indent(last_output, ' ')}" + ) + + +def test_osmo_ray_cluster_accepts_host_submitted_ray_job( + osmo_harness: dict[str, str], +) -> None: + del osmo_harness + ray_cli = _host_ray_cli() + ray_version = _host_ray_version(ray_cli) + workflow_name = f"roar-osmo-ray-{uuid.uuid4().hex[:8]}" + + submit = osmo_exec( + [ + "osmo", + "workflow", + "submit", + "tests/backends/osmo/workflows/ray_cluster.yaml", + "--pool", + "default", + "--set-string", + f"workflow_name={workflow_name}", + f"ray_version={ray_version}", + "--format-type", + "json", + ], + timeout=10 * 60, + ) + workflow_id = str(json.loads(submit.stdout)["name"]) + + try: + wait_for_workflow_status(workflow_id, lambda status: status == "RUNNING") + _wait_for_task_log_marker(workflow_id, "master", "ROAR_OSMO_RAY_HEAD_READY") + + with osmo_port_forward( + workflow_id, + "master", + local_port=RAY_DASHBOARD_PORT, + task_port=8265, + ) as forwarded: + dashboard_payload = _wait_for_dashboard(forwarded["url"]) + assert dashboard_payload.get("ray_version") == ray_version + + result = subprocess.run( + [ + ray_cli, + "job", + "submit", + "--address", + forwarded["url"], + "--log-style", + "record", + "--log-color", + "false", + "--working-dir", + "tests/backends/osmo/workloads", + "--", + "python", + "ray_smoke_job.py", + ], + cwd=REPO_ROOT, + env={**dict(os.environ), "ROAR_EXPECTED_RAY_NODES": "1"}, + capture_output=True, + text=True, + timeout=10 * 60, + ) + + submit_output = f"{result.stdout}\n{result.stderr}" + job_id_match = re.search(r"Job '([^']+)'", submit_output) + assert job_id_match is not None, submit_output + job_id = job_id_match.group(1) + + logs_result = subprocess.run( + [ + ray_cli, + "job", + "logs", + "--address", + forwarded["url"], + "--log-style", + "record", + "--log-color", + "false", + job_id, + ], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=5 * 60, + ) + + logs_output = f"{logs_result.stdout}\n{logs_result.stderr}" + output = f"{submit_output}\n{logs_result.stdout}\n{logs_result.stderr}" + assert result.returncode == 0, ( + f"ray job submit failed (rc={result.returncode}).\n" + f"stdout/stderr:\n{textwrap.indent(submit_output, ' ')}" + ) + assert logs_result.returncode == 0, ( + f"ray job logs failed (rc={logs_result.returncode}).\n" + f"stdout/stderr:\n{textwrap.indent(logs_output, ' ')}" + ) + marker_line = next( + (line for line in output.splitlines() if "ROAR_OSMO_RAY_OK " in line), + None, + ) + assert marker_line is not None, output + payload = json.loads(marker_line.split("ROAR_OSMO_RAY_OK ", 1)[1]) + assert payload["node_count"] >= 1, output + assert payload["total"] == 55, output + finally: + osmo_exec( + ["osmo", "workflow", "cancel", workflow_id], + timeout=5 * 60, + check=False, + ) diff --git a/tests/backends/osmo/test_osmo_smoke.py b/tests/backends/osmo/test_osmo_smoke.py new file mode 100644 index 00000000..97b64e70 --- /dev/null +++ b/tests/backends/osmo/test_osmo_smoke.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import json +import shutil +import sqlite3 +import subprocess +import textwrap +import uuid +from pathlib import Path + +import pytest + +from .conftest import ( + HOST_PROJECTS_DIR, + OSMO_TEST_PYTHON_IMAGE, + allow_git_safe_directory, + container_repo_path, + restore_host_path_ownership, + roar_exec, +) + +pytestmark = [pytest.mark.e2e, pytest.mark.osmo_e2e] +OSMO_SMOKE_TASK_IMAGE = OSMO_TEST_PYTHON_IMAGE + + +def _run_host(args: list[str], *, cwd: Path) -> None: + result = subprocess.run( + args, + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + "host command failed:\n" + f"command: {' '.join(args)}\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + + +def _prepare_product_project( + project_dir: Path, + *, + runtime_install_requirement: str, + task_image: str, +) -> None: + shutil.rmtree(project_dir, ignore_errors=True) + project_dir.mkdir(parents=True, exist_ok=True) + + workflow_path = project_dir / "workflow.yaml" + task_path = project_dir / "task.py" + task_contents = """ +from __future__ import annotations + +import sys +from pathlib import Path + + +def main() -> int: + output_path = Path(sys.argv[1]) + output_path.parent.mkdir(parents=True, exist_ok=True) + message = "ROAR_OSMO_BASIC_OK" + print(message) + output_path.write_text(f"{message}\\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +""".strip() + + workflow_contents = ( + """ +workflow: + name: {{ workflow_name }} + resources: + default: + cpu: 1 + memory: 1Gi + storage: 1Gi + tasks: + - name: basic + image: {{ task_image }} + command: ["python3"] + args: ["/workspace/task.py", "{{output}}/result.txt"] + files: + - path: /workspace/task.py + contents: | +""".strip("\n") + + "\n" + + textwrap.indent(task_contents, " " * 12) + + """ + outputs: + - dataset: + name: {{ output_dataset }} + path: result.txt + +default-values: + workflow_name: roar-osmo-basic + output_dataset: roar-osmo-basic-output + task_image: """ + + json.dumps(task_image) + + "\n" + ) + workflow_path.write_text(workflow_contents, encoding="utf-8") + task_path.write_text(task_contents + "\n", encoding="utf-8") + + _run_host(["git", "init"], cwd=project_dir) + _run_host(["git", "config", "user.email", "test@example.com"], cwd=project_dir) + _run_host(["git", "config", "user.name", "Test User"], cwd=project_dir) + + container_project_dir = container_repo_path(project_dir) + allow_git_safe_directory(container_project_dir) + roar_exec(["init", "-y"], cwd=str(container_project_dir), timeout=5 * 60) + for key, value in ( + ("osmo.wait_for_completion", "true"), + ("osmo.download_declared_outputs", "true"), + ("osmo.ingest_lineage_bundles", "true"), + ("osmo.poll_interval_seconds", "2.0"), + ("osmo.query_timeout_seconds", "900"), + ("osmo.runtime_install_requirement", runtime_install_requirement), + ): + roar_exec( + ["config", "set", key, value], + cwd=str(container_project_dir), + timeout=5 * 60, + ) + + _run_host(["git", "add", "-A"], cwd=project_dir) + _run_host(["git", "commit", "-m", "Initialize OSMO product project"], cwd=project_dir) + + +def _host_visible_path(path: Path, *, project_dir: Path) -> Path: + container_project_dir = container_repo_path(project_dir) + if path.is_relative_to(container_project_dir): + return project_dir / path.relative_to(container_project_dir) + return path + + +def test_osmo_basic_workflow_submit_and_complete( + osmo_harness: dict[str, str], + osmo_runtime_wheel: dict[str, str], +) -> None: + del osmo_harness + workflow_name = f"roar-osmo-basic-{uuid.uuid4().hex[:8]}" + output_dataset = f"{workflow_name}-output" + project_dir = HOST_PROJECTS_DIR / workflow_name + _prepare_product_project( + project_dir, + runtime_install_requirement=osmo_runtime_wheel["cluster_url"], + task_image=OSMO_SMOKE_TASK_IMAGE, + ) + + submit = roar_exec( + [ + "run", + "osmo", + "workflow", + "submit", + "workflow.yaml", + "--pool", + "default", + "--set-string", + f"workflow_name={workflow_name}", + "--set-string", + f"output_dataset={output_dataset}", + ], + cwd=str(container_repo_path(project_dir)), + timeout=15 * 60, + ) + restore_host_path_ownership(container_repo_path(project_dir)) + + assert workflow_name in submit.stdout + + db_path = project_dir / ".roar" / "roar.db" + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + job = conn.execute( + """ + SELECT id, job_uid, execution_backend, execution_role, exit_code + FROM jobs + WHERE execution_role = 'submit' + ORDER BY id DESC + LIMIT 1 + """ + ).fetchone() + child_jobs = conn.execute( + """ + SELECT id, parent_job_uid, execution_backend, execution_role, command + FROM jobs + WHERE job_type = 'osmo_task' + ORDER BY id ASC + """ + ).fetchall() + assert job is not None + output_rows = conn.execute( + """ + SELECT path + FROM job_outputs + WHERE job_id = ? + ORDER BY path + """, + (int(job["id"]),), + ).fetchall() + finally: + conn.close() + + assert job["execution_backend"] == "osmo" + assert job["execution_role"] == "submit" + assert job["exit_code"] == 0 + assert len(child_jobs) >= 1 + assert child_jobs[0]["parent_job_uid"] == job["job_uid"] + assert child_jobs[0]["execution_backend"] == "osmo" + assert child_jobs[0]["execution_role"] == "task" + + output_paths = [ + _host_visible_path(Path(str(row["path"])), project_dir=project_dir) for row in output_rows + ] + receipt_path = next(path for path in output_paths if "submissions" in str(path)) + query_path = next(path for path in output_paths if path.name == "query-COMPLETED.json") + downloaded_result = next(path for path in output_paths if path.name == "result.txt") + downloaded_bundle = next(path for path in output_paths if path.name == "roar-fragments.json") + + payload = json.loads(receipt_path.read_text(encoding="utf-8")) + assert payload["osmo_submit"]["workflow_status"] == "COMPLETED" + assert payload["osmo_submit"]["lineage_reconstitution"]["fragments_processed"] >= 1 + + query_payload = json.loads(query_path.read_text(encoding="utf-8")) + assert query_payload["status"] == "COMPLETED" + + assert downloaded_result.read_text(encoding="utf-8").strip() == "ROAR_OSMO_BASIC_OK" + bundle_payload = json.loads(downloaded_bundle.read_text(encoding="utf-8")) + assert len(bundle_payload.get("fragments", [])) >= 1 diff --git a/tests/backends/osmo/workflows/basic.yaml b/tests/backends/osmo/workflows/basic.yaml new file mode 100644 index 00000000..c66a3355 --- /dev/null +++ b/tests/backends/osmo/workflows/basic.yaml @@ -0,0 +1,23 @@ +workflow: + name: {{ workflow_name }} + resources: + default: + cpu: 1 + memory: 1Gi + storage: 1Gi + tasks: + - name: basic + image: python:3.11-slim + command: ["python"] + args: ["/workspace/basic_task.py", "{{output}}/result.txt"] + files: + - localpath: ../workloads/basic_task.py + path: /workspace/basic_task.py + outputs: + - dataset: + name: {{ output_dataset }} + path: result.txt + +default-values: + workflow_name: roar-osmo-basic + output_dataset: roar-osmo-basic-output diff --git a/tests/backends/osmo/workflows/ray_cluster.yaml b/tests/backends/osmo/workflows/ray_cluster.yaml new file mode 100644 index 00000000..8e74aa70 --- /dev/null +++ b/tests/backends/osmo/workflows/ray_cluster.yaml @@ -0,0 +1,54 @@ +workflow: + name: {{workflow_name}} + resources: {{resources}} + timeout: + exec_timeout: {{timeout}} + tasks: + - name: master + image: public.ecr.aws/docker/library/python:3.11-slim + command: [bash, /tmp/master.sh] + files: + - path: /tmp/master.sh + contents: | + set -euo pipefail + python3 -m pip install --no-cache-dir "ray[default]=={{ray_version}}" + export RAY_USAGE_STATS_ENABLED=0 + ray start \ + --head \ + --port={{ray_port}} \ + --dashboard-host=0.0.0.0 \ + --dashboard-port={{dashboard_port}} \ + --num-cpus={{ray_cpus_per_node}} + python3 - <<'PY' + import json + import time + import urllib.request + + url = "http://127.0.0.1:{{dashboard_port}}/api/version" + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=5) as response: + payload = json.load(response) + if payload.get("ray_version"): + break + except Exception: + time.sleep(2) + else: + raise SystemExit("ray dashboard did not become ready") + PY + echo "ROAR_OSMO_RAY_HEAD_READY" + sleep infinity + +default-values: + workflow_name: roar-osmo-ray + ray_version: "2.44.1" + ray_port: 6379 + dashboard_port: 8265 + ray_cpus_per_node: 1 + resources: + default: + cpu: 1 + memory: 2Gi + storage: 4Gi + timeout: 1h diff --git a/tests/backends/osmo/workloads/basic_task.py b/tests/backends/osmo/workloads/basic_task.py new file mode 100644 index 00000000..037643f5 --- /dev/null +++ b/tests/backends/osmo/workloads/basic_task.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import sys +from pathlib import Path + + +def main() -> int: + output_path = Path(sys.argv[1]) + output_path.parent.mkdir(parents=True, exist_ok=True) + message = "ROAR_OSMO_BASIC_OK" + print(message) + output_path.write_text(f"{message}\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/backends/osmo/workloads/ray_smoke_job.py b/tests/backends/osmo/workloads/ray_smoke_job.py new file mode 100644 index 00000000..7bd3bea8 --- /dev/null +++ b/tests/backends/osmo/workloads/ray_smoke_job.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import json +import os +import socket +import time + +import ray + + +@ray.remote(num_cpus=1) +def square(value: int) -> dict[str, object]: + time.sleep(2) + return { + "value": value, + "square": value * value, + "hostname": socket.gethostname(), + } + + +def _wait_for_cluster() -> list[dict[str, object]]: + expected_nodes = int(os.environ.get("ROAR_EXPECTED_RAY_NODES", "1")) + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + nodes = [node for node in ray.nodes() if node.get("Alive")] + if len(nodes) >= expected_nodes: + return nodes + time.sleep(2) + raise SystemExit(f"expected at least {expected_nodes} live Ray nodes") + + +def main() -> None: + ray.init(address="auto", ignore_reinit_error=True, logging_level="ERROR") + _wait_for_cluster() + + results = ray.get([square.remote(value) for value in range(6)]) + hostnames = sorted({str(result["hostname"]) for result in results}) + payload = { + "host_count": len(hostnames), + "hosts": hostnames, + "node_count": len([node for node in ray.nodes() if node.get("Alive")]), + "total": sum(int(result["square"]) for result in results), + } + + if payload["total"] != 55: + raise SystemExit(f"unexpected total: {payload['total']}") + if payload["node_count"] < int(os.environ.get("ROAR_EXPECTED_RAY_NODES", "1")): + raise SystemExit(f"expected more live Ray nodes, got {payload['node_count']}") + + print(f"ROAR_OSMO_RAY_OK {json.dumps(payload, sort_keys=True)}", flush=True) + ray.shutdown() + + +if __name__ == "__main__": + main() diff --git a/tests/backends/test_osmo_attach.py b/tests/backends/test_osmo_attach.py new file mode 100644 index 00000000..1ee1d402 --- /dev/null +++ b/tests/backends/test_osmo_attach.py @@ -0,0 +1,443 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +from pathlib import Path + +from roar.backends.osmo import OsmoAttachOptions, attach_osmo_workflow +from roar.db.context import create_database_context + + +def test_attach_osmo_workflow_can_download_and_reconstitute_lineage( + monkeypatch, + tmp_path: Path, +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + workflow_path = repo_root / "workflow.yaml" + workflow_path.write_text( + """ +workflow: + name: {{ workflow_name }} + tasks: + - name: basic + outputs: + - dataset: + name: {{ output_dataset }} + path: result.txt + - dataset: + name: roar-lineage + path: roar-fragments.json + +default-values: + workflow_name: roar-osmo-attach + output_dataset: roar-osmo-attach-output +""".strip() + + "\n", + encoding="utf-8", + ) + (roar_dir / "config.toml").write_text( + "[osmo]\ndownload_declared_outputs = true\ningest_lineage_bundles = true\n", + encoding="utf-8", + ) + + def _run(command, *args, **kwargs): + del args, kwargs + if command[:2] == ["git", "rev-parse"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="deadbeef\n", + stderr="", + ) + if command[1:3] == ["workflow", "query"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-attach","status":"COMPLETED"}\n', + stderr="", + ) + if command[1:3] == ["dataset", "download"]: + dataset_ref = command[3] + target_dir = Path(command[-1]) + target_dir.mkdir(parents=True, exist_ok=True) + if dataset_ref.startswith("roar-lineage:"): + payload = { + "fragments": [ + { + "job_uid": "osmo-attach-task", + "task_id": "basic-task", + "worker_id": "worker-1", + "node_id": "node-1", + "task_name": "basic", + "started_at": 1.0, + "ended_at": 2.0, + "exit_code": 0, + "backend": "osmo", + "reads": [ + { + "path": "workflow.yaml", + "hash": "workflowhash", + "hash_algorithm": "blake3", + "size": workflow_path.stat().st_size, + "capture_method": "python", + } + ], + "writes": [ + { + "path": "${ROAR_PROJECT_DIR}/outputs/attach-output.txt", + "hash": "attachoutputhash", + "hash_algorithm": "blake3", + "size": 18, + "capture_method": "python", + } + ], + "backend_metadata": {"execution_role": "task"}, + } + ] + } + (target_dir / "roar-fragments.json").write_text( + json.dumps(payload), + encoding="utf-8", + ) + else: + (target_dir / "result.txt").write_text("ROAR_OSMO_ATTACH_OK\n", encoding="utf-8") + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="downloaded\n", + stderr="", + ) + raise AssertionError(f"unexpected subprocess.run command: {command!r}") + + monkeypatch.setattr(subprocess, "run", _run) + + result = attach_osmo_workflow( + roar_dir=roar_dir, + repo_root=str(repo_root), + workflow_id="workflow-attach", + options=OsmoAttachOptions( + workflow_spec_argument="workflow.yaml", + workflow_spec_path="workflow.yaml", + set_strings={ + "workflow_name": "workflow-attach", + "output_dataset": "workflow-attach-output", + }, + ), + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert metadata["osmo_attach"]["workflow_id"] == "workflow-attach" + assert metadata["osmo_attach"]["workflow_status"] == "COMPLETED" + assert metadata["osmo_attach"]["lineage_reconstitution"]["fragments_processed"] == 1 + assert metadata["osmo_attach"]["lineage_reconstitution"]["jobs_merged"] == 1 + assert metadata["osmo_attach"]["downloaded_outputs"][0]["dataset_name"] == ( + "workflow-attach-output" + ) + + output_paths = {Path(str(entry["path"])) for entry in result.outputs} + assert any(path.name == "query-COMPLETED.json" for path in output_paths) + assert any(path.name == "result.txt" for path in output_paths) + assert any(path.name == "roar-fragments.json" for path in output_paths) + receipt_path = next(path for path in output_paths if "attachments" in str(path)) + receipt_payload = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt_payload["osmo_attach"]["workflow_id"] == "workflow-attach" + + conn = sqlite3.connect(roar_dir / "roar.db") + conn.row_factory = sqlite3.Row + try: + child_jobs = conn.execute( + """ + SELECT id, parent_job_uid, execution_backend, execution_role, command + FROM jobs + WHERE parent_job_uid = ? AND job_type = 'osmo_task' + ORDER BY id ASC + """, + (result.job_uid,), + ).fetchall() + finally: + conn.close() + + assert len(child_jobs) == 1 + assert child_jobs[0]["execution_backend"] == "osmo" + assert child_jobs[0]["execution_role"] == "task" + assert child_jobs[0]["command"] == "osmo_task:basic" + + +def test_attach_osmo_workflow_supports_dataset_hints_without_workflow_spec( + monkeypatch, + tmp_path: Path, +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + workflow_path = repo_root / "workflow.yaml" + workflow_path.write_text("workflow: {}\n", encoding="utf-8") + (roar_dir / "config.toml").write_text( + "[osmo]\ndownload_declared_outputs = true\ningest_lineage_bundles = true\n", + encoding="utf-8", + ) + + def _run(command, *args, **kwargs): + del args, kwargs + if command[:2] == ["git", "rev-parse"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="deadbeef\n", + stderr="", + ) + if command[1:3] == ["workflow", "query"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-attach-hints","status":"COMPLETED"}\n', + stderr="", + ) + if command[1:3] == ["dataset", "download"]: + dataset_ref = command[3] + target_dir = Path(command[-1]) + target_dir.mkdir(parents=True, exist_ok=True) + if dataset_ref.startswith("roar-lineage:"): + payload = { + "fragments": [ + { + "job_uid": "osmo-attach-hints-task", + "task_id": "basic-task", + "worker_id": "worker-1", + "node_id": "node-1", + "task_name": "basic", + "started_at": 1.0, + "ended_at": 2.0, + "exit_code": 0, + "backend": "osmo", + "reads": [ + { + "path": "workflow.yaml", + "hash": "workflowhash", + "hash_algorithm": "blake3", + "size": workflow_path.stat().st_size, + "capture_method": "python", + } + ], + "writes": [ + { + "path": "${ROAR_PROJECT_DIR}/outputs/hints-output.txt", + "hash": "hintsoutputhash", + "hash_algorithm": "blake3", + "size": 17, + "capture_method": "python", + } + ], + "backend_metadata": {"execution_role": "task"}, + } + ] + } + (target_dir / "roar-fragments.json").write_text( + json.dumps(payload), + encoding="utf-8", + ) + else: + (target_dir / "result.txt").write_text("ROAR_OSMO_HINTS_OK\n", encoding="utf-8") + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="downloaded\n", + stderr="", + ) + raise AssertionError(f"unexpected subprocess.run command: {command!r}") + + monkeypatch.setattr(subprocess, "run", _run) + + result = attach_osmo_workflow( + roar_dir=roar_dir, + repo_root=str(repo_root), + workflow_id="workflow-attach-hints", + options=OsmoAttachOptions( + dataset_names=["workflow-attach-hints-output", "roar-lineage"], + ), + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert metadata["osmo_attach"]["attach"]["dataset_hints"] == [ + "workflow-attach-hints-output", + "roar-lineage", + ] + assert metadata["osmo_attach"]["downloaded_outputs"][0]["dataset_name"] == ( + "workflow-attach-hints-output" + ) + assert metadata["osmo_attach"]["lineage_reconstitution"]["fragments_processed"] == 1 + + +def test_attach_osmo_workflow_uses_configured_lineage_dataset_name_without_hints( + monkeypatch, + tmp_path: Path, +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + workflow_path = repo_root / "workflow.yaml" + workflow_path.write_text("workflow: {}\n", encoding="utf-8") + (roar_dir / "config.toml").write_text( + "[osmo]\n" + "download_declared_outputs = true\n" + "ingest_lineage_bundles = true\n" + 'lineage_bundle_dataset_name = "roar-lineage"\n', + encoding="utf-8", + ) + + def _run(command, *args, **kwargs): + del args, kwargs + if command[:2] == ["git", "rev-parse"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="deadbeef\n", + stderr="", + ) + if command[1:3] == ["workflow", "query"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-attach-config","status":"COMPLETED"}\n', + stderr="", + ) + if command[1:3] == ["dataset", "download"]: + dataset_ref = command[3] + target_dir = Path(command[-1]) + target_dir.mkdir(parents=True, exist_ok=True) + if dataset_ref.startswith("roar-lineage:"): + payload = { + "fragments": [ + { + "job_uid": "osmo-attach-config-task", + "task_id": "basic-task", + "worker_id": "worker-1", + "node_id": "node-1", + "task_name": "basic", + "started_at": 1.0, + "ended_at": 2.0, + "exit_code": 0, + "backend": "osmo", + "reads": [ + { + "path": "workflow.yaml", + "hash": "workflowhash", + "hash_algorithm": "blake3", + "size": workflow_path.stat().st_size, + "capture_method": "python", + } + ], + "writes": [ + { + "path": "${ROAR_PROJECT_DIR}/outputs/config-attach-output.txt", + "hash": "configattachhash", + "hash_algorithm": "blake3", + "size": 24, + "capture_method": "python", + } + ], + "backend_metadata": {"execution_role": "task"}, + } + ] + } + (target_dir / "roar-fragments.json").write_text( + json.dumps(payload), + encoding="utf-8", + ) + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="downloaded\n", + stderr="", + ) + raise AssertionError(f"unexpected subprocess.run command: {command!r}") + + monkeypatch.setattr(subprocess, "run", _run) + + result = attach_osmo_workflow( + roar_dir=roar_dir, + repo_root=str(repo_root), + workflow_id="workflow-attach-config", + options=OsmoAttachOptions(), + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert metadata["osmo_attach"]["attach"]["dataset_hints"] == ["roar-lineage"] + assert metadata["osmo_attach"]["lineage_reconstitution"]["fragments_processed"] == 1 + output_paths = {Path(str(entry["path"])) for entry in result.outputs} + assert any(path.name == "roar-fragments.json" for path in output_paths) + + +def test_attach_osmo_workflow_supports_task_hints_without_workflow_spec( + monkeypatch, + tmp_path: Path, +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + + def _run(command, *args, **kwargs): + del args, kwargs + if command[:2] == ["git", "rev-parse"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="deadbeef\n", + stderr="", + ) + if command[1:3] == ["workflow", "query"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-attach-failed","status":"FAILED"}\n', + stderr="", + ) + if command[1:3] == ["workflow", "logs"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="task failed\n", + stderr="traceback line\n", + ) + raise AssertionError(f"unexpected subprocess.run command: {command!r}") + + monkeypatch.setattr(subprocess, "run", _run) + + result = attach_osmo_workflow( + roar_dir=roar_dir, + repo_root=str(repo_root), + workflow_id="workflow-attach-failed", + options=OsmoAttachOptions(task_names=["basic"]), + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert result.exit_code == 0 + assert metadata["osmo_attach"]["attach"]["task_name_hints"] == ["basic"] + assert metadata["osmo_attach"]["workflow_status"] == "FAILED" + assert metadata["osmo_attach"]["workflow_diagnostics"]["task_logs"][0]["task_name"] == "basic" + output_paths = {Path(str(entry["path"])) for entry in result.outputs} + log_path = next(path for path in output_paths if path.name == "basic.log") + assert "task failed" in log_path.read_text(encoding="utf-8") diff --git a/tests/backends/test_osmo_attach_integration.py b/tests/backends/test_osmo_attach_integration.py new file mode 100644 index 00000000..d06745c3 --- /dev/null +++ b/tests/backends/test_osmo_attach_integration.py @@ -0,0 +1,289 @@ +from __future__ import annotations + +import json +import sqlite3 +import stat +import textwrap +from pathlib import Path + + +def _write_fake_osmo(temp_git_repo: Path) -> None: + script = temp_git_repo / "osmo" + script.write_text( + textwrap.dedent( + """#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +args = sys.argv[1:] +if args[:2] == ["workflow", "query"] and len(args) >= 3: + print(json.dumps({"name": args[2], "status": "COMPLETED"})) + raise SystemExit(0) +if args[:2] == ["dataset", "download"] and len(args) >= 4: + target = Path(args[3]) + target.mkdir(parents=True, exist_ok=True) + dataset_ref = args[2] + if dataset_ref.startswith("roar-lineage:"): + payload = { + "fragments": [ + { + "job_uid": "osmo-attach-product-task", + "task_id": "basic-task", + "worker_id": "worker-1", + "node_id": "node-1", + "task_name": "basic", + "started_at": 1.0, + "ended_at": 2.0, + "exit_code": 0, + "backend": "osmo", + "reads": [ + { + "path": "workflow.yaml", + "hash": "workflowhash", + "hash_algorithm": "blake3", + "size": 1, + "capture_method": "python", + } + ], + "writes": [ + { + "path": "${ROAR_PROJECT_DIR}/outputs/attach-product-output.txt", + "hash": "attachproducthash", + "hash_algorithm": "blake3", + "size": 25, + "capture_method": "python", + } + ], + "backend_metadata": {"execution_role": "task"}, + } + ] + } + (target / "roar-fragments.json").write_text(json.dumps(payload), encoding="utf-8") + else: + (target / "result.txt").write_text("ROAR_OSMO_ATTACH_OK\\n", encoding="utf-8") + raise SystemExit(0) + +print(f"unexpected args: {args!r}", file=sys.stderr) +raise SystemExit(2) +""" + ), + encoding="utf-8", + ) + script.chmod(script.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def test_roar_osmo_attach_records_receipt_and_reconstitutes_lineage( + temp_git_repo: Path, + roar_cli, +) -> None: + _write_fake_osmo(temp_git_repo) + (temp_git_repo / "workflow.yaml").write_text( + """ +workflow: + name: {{ workflow_name }} + tasks: + - name: basic + outputs: + - dataset: + name: {{ output_dataset }} + path: result.txt + - dataset: + name: roar-lineage + path: roar-fragments.json + +default-values: + workflow_name: workflow-attach-product + output_dataset: workflow-attach-product-output +""".strip() + + "\n", + encoding="utf-8", + ) + + config_path = temp_git_repo / ".roar" / "config.toml" + config_path.write_text( + "[osmo]\ndownload_declared_outputs = true\ningest_lineage_bundles = true\n", + encoding="utf-8", + ) + + result = roar_cli( + "osmo", + "attach", + "workflow-attach-product", + "--osmo-binary", + "./osmo", + "--workflow-spec", + "workflow.yaml", + "--set-string", + "workflow_name=workflow-attach-product", + "--set-string", + "output_dataset=workflow-attach-product-output", + ) + + assert result.returncode == 0 + + db_path = temp_git_repo / ".roar" / "roar.db" + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + job = conn.execute( + """ + SELECT id, job_uid, execution_backend, execution_role, exit_code + FROM jobs + WHERE execution_role = 'attach' + ORDER BY id DESC + LIMIT 1 + """ + ).fetchone() + child_jobs = conn.execute( + """ + SELECT id, parent_job_uid, execution_backend, execution_role, command + FROM jobs + WHERE job_type = 'osmo_task' + ORDER BY id ASC + """ + ).fetchall() + output = conn.execute( + """ + SELECT path + FROM job_outputs + WHERE job_id = ? + ORDER BY path + """, + (int(job["id"]),), + ).fetchall() + finally: + conn.close() + + assert job is not None + assert job["execution_backend"] == "osmo" + assert job["execution_role"] == "attach" + assert job["exit_code"] == 0 + assert len(child_jobs) == 1 + assert child_jobs[0]["parent_job_uid"] == job["job_uid"] + assert child_jobs[0]["execution_role"] == "task" + assert child_jobs[0]["command"] == "osmo_task:basic" + + output_paths = [Path(str(row["path"])) for row in output] + receipt_path = next(path for path in output_paths if "attachments" in str(path)) + query_path = next(path for path in output_paths if path.name == "query-COMPLETED.json") + downloaded_result = next(path for path in output_paths if path.name == "result.txt") + downloaded_bundle = next(path for path in output_paths if path.name == "roar-fragments.json") + assert ( + receipt_path + == temp_git_repo + / ".roar" + / "osmo" + / "attachments" + / "workflow-attach-product-COMPLETED.json" + ) + assert query_path.exists() + assert downloaded_result.read_text(encoding="utf-8").strip() == "ROAR_OSMO_ATTACH_OK" + assert downloaded_bundle.exists() + + payload = json.loads(receipt_path.read_text(encoding="utf-8")) + assert payload["osmo_attach"]["workflow_id"] == "workflow-attach-product" + assert payload["osmo_attach"]["workflow_status"] == "COMPLETED" + assert payload["osmo_attach"]["lineage_reconstitution"]["fragments_processed"] == 1 + assert payload["osmo_attach"]["lineage_reconstitution"]["jobs_merged"] == 1 + + +def test_roar_osmo_attach_supports_dataset_hints_without_workflow_spec( + temp_git_repo: Path, + roar_cli, +) -> None: + _write_fake_osmo(temp_git_repo) + (temp_git_repo / "workflow.yaml").write_text("workflow: {}\n", encoding="utf-8") + + config_path = temp_git_repo / ".roar" / "config.toml" + config_path.write_text( + "[osmo]\ndownload_declared_outputs = true\ningest_lineage_bundles = true\n", + encoding="utf-8", + ) + + result = roar_cli( + "osmo", + "attach", + "workflow-attach-product", + "--osmo-binary", + "./osmo", + "--dataset", + "workflow-attach-product-output", + "--dataset", + "roar-lineage", + ) + + assert result.returncode == 0 + + db_path = temp_git_repo / ".roar" / "roar.db" + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + job = conn.execute( + """ + SELECT id, metadata + FROM jobs + WHERE execution_role = 'attach' + ORDER BY id DESC + LIMIT 1 + """ + ).fetchone() + finally: + conn.close() + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert metadata["osmo_attach"]["attach"]["dataset_hints"] == [ + "workflow-attach-product-output", + "roar-lineage", + ] + assert metadata["osmo_attach"]["downloaded_outputs"][0]["dataset_name"] == ( + "workflow-attach-product-output" + ) + assert metadata["osmo_attach"]["lineage_reconstitution"]["fragments_processed"] == 1 + + +def test_roar_osmo_attach_uses_configured_lineage_dataset_name_without_hints( + temp_git_repo: Path, + roar_cli, +) -> None: + _write_fake_osmo(temp_git_repo) + config_path = temp_git_repo / ".roar" / "config.toml" + config_path.write_text( + "[osmo]\n" + "download_declared_outputs = true\n" + "ingest_lineage_bundles = true\n" + 'lineage_bundle_dataset_name = "roar-lineage"\n', + encoding="utf-8", + ) + + result = roar_cli( + "osmo", + "attach", + "workflow-attach-product", + "--osmo-binary", + "./osmo", + ) + + assert result.returncode == 0 + + db_path = temp_git_repo / ".roar" / "roar.db" + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + job = conn.execute( + """ + SELECT id, metadata + FROM jobs + WHERE execution_role = 'attach' + ORDER BY id DESC + LIMIT 1 + """ + ).fetchone() + finally: + conn.close() + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert metadata["osmo_attach"]["attach"]["dataset_hints"] == ["roar-lineage"] + assert metadata["osmo_attach"]["lineage_reconstitution"]["fragments_processed"] == 1 diff --git a/tests/backends/test_osmo_backend.py b/tests/backends/test_osmo_backend.py new file mode 100644 index 00000000..25536a36 --- /dev/null +++ b/tests/backends/test_osmo_backend.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import importlib + +from roar.cli.commands.init import build_default_config_template +from roar.execution.framework.planning import plan_execution_command +from roar.execution.framework.registry import ( + iter_execution_backend_configurable_keys, + iter_execution_backends, +) + + +def _module(): + return importlib.import_module("roar.backends.osmo.submit") + + +def test_matches_osmo_workflow_submit_command() -> None: + assert _module().matches_osmo_workflow_submit_command( + ["osmo", "workflow", "submit", "workflow.yaml"] + ) + + +def test_non_submit_osmo_commands_do_not_match_osmo_backend() -> None: + assert not _module().matches_osmo_workflow_submit_command( + ["osmo", "workflow", "logs", "workflow-123"] + ) + assert not _module().matches_osmo_workflow_submit_command( + ["osmo", "dataset", "download", "dataset:latest", "./downloads"] + ) + + +def test_osmo_workflow_submit_command_is_planned_as_submit_role() -> None: + command = ["osmo", "workflow", "submit", "workflow.yaml", "--format-type", "json"] + + planned = _module().plan_osmo_workflow_submit_command(command) + + assert planned.backend_name == "osmo" + assert planned.command == command + assert planned.execution_role == "submit" + assert planned.session_id is None + assert planned.finalize_run is None + + +def test_osmo_workflow_submit_command_appends_json_output_flag_by_default() -> None: + command = ["osmo", "workflow", "submit", "workflow.yaml"] + + planned = _module().plan_osmo_workflow_submit_command(command) + + assert planned.command == [ + "osmo", + "workflow", + "submit", + "workflow.yaml", + "--format-type", + "json", + ] + + +def test_osmo_workflow_submit_preserves_explicit_format_type() -> None: + command = ["osmo", "workflow", "submit", "workflow.yaml", "--format-type", "yaml"] + + planned = _module().plan_osmo_workflow_submit_command(command) + + assert planned.command == command + + +def test_plan_execution_command_prefers_osmo_backend_for_osmo_workflow_submit() -> None: + planned = plan_execution_command(["osmo", "workflow", "submit", "workflow.yaml"]) + + assert planned.backend_name == "osmo" + assert planned.execution_role == "submit" + assert planned.command[-2:] == ["--format-type", "json"] + + +def test_non_submit_osmo_commands_fall_back_to_local_backend() -> None: + planned = plan_execution_command(["osmo", "workflow", "logs", "workflow-123"]) + + assert planned.backend_name == "local" + assert planned.execution_role == "host" + + +def test_disabled_osmo_backend_falls_back_to_local_backend(monkeypatch, tmp_path) -> None: + monkeypatch.chdir(tmp_path) + roar_dir = tmp_path / ".roar" + roar_dir.mkdir() + (roar_dir / "config.toml").write_text("[osmo]\nenabled = false\n", encoding="utf-8") + + planned = plan_execution_command(["osmo", "workflow", "submit", "workflow.yaml"]) + + assert planned.backend_name == "local" + assert planned.execution_role == "host" + + +def test_force_json_output_can_be_disabled(monkeypatch, tmp_path) -> None: + monkeypatch.chdir(tmp_path) + roar_dir = tmp_path / ".roar" + roar_dir.mkdir() + (roar_dir / "config.toml").write_text( + "[osmo]\nforce_json_output = false\n", + encoding="utf-8", + ) + + planned = plan_execution_command(["osmo", "workflow", "submit", "workflow.yaml"]) + + assert planned.backend_name == "osmo" + assert planned.command == ["osmo", "workflow", "submit", "workflow.yaml"] + + +def test_osmo_backend_is_registered_with_framework_config() -> None: + backend_names = [backend.name for backend in iter_execution_backends()] + + assert "osmo" in backend_names + assert "osmo.enabled" in iter_execution_backend_configurable_keys() + assert "osmo.auto_prepare_submissions" in iter_execution_backend_configurable_keys() + assert "osmo.force_json_output" in iter_execution_backend_configurable_keys() + assert "osmo.wait_for_completion" in iter_execution_backend_configurable_keys() + assert "osmo.download_declared_outputs" in iter_execution_backend_configurable_keys() + assert "osmo.download_directory" in iter_execution_backend_configurable_keys() + assert "osmo.ingest_lineage_bundles" in iter_execution_backend_configurable_keys() + assert "osmo.lineage_bundle_dataset_name" in iter_execution_backend_configurable_keys() + assert "osmo.lineage_bundle_filename" in iter_execution_backend_configurable_keys() + assert "osmo.runtime_install_requirement" in iter_execution_backend_configurable_keys() + assert "osmo.runtime_install_local_path" in iter_execution_backend_configurable_keys() + assert "osmo.runtime_install_remote_path" in iter_execution_backend_configurable_keys() + + +def test_init_template_includes_osmo_section() -> None: + template = build_default_config_template() + + assert "[osmo]" in template + assert "enabled = true" in template + assert "auto_prepare_submissions = true" in template + assert "force_json_output = true" in template + assert "wait_for_completion = false" in template + assert "download_declared_outputs = false" in template + assert "ingest_lineage_bundles = false" in template + assert 'lineage_bundle_dataset_name = "roar-lineage"' in template + assert 'runtime_install_requirement = ""' in template + assert 'runtime_install_local_path = ""' in template diff --git a/tests/backends/test_osmo_export.py b/tests/backends/test_osmo_export.py new file mode 100644 index 00000000..f7a884ea --- /dev/null +++ b/tests/backends/test_osmo_export.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path + +from roar.backends.osmo import export_osmo_lineage_bundle +from roar.db.context import create_database_context + + +def test_export_osmo_lineage_bundle_serializes_latest_job_as_fragment(temp_git_repo: Path) -> None: + input_path = temp_git_repo / "inputs" / "source.txt" + output_path = temp_git_repo / "outputs" / "result.txt" + bundle_path = temp_git_repo / "roar-fragments.json" + input_path.parent.mkdir(parents=True, exist_ok=True) + output_path.parent.mkdir(parents=True, exist_ok=True) + input_path.write_text("source\n", encoding="utf-8") + output_path.write_text("result\n", encoding="utf-8") + + with create_database_context(temp_git_repo / ".roar") as db_ctx: + db_ctx.job_recording.record_job( + command="python task.py", + timestamp=time.time(), + job_uid="local-job-uid", + duration_seconds=1.25, + exit_code=0, + input_files=[str(input_path)], + output_files=[str(output_path)], + execution_backend="local", + execution_role="host", + repo_root=str(temp_git_repo), + ) + + exported = export_osmo_lineage_bundle( + roar_dir=temp_git_repo / ".roar", + output_path=bundle_path, + task_id="osmo-task-1", + task_name="basic", + ) + + assert exported.exported_job_uid == "local-job-uid" + assert exported.fragment_count == 1 + assert exported.task_id == "osmo-task-1" + assert exported.task_name == "basic" + + payload = json.loads(bundle_path.read_text(encoding="utf-8")) + assert payload["metadata"]["exported_job_uid"] == "local-job-uid" + fragment = payload["fragments"][0] + assert fragment["backend"] == "osmo" + assert fragment["job_uid"] == "local-job-uid" + assert fragment["task_id"] == "osmo-task-1" + assert fragment["task_name"] == "basic" + assert fragment["backend_metadata"]["execution_role"] == "task" + assert fragment["backend_metadata"]["source_execution_backend"] == "local" + assert fragment["reads"][0]["path"] == "${ROAR_PROJECT_DIR}/inputs/source.txt" + assert fragment["writes"][0]["path"] == "${ROAR_PROJECT_DIR}/outputs/result.txt" + + +def test_export_osmo_lineage_bundle_can_select_job_uid(temp_git_repo: Path) -> None: + first_output = temp_git_repo / "outputs" / "first.txt" + second_output = temp_git_repo / "outputs" / "second.txt" + first_output.parent.mkdir(parents=True, exist_ok=True) + first_output.write_text("first\n", encoding="utf-8") + second_output.write_text("second\n", encoding="utf-8") + + with create_database_context(temp_git_repo / ".roar") as db_ctx: + db_ctx.job_recording.record_job( + command="python first.py", + timestamp=time.time() - 10, + job_uid="first-job", + duration_seconds=0.5, + exit_code=0, + output_files=[str(first_output)], + execution_backend="local", + execution_role="host", + repo_root=str(temp_git_repo), + ) + db_ctx.job_recording.record_job( + command="python second.py", + timestamp=time.time(), + job_uid="second-job", + duration_seconds=0.5, + exit_code=0, + output_files=[str(second_output)], + execution_backend="local", + execution_role="host", + repo_root=str(temp_git_repo), + ) + + bundle_path = temp_git_repo / "selected.json" + exported = export_osmo_lineage_bundle( + roar_dir=temp_git_repo / ".roar", + output_path=bundle_path, + job_uid="first-job", + task_name="selected-task", + ) + + payload = json.loads(bundle_path.read_text(encoding="utf-8")) + assert exported.exported_job_uid == "first-job" + assert payload["metadata"]["exported_job_uid"] == "first-job" + assert payload["fragments"][0]["writes"][0]["path"] == "${ROAR_PROJECT_DIR}/outputs/first.txt" + assert payload["fragments"][0]["task_name"] == "selected-task" diff --git a/tests/backends/test_osmo_export_integration.py b/tests/backends/test_osmo_export_integration.py new file mode 100644 index 00000000..c15c008e --- /dev/null +++ b/tests/backends/test_osmo_export_integration.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path + +from roar.db.context import create_database_context + + +def test_roar_osmo_export_lineage_bundle_writes_bundle_for_latest_job( + temp_git_repo: Path, + roar_cli, +) -> None: + output_path = temp_git_repo / "outputs" / "result.txt" + bundle_path = temp_git_repo / "dist" / "roar-fragments.json" + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text("result\n", encoding="utf-8") + + with create_database_context(temp_git_repo / ".roar") as db_ctx: + db_ctx.job_recording.record_job( + command="python worker.py", + timestamp=time.time(), + job_uid="remote-job", + duration_seconds=2.0, + exit_code=0, + output_files=[str(output_path)], + execution_backend="local", + execution_role="host", + repo_root=str(temp_git_repo), + ) + + result = roar_cli( + "osmo", + "export-lineage-bundle", + "dist/roar-fragments.json", + "--task-id", + "osmo-task-remote", + "--task-name", + "basic", + ) + + assert result.returncode == 0 + assert "dist/roar-fragments.json" in result.stdout + + payload = json.loads(bundle_path.read_text(encoding="utf-8")) + assert payload["metadata"]["exported_job_uid"] == "remote-job" + assert payload["metadata"]["task_id"] == "osmo-task-remote" + assert payload["fragments"][0]["task_name"] == "basic" + assert payload["fragments"][0]["writes"][0]["path"] == "${ROAR_PROJECT_DIR}/outputs/result.txt" diff --git a/tests/backends/test_osmo_host_execution.py b/tests/backends/test_osmo_host_execution.py new file mode 100644 index 00000000..7fc9d3d8 --- /dev/null +++ b/tests/backends/test_osmo_host_execution.py @@ -0,0 +1,838 @@ +from __future__ import annotations + +import json +import sqlite3 +import subprocess +from pathlib import Path + +import pytest + +from roar.backends.osmo.host_execution import execute_osmo_workflow_submit +from roar.core.models.run import RunContext +from roar.db.context import create_database_context +from roar.execution.runtime.host_execution import ExecutionSetupError + + +def test_execute_osmo_workflow_submit_records_local_job( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + + completed = subprocess.CompletedProcess( + args=["osmo", "workflow", "submit", "workflow.yaml", "--format-type", "json"], + returncode=0, + stdout='{"name":"workflow-123","overview":"https://osmo.example/workflows/123"}\n', + stderr="", + ) + monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: completed) + + result = execute_osmo_workflow_submit( + RunContext( + roar_dir=roar_dir, + repo_root=str(repo_root), + command=["osmo", "workflow", "submit", "workflow.yaml", "--format-type", "json"], + execution_backend="osmo", + execution_role="submit", + ) + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + assert job is not None + assert job["execution_backend"] == "osmo" + assert job["execution_role"] == "submit" + assert job["job_type"] == "run" + metadata = json.loads(str(job["metadata"])) + assert metadata["osmo_submit"]["workflow_id"] == "workflow-123" + assert metadata["osmo_submit"]["response"]["overview"] == "https://osmo.example/workflows/123" + assert result.exit_code == 0 + assert result.inputs == [] + assert len(result.outputs) == 1 + receipt_path = Path(str(result.outputs[0]["path"])) + assert receipt_path.exists() + receipt_payload = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt_payload["osmo_submit"]["workflow_id"] == "workflow-123" + assert receipt_path.name == "workflow-123.json" + + captured = capsys.readouterr() + assert '"name":"workflow-123"' in captured.out + assert captured.err == "" + + +def test_execute_osmo_workflow_submit_records_text_output_and_failure( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + + completed = subprocess.CompletedProcess( + args=["osmo", "workflow", "submit", "workflow.yaml"], + returncode=7, + stdout="submitted workflow workflow-456\n", + stderr="permission denied\n", + ) + monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: completed) + + result = execute_osmo_workflow_submit( + RunContext( + roar_dir=roar_dir, + repo_root=str(repo_root), + command=["osmo", "workflow", "submit", "workflow.yaml"], + execution_backend="osmo", + execution_role="submit", + ) + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert metadata["osmo_submit"]["workflow_id"] is None + assert metadata["osmo_submit"]["response_format"] == "text" + assert metadata["osmo_submit"]["stdout"] == "submitted workflow workflow-456" + assert metadata["osmo_submit"]["stderr"] == "permission denied" + assert result.exit_code == 7 + assert len(result.outputs) == 1 + receipt_path = Path(str(result.outputs[0]["path"])) + receipt_payload = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt_payload["osmo_submit"]["return_code"] == 7 + assert receipt_path.name == "submit.json" + + captured = capsys.readouterr() + assert "submitted workflow workflow-456" in captured.out + assert "permission denied" in captured.err + + +def test_execute_osmo_workflow_submit_records_workflow_and_set_file_inputs( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + workflow_path = repo_root / "workflow.yaml" + workflow_path.write_text( + """ +workflow: + tasks: + - name: basic + command: ["python"] + args: ["task.py"] +""".strip() + + "\n", + encoding="utf-8", + ) + dataset_path = repo_root / "params.json" + dataset_path.write_text('{"epochs": 10}\n', encoding="utf-8") + + completed = subprocess.CompletedProcess( + args=[ + "osmo", + "workflow", + "submit", + "workflow.yaml", + "--set-file", + "params=params.json", + "--set-string", + "mode=test", + "--format-type", + "json", + ], + returncode=0, + stdout='{"name":"workflow-inputs"}\n', + stderr="", + ) + monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: completed) + + result = execute_osmo_workflow_submit( + RunContext( + roar_dir=roar_dir, + repo_root=str(repo_root), + command=[ + "osmo", + "workflow", + "submit", + "workflow.yaml", + "--set-file", + "params=params.json", + "--set-string", + "mode=test", + "--format-type", + "json", + ], + execution_backend="osmo", + execution_role="submit", + ) + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert metadata["osmo_submit"]["submit"]["workflow_spec"]["argument"] == "workflow.yaml" + assert metadata["osmo_submit"]["submit"]["workflow_spec"]["path"] == str(workflow_path) + assert metadata["osmo_submit"]["submit"]["set_files"] == {"params": "params.json"} + assert metadata["osmo_submit"]["submit"]["set_strings"] == {"mode": "test"} + + input_paths = {str(entry["path"]) for entry in result.inputs} + assert input_paths == {str(workflow_path), str(dataset_path)} + + +def test_execute_osmo_workflow_submit_can_wait_for_workflow_completion( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + (roar_dir / "config.toml").write_text( + "[osmo]\nwait_for_completion = true\npoll_interval_seconds = 0.01\nquery_timeout_seconds = 1\n", + encoding="utf-8", + ) + + def _run(command, *args, **kwargs): + del args, kwargs + if command[:2] == ["git", "rev-parse"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="deadbeef\n", + stderr="", + ) + if command[1:3] == ["workflow", "submit"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-123"}\n', + stderr="", + ) + if command[1:3] == ["workflow", "query"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-123","status":"COMPLETED"}\n', + stderr="", + ) + raise AssertionError(f"unexpected subprocess.run command: {command!r}") + + monkeypatch.setattr(subprocess, "run", _run) + + result = execute_osmo_workflow_submit( + RunContext( + roar_dir=roar_dir, + repo_root=str(repo_root), + command=["osmo", "workflow", "submit", "workflow.yaml", "--format-type", "json"], + execution_backend="osmo", + execution_role="submit", + ) + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert result.exit_code == 0 + assert metadata["osmo_submit"]["wait_for_completion"] is True + assert metadata["osmo_submit"]["workflow_status"] == "COMPLETED" + assert metadata["osmo_submit"]["workflow_query"]["status"] == "COMPLETED" + assert len(result.outputs) == 2 + output_paths = {Path(str(entry["path"])) for entry in result.outputs} + query_path = next(path for path in output_paths if "diagnostics" in str(path)) + receipt_path = next(path for path in output_paths if "submissions" in str(path)) + assert query_path.name == "query-COMPLETED.json" + receipt_payload = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt_payload["osmo_submit"]["workflow_status"] == "COMPLETED" + assert receipt_path.name == "workflow-123-COMPLETED.json" + assert metadata["osmo_submit"]["workflow_diagnostics"]["query_artifact_path"] == str(query_path) + + captured = capsys.readouterr() + assert "waiting for OSMO workflow workflow-123" in captured.err + assert "finished with status COMPLETED" in captured.err + + +def test_execute_osmo_workflow_submit_can_download_declared_dataset_outputs( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + workflow_path = repo_root / "workflow.yaml" + workflow_path.write_text( + """ +workflow: + name: {{ workflow_name }} + tasks: + - name: basic + command: ["python"] + args: ["task.py"] + outputs: + - dataset: + name: {{ output_dataset }} + path: result.txt + +default-values: + workflow_name: roar-osmo-basic + output_dataset: roar-osmo-basic-output +""".strip() + + "\n", + encoding="utf-8", + ) + (roar_dir / "config.toml").write_text( + "[osmo]\nwait_for_completion = true\ndownload_declared_outputs = true\npoll_interval_seconds = 0.01\nquery_timeout_seconds = 1\n", + encoding="utf-8", + ) + + def _run(command, *args, **kwargs): + del args, kwargs + if command[:2] == ["git", "rev-parse"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="deadbeef\n", + stderr="", + ) + if command[1:3] == ["workflow", "submit"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-download"}\n', + stderr="", + ) + if command[1:3] == ["workflow", "query"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-download","status":"COMPLETED"}\n', + stderr="", + ) + if command[1:3] == ["dataset", "download"]: + target_dir = Path(command[-1]) + target_dir.mkdir(parents=True, exist_ok=True) + (target_dir / "result.txt").write_text("ROAR_OSMO_BASIC_OK\n", encoding="utf-8") + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="downloaded\n", + stderr="", + ) + raise AssertionError(f"unexpected subprocess.run command: {command!r}") + + monkeypatch.setattr(subprocess, "run", _run) + + result = execute_osmo_workflow_submit( + RunContext( + roar_dir=roar_dir, + repo_root=str(repo_root), + command=[ + "osmo", + "workflow", + "submit", + "workflow.yaml", + "--set-string", + "workflow_name=workflow-download", + "--set-string", + "output_dataset=workflow-download-output", + "--format-type", + "json", + ], + execution_backend="osmo", + execution_role="submit", + ) + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert result.exit_code == 0 + assert metadata["osmo_submit"]["download_declared_outputs"] is True + assert metadata["osmo_submit"]["downloaded_outputs"][0]["dataset_name"] == ( + "workflow-download-output" + ) + output_paths = {Path(str(entry["path"])) for entry in result.outputs} + assert any(path.name == "result.txt" for path in output_paths) + assert any(path.name == "query-COMPLETED.json" for path in output_paths) + assert any(path.name == "workflow-download-COMPLETED.json" for path in output_paths) + + +def test_execute_osmo_workflow_submit_can_reconstitute_downloaded_lineage_bundle( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + workflow_path = repo_root / "workflow.yaml" + workflow_path.write_text( + """ +workflow: + name: {{ workflow_name }} + tasks: + - name: basic + command: ["python"] + args: ["task.py"] + outputs: + - dataset: + name: {{ output_dataset }} + path: result.txt + - dataset: + name: roar-lineage + path: roar-fragments.json + +default-values: + workflow_name: roar-osmo-lineage + output_dataset: roar-osmo-lineage-output +""".strip() + + "\n", + encoding="utf-8", + ) + (roar_dir / "config.toml").write_text( + "[osmo]\n" + "wait_for_completion = true\n" + "download_declared_outputs = true\n" + "ingest_lineage_bundles = true\n" + "poll_interval_seconds = 0.01\n" + "query_timeout_seconds = 1\n", + encoding="utf-8", + ) + + def _run(command, *args, **kwargs): + del args, kwargs + if command[:2] == ["git", "rev-parse"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="deadbeef\n", + stderr="", + ) + if command[1:3] == ["workflow", "submit"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-lineage"}\n', + stderr="", + ) + if command[1:3] == ["workflow", "query"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-lineage","status":"COMPLETED"}\n', + stderr="", + ) + if command[1:3] == ["dataset", "download"]: + dataset_ref = command[3] + target_dir = Path(command[-1]) + target_dir.mkdir(parents=True, exist_ok=True) + if dataset_ref.startswith("roar-lineage:"): + payload = { + "fragments": [ + { + "job_uid": "osmo-task-basic", + "task_id": "basic-task", + "worker_id": "worker-1", + "node_id": "node-1", + "task_name": "basic", + "started_at": 1.0, + "ended_at": 2.0, + "exit_code": 0, + "backend": "osmo", + "reads": [ + { + "path": "workflow.yaml", + "hash": "workflowhash", + "hash_algorithm": "blake3", + "size": workflow_path.stat().st_size, + "capture_method": "python", + } + ], + "writes": [ + { + "path": "${ROAR_PROJECT_DIR}/outputs/worker-output.txt", + "hash": "workeroutputhash", + "hash_algorithm": "blake3", + "size": 17, + "capture_method": "python", + } + ], + "backend_metadata": {"execution_role": "task"}, + } + ] + } + (target_dir / "roar-fragments.json").write_text( + json.dumps(payload), + encoding="utf-8", + ) + else: + (target_dir / "result.txt").write_text("ROAR_OSMO_BASIC_OK\n", encoding="utf-8") + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="downloaded\n", + stderr="", + ) + raise AssertionError(f"unexpected subprocess.run command: {command!r}") + + monkeypatch.setattr(subprocess, "run", _run) + + result = execute_osmo_workflow_submit( + RunContext( + roar_dir=roar_dir, + repo_root=str(repo_root), + command=[ + "osmo", + "workflow", + "submit", + "workflow.yaml", + "--set-string", + "workflow_name=workflow-lineage", + "--set-string", + "output_dataset=workflow-lineage-output", + "--format-type", + "json", + ], + execution_backend="osmo", + execution_role="submit", + ) + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + conn = sqlite3.connect(roar_dir / "roar.db") + conn.row_factory = sqlite3.Row + try: + child_jobs = conn.execute( + """ + SELECT id, parent_job_uid, execution_backend, execution_role, command + FROM jobs + WHERE parent_job_uid = ? AND job_type = 'osmo_task' + ORDER BY id ASC + """, + (result.job_uid,), + ).fetchall() + finally: + conn.close() + + assert job is not None + metadata = json.loads(str(job["metadata"])) + lineage = metadata["osmo_submit"]["lineage_reconstitution"] + assert lineage["fragments_processed"] == 1 + assert lineage["jobs_merged"] == 1 + assert lineage["bundle_count"] == 1 + assert lineage["bundles"][0]["dataset_name"] == "roar-lineage" + + assert len(child_jobs) == 1 + child_job = child_jobs[0] + assert child_job["execution_backend"] == "osmo" + assert child_job["execution_role"] == "task" + assert child_job["parent_job_uid"] == result.job_uid + assert child_job["command"] == "osmo_task:basic" + + with create_database_context(roar_dir) as db_ctx: + child_outputs = db_ctx.jobs.get_outputs(int(child_job["id"])) + child_inputs = db_ctx.jobs.get_inputs(int(child_job["id"])) + + assert [entry["path"] for entry in child_inputs] == [str(workflow_path)] + assert [entry["path"] for entry in child_outputs] == [ + str(repo_root / "outputs" / "worker-output.txt") + ] + + output_paths = {Path(str(entry["path"])) for entry in result.outputs} + assert any(path.name == "roar-fragments.json" for path in output_paths) + assert any(path.name == "result.txt" for path in output_paths) + assert any(path.name == "query-COMPLETED.json" for path in output_paths) + assert any(path.name == "workflow-lineage-COMPLETED.json" for path in output_paths) + + +def test_execute_osmo_workflow_submit_uses_configured_lineage_dataset_name( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + workflow_path = repo_root / "workflow.yaml" + workflow_path.write_text( + """ +workflow: + name: {{ workflow_name }} + tasks: + - name: basic + command: ["python"] + args: ["task.py"] + outputs: + - dataset: + name: {{ output_dataset }} + path: result.txt + +default-values: + workflow_name: roar-osmo-config-lineage + output_dataset: roar-osmo-config-lineage-output +""".strip() + + "\n", + encoding="utf-8", + ) + (roar_dir / "config.toml").write_text( + "[osmo]\n" + "wait_for_completion = true\n" + "download_declared_outputs = true\n" + "ingest_lineage_bundles = true\n" + 'lineage_bundle_dataset_name = "roar-lineage"\n' + "poll_interval_seconds = 0.01\n" + "query_timeout_seconds = 1\n", + encoding="utf-8", + ) + + def _run(command, *args, **kwargs): + del args, kwargs + if command[:2] == ["git", "rev-parse"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="deadbeef\n", + stderr="", + ) + if command[1:3] == ["workflow", "submit"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-config-lineage"}\n', + stderr="", + ) + if command[1:3] == ["workflow", "query"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-config-lineage","status":"COMPLETED"}\n', + stderr="", + ) + if command[1:3] == ["dataset", "download"]: + dataset_ref = command[3] + target_dir = Path(command[-1]) + target_dir.mkdir(parents=True, exist_ok=True) + if dataset_ref.startswith("roar-lineage:"): + payload = { + "fragments": [ + { + "job_uid": "osmo-config-lineage-task", + "task_id": "basic-task", + "worker_id": "worker-1", + "node_id": "node-1", + "task_name": "basic", + "started_at": 1.0, + "ended_at": 2.0, + "exit_code": 0, + "backend": "osmo", + "reads": [ + { + "path": "workflow.yaml", + "hash": "workflowhash", + "hash_algorithm": "blake3", + "size": workflow_path.stat().st_size, + "capture_method": "python", + } + ], + "writes": [ + { + "path": "${ROAR_PROJECT_DIR}/outputs/config-lineage-output.txt", + "hash": "configlineagehash", + "hash_algorithm": "blake3", + "size": 25, + "capture_method": "python", + } + ], + "backend_metadata": {"execution_role": "task"}, + } + ] + } + (target_dir / "roar-fragments.json").write_text( + json.dumps(payload), + encoding="utf-8", + ) + else: + (target_dir / "result.txt").write_text("ROAR_OSMO_CONFIG_OK\n", encoding="utf-8") + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="downloaded\n", + stderr="", + ) + raise AssertionError(f"unexpected subprocess.run command: {command!r}") + + monkeypatch.setattr(subprocess, "run", _run) + + result = execute_osmo_workflow_submit( + RunContext( + roar_dir=roar_dir, + repo_root=str(repo_root), + command=[ + "osmo", + "workflow", + "submit", + "workflow.yaml", + "--set-string", + "workflow_name=workflow-config-lineage", + "--set-string", + "output_dataset=workflow-config-lineage-output", + "--format-type", + "json", + ], + execution_backend="osmo", + execution_role="submit", + ) + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert metadata["osmo_submit"]["submit"]["dataset_hints"] == ["roar-lineage"] + assert metadata["osmo_submit"]["lineage_reconstitution"]["fragments_processed"] == 1 + downloaded_names = [ + item["dataset_name"] for item in metadata["osmo_submit"]["downloaded_outputs"] + ] + assert downloaded_names == ["workflow-config-lineage-output", "roar-lineage"] + + +def test_execute_osmo_workflow_submit_fails_when_waited_workflow_fails( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + workflow_path = repo_root / "workflow.yaml" + workflow_path.write_text( + """ +workflow: + tasks: + - name: basic + command: ["python"] + args: ["task.py"] + outputs: [] +""".strip() + + "\n", + encoding="utf-8", + ) + (roar_dir / "config.toml").write_text( + "[osmo]\nwait_for_completion = true\npoll_interval_seconds = 0.01\nquery_timeout_seconds = 1\n", + encoding="utf-8", + ) + + def _run(command, *args, **kwargs): + del args, kwargs + if command[:2] == ["git", "rev-parse"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="deadbeef\n", + stderr="", + ) + if command[1:3] == ["workflow", "submit"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-999"}\n', + stderr="", + ) + if command[1:3] == ["workflow", "query"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout='{"name":"workflow-999","status":"FAILED"}\n', + stderr="", + ) + if command[1:3] == ["workflow", "logs"]: + return subprocess.CompletedProcess( + args=command, + returncode=0, + stdout="task failed\n", + stderr="traceback line\n", + ) + raise AssertionError(f"unexpected subprocess.run command: {command!r}") + + monkeypatch.setattr(subprocess, "run", _run) + + result = execute_osmo_workflow_submit( + RunContext( + roar_dir=roar_dir, + repo_root=str(repo_root), + command=["osmo", "workflow", "submit", "workflow.yaml", "--format-type", "json"], + execution_backend="osmo", + execution_role="submit", + ) + ) + + with create_database_context(roar_dir) as db_ctx: + job = db_ctx.jobs.get(result.job_id) + + assert job is not None + metadata = json.loads(str(job["metadata"])) + assert result.exit_code == 1 + assert job["exit_code"] == 1 + assert metadata["osmo_submit"]["submit_return_code"] == 0 + assert metadata["osmo_submit"]["workflow_status"] == "FAILED" + assert len(result.outputs) == 3 + output_paths = {Path(str(entry["path"])) for entry in result.outputs} + query_path = next(path for path in output_paths if path.name == "query-FAILED.json") + log_path = next(path for path in output_paths if path.name == "basic.log") + receipt_path = next(path for path in output_paths if "submissions" in str(path)) + receipt_payload = json.loads(receipt_path.read_text(encoding="utf-8")) + assert receipt_payload["osmo_submit"]["workflow_status"] == "FAILED" + assert receipt_path.name == "workflow-999-FAILED.json" + assert metadata["osmo_submit"]["workflow_diagnostics"]["query_artifact_path"] == str(query_path) + assert metadata["osmo_submit"]["workflow_diagnostics"]["task_logs"][0]["path"] == str(log_path) + assert "task failed" in log_path.read_text(encoding="utf-8") + + captured = capsys.readouterr() + assert "finished with status FAILED" in captured.err + + +def test_execute_osmo_workflow_submit_raises_setup_error_when_osmo_cli_missing( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + repo_root = tmp_path / "repo" + repo_root.mkdir() + roar_dir = repo_root / ".roar" + roar_dir.mkdir() + + def _raise_missing(*args, **kwargs): + raise FileNotFoundError("osmo") + + monkeypatch.setattr(subprocess, "run", _raise_missing) + + with pytest.raises(ExecutionSetupError, match="osmo CLI not found"): + execute_osmo_workflow_submit( + RunContext( + roar_dir=roar_dir, + repo_root=str(repo_root), + command=["osmo", "workflow", "submit", "workflow.yaml"], + execution_backend="osmo", + execution_role="submit", + ) + ) diff --git a/tests/backends/test_osmo_run_integration.py b/tests/backends/test_osmo_run_integration.py new file mode 100644 index 00000000..14423cb0 --- /dev/null +++ b/tests/backends/test_osmo_run_integration.py @@ -0,0 +1,374 @@ +from __future__ import annotations + +import json +import sqlite3 +import stat +import textwrap +from pathlib import Path + + +def _write_fake_osmo(temp_git_repo: Path) -> None: + script = temp_git_repo / "osmo" + script.write_text( + textwrap.dedent( + """#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +args = sys.argv[1:] +if args[:2] == ["workflow", "submit"]: + if len(args) >= 3: + submit_path = Path(args[2]) + (Path.cwd() / ".submitted-osmo-command.json").write_text( + json.dumps(args), + encoding="utf-8", + ) + (Path.cwd() / ".submitted-osmo-workflow.yaml").write_text( + submit_path.read_text(encoding="utf-8"), + encoding="utf-8", + ) + print( + json.dumps( + { + "name": "workflow-product", + "overview": "https://osmo.example/workflows/workflow-product", + } + ) + ) + raise SystemExit(0) +if args[:2] == ["workflow", "query"] and len(args) >= 3: + print(json.dumps({"name": args[2], "status": "COMPLETED"})) + raise SystemExit(0) +if args[:2] == ["dataset", "download"] and len(args) >= 4: + target = Path(args[3]) + target.mkdir(parents=True, exist_ok=True) + dataset_ref = args[2] + if dataset_ref.startswith("roar-lineage:"): + payload = { + "fragments": [ + { + "job_uid": "osmo-product-task", + "task_id": "basic-task", + "worker_id": "worker-1", + "node_id": "node-1", + "task_name": "basic", + "started_at": 1.0, + "ended_at": 2.0, + "exit_code": 0, + "backend": "osmo", + "reads": [ + { + "path": "workflow.yaml", + "hash": "workflowhash", + "hash_algorithm": "blake3", + "size": 1, + "capture_method": "python", + } + ], + "writes": [ + { + "path": "${ROAR_PROJECT_DIR}/outputs/worker-output.txt", + "hash": "workeroutputhash", + "hash_algorithm": "blake3", + "size": 17, + "capture_method": "python", + } + ], + "backend_metadata": {"execution_role": "task"}, + } + ] + } + (target / "roar-fragments.json").write_text(json.dumps(payload), encoding="utf-8") + else: + (target / "result.txt").write_text("ROAR_OSMO_BASIC_OK\\n", encoding="utf-8") + raise SystemExit(0) + +print(f"unexpected args: {args!r}", file=sys.stderr) +raise SystemExit(2) +""" + ), + encoding="utf-8", + ) + script.chmod(script.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def test_roar_run_osmo_workflow_submit_records_receipt_and_waits( + temp_git_repo: Path, + roar_cli, + git_commit, +) -> None: + _write_fake_osmo(temp_git_repo) + (temp_git_repo / "task.py").write_text("print('hello')\n", encoding="utf-8") + (temp_git_repo / "workflow.yaml").write_text( + """ +workflow: + name: {{ workflow_name }} + tasks: + - name: basic + command: ["python"] + args: ["task.py"] + files: + - localpath: ./task.py + path: /workspace/task.py + outputs: + - dataset: + name: {{ output_dataset }} + path: result.txt + - dataset: + name: roar-lineage + path: roar-fragments.json + +default-values: + workflow_name: roar-osmo-basic + output_dataset: roar-osmo-basic-output +""".strip() + + "\n", + encoding="utf-8", + ) + + config_path = temp_git_repo / ".roar" / "config.toml" + config_path.write_text( + "[osmo]\n" + "wait_for_completion = true\n" + "download_declared_outputs = true\n" + "ingest_lineage_bundles = true\n" + "poll_interval_seconds = 0.01\n" + "query_timeout_seconds = 2\n", + encoding="utf-8", + ) + git_commit("add fake osmo submit backend") + + result = roar_cli( + "run", + "./osmo", + "workflow", + "submit", + "workflow.yaml", + "--set-string", + "workflow_name=workflow-product", + "--set-string", + "output_dataset=workflow-product-output", + ) + + assert result.returncode == 0 + assert "workflow-product" in result.stdout + + db_path = temp_git_repo / ".roar" / "roar.db" + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + job = conn.execute( + """ + SELECT id, job_uid, execution_backend, execution_role, exit_code + FROM jobs + WHERE execution_role = 'submit' + ORDER BY id DESC + LIMIT 1 + """ + ).fetchone() + child_jobs = conn.execute( + """ + SELECT id, parent_job_uid, execution_backend, execution_role, command + FROM jobs + WHERE job_type = 'osmo_task' + ORDER BY id ASC + """ + ).fetchall() + child_outputs = conn.execute( + """ + SELECT jo.path + FROM job_outputs jo + JOIN jobs j ON j.id = jo.job_id + WHERE j.job_type = 'osmo_task' + ORDER BY jo.path + """ + ).fetchall() + assert job is not None + output = conn.execute( + """ + SELECT path + FROM job_outputs + WHERE job_id = ? + ORDER BY path + """, + (int(job["id"]),), + ).fetchall() + inputs = conn.execute( + """ + SELECT path + FROM job_inputs + WHERE job_id = ? + ORDER BY path + """, + (int(job["id"]),), + ).fetchall() + finally: + conn.close() + + assert int(job["id"]) > 0 + assert job["execution_backend"] == "osmo" + assert job["execution_role"] == "submit" + assert job["exit_code"] == 0 + assert len(child_jobs) == 1 + assert child_jobs[0]["parent_job_uid"] == job["job_uid"] + assert child_jobs[0]["execution_backend"] == "osmo" + assert child_jobs[0]["execution_role"] == "task" + assert child_jobs[0]["command"] == "osmo_task:basic" + assert [str(row["path"]) for row in child_outputs] == [ + str(temp_git_repo / "outputs" / "worker-output.txt") + ] + output_paths = [Path(str(row["path"])) for row in output] + receipt_path = next(path for path in output_paths if "submissions" in str(path)) + query_path = next(path for path in output_paths if path.name == "query-COMPLETED.json") + assert ( + receipt_path + == temp_git_repo / ".roar" / "osmo" / "submissions" / "workflow-product-COMPLETED.json" + ) + downloaded_result = next(path for path in output_paths if path.name == "result.txt") + downloaded_bundle = next(path for path in output_paths if path.name == "roar-fragments.json") + assert downloaded_result.exists() + assert downloaded_result.read_text(encoding="utf-8").strip() == "ROAR_OSMO_BASIC_OK" + assert downloaded_bundle.exists() + assert query_path.exists() + assert [str(row["path"]) for row in inputs] == [str(temp_git_repo / "workflow.yaml")] + + payload = json.loads(receipt_path.read_text(encoding="utf-8")) + assert payload["osmo_submit"]["workflow_id"] == "workflow-product" + assert payload["osmo_submit"]["workflow_status"] == "COMPLETED" + assert ( + payload["osmo_submit"]["response"]["overview"] + == "https://osmo.example/workflows/workflow-product" + ) + + +def test_roar_run_osmo_workflow_submit_transparently_prepares_install_wrapper( + temp_git_repo: Path, + roar_cli, + git_commit, +) -> None: + _write_fake_osmo(temp_git_repo) + (temp_git_repo / "task.py").write_text("print('hello')\n", encoding="utf-8") + original_workflow = ( + """ +workflow: + name: sample + tasks: + - name: basic + command: ["python"] + args: ["task.py", "{{output}}/result.txt"] + files: + - localpath: ./task.py + path: /workspace/task.py +""".strip() + + "\n" + ) + (temp_git_repo / "workflow.yaml").write_text(original_workflow, encoding="utf-8") + + config_path = temp_git_repo / ".roar" / "config.toml" + config_path.write_text( + '[osmo]\nruntime_install_requirement = "roar-cli==9.9.9"\n', + encoding="utf-8", + ) + git_commit("enable osmo transparent workflow preparation") + + result = roar_cli("run", "./osmo", "workflow", "submit", "workflow.yaml") + + assert result.returncode == 0 + + original_rendered = (temp_git_repo / "workflow.yaml").read_text(encoding="utf-8") + submitted_rendered = (temp_git_repo / ".submitted-osmo-workflow.yaml").read_text( + encoding="utf-8" + ) + submitted_command = json.loads( + (temp_git_repo / ".submitted-osmo-command.json").read_text(encoding="utf-8") + ) + + assert original_rendered == original_workflow + assert submitted_command[2] != "workflow.yaml" + assert Path(submitted_command[2]).resolve().parent == temp_git_repo.resolve() + assert "path: /tmp/roar-osmo-wrapper.sh" in submitted_rendered + assert "name: roar-lineage" in submitted_rendered + assert "localpath: ./task.py" in submitted_rendered + assert ( + '"$python_bin" -m pip install --disable-pip-version-check --no-input --target "$install_root" "roar-cli==9.9.9"' + in submitted_rendered + ) + assert "urlopen(tracer_url)" not in submitted_rendered + assert "find_ptrace_tracer" in submitted_rendered + receipt_dir = temp_git_repo / ".roar" / "osmo" / "submissions" + receipt_path = max(receipt_dir.glob("*.json"), key=lambda path: path.stat().st_mtime) + payload = json.loads(receipt_path.read_text(encoding="utf-8")) + assert payload["osmo_submit"]["submit"]["workflow_spec"]["path"] == str( + temp_git_repo / "workflow.yaml" + ) + assert payload["osmo_submit"]["submit"]["prepared_workflow"]["workflow_spec"]["path"] == str( + Path(submitted_command[2]).resolve() + ) + assert payload["osmo_submit"]["submit"]["prepared_workflow"]["wrapped_tasks"] == ["basic"] + assert ( + payload["osmo_submit"]["submit"]["prepared_workflow"]["runtime_install_requirement"] + == "roar-cli==9.9.9" + ) + assert ( + "runtime_tracer_download_url" not in payload["osmo_submit"]["submit"]["prepared_workflow"] + ) + + +def test_roar_run_osmo_workflow_submit_can_inject_local_install_artifact( + temp_git_repo: Path, + roar_cli, + git_commit, +) -> None: + _write_fake_osmo(temp_git_repo) + wheel_path = temp_git_repo / "dist" / "roar_cli.whl" + wheel_path.parent.mkdir(parents=True, exist_ok=True) + wheel_path.write_bytes(b"\xfcwheel") + (temp_git_repo / "task.py").write_text("print('hello')\n", encoding="utf-8") + (temp_git_repo / "workflow.yaml").write_text( + """ +workflow: + name: sample + tasks: + - name: basic + command: ["python"] + args: ["task.py"] + files: + - localpath: ./task.py + path: /workspace/task.py +""".strip() + + "\n", + encoding="utf-8", + ) + + config_path = temp_git_repo / ".roar" / "config.toml" + config_path.write_text( + '[osmo]\nruntime_install_local_path = "dist/roar_cli.whl"\n', + encoding="utf-8", + ) + git_commit("enable osmo runtime install artifact") + + result = roar_cli("run", "./osmo", "workflow", "submit", "workflow.yaml") + + assert result.returncode == 0 + + submitted_rendered = (temp_git_repo / ".submitted-osmo-workflow.yaml").read_text( + encoding="utf-8" + ) + receipt_dir = temp_git_repo / ".roar" / "osmo" / "submissions" + receipt_path = max(receipt_dir.glob("*.json"), key=lambda path: path.stat().st_mtime) + payload = json.loads(receipt_path.read_text(encoding="utf-8")) + prepared = payload["osmo_submit"]["submit"]["prepared_workflow"] + + assert f"localpath: {wheel_path}" in submitted_rendered + assert "path: /tmp/roar-osmo-install.whl" in submitted_rendered + assert ( + '"$python_bin" -m pip install --disable-pip-version-check --no-input --target "$install_root" "/tmp/roar-osmo-install.whl"' + in submitted_rendered + ) + assert prepared["runtime_install_local_path"] == str(wheel_path) + assert prepared["runtime_install_remote_path"] == "/tmp/roar-osmo-install.whl" + assert "runtime_tracer_download_url" not in prepared + assert "runtime_tracer_remote_path" not in prepared + assert "base64.b64decode(payload)" not in submitted_rendered + assert "find_ptrace_tracer" in submitted_rendered diff --git a/tests/backends/test_osmo_runtime_bundle.py b/tests/backends/test_osmo_runtime_bundle.py new file mode 100644 index 00000000..80e5de9c --- /dev/null +++ b/tests/backends/test_osmo_runtime_bundle.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import tarfile +from pathlib import Path + +from roar.backends.osmo import build_osmo_runtime_bundle + + +def test_build_osmo_runtime_bundle_packages_roar_python_roots_and_tracer(tmp_path: Path) -> None: + roar_package_dir = tmp_path / "runtime" / "roar" + python_root = tmp_path / "site-packages" + tracer_path = tmp_path / "bin" / "roar-tracer" + output_path = tmp_path / "bundle" / "roar-osmo-runtime.tar.gz" + + (roar_package_dir / "cli").mkdir(parents=True) + (roar_package_dir / "__main__.py").write_text("print('roar')\n", encoding="utf-8") + (roar_package_dir / "cli" / "__init__.py").write_text("", encoding="utf-8") + (python_root / "click").mkdir(parents=True) + (python_root / "click" / "__init__.py").write_text("", encoding="utf-8") + (python_root / "yaml").mkdir(parents=True) + (python_root / "yaml" / "__init__.py").write_text("", encoding="utf-8") + tracer_path.parent.mkdir(parents=True) + tracer_path.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + + bundle = build_osmo_runtime_bundle( + output_path=output_path, + roar_package_dir=roar_package_dir, + python_roots=[python_root], + ptrace_tracer_path=tracer_path, + ) + + assert bundle.output_path == str(output_path) + assert bundle.roar_package_dir == str(roar_package_dir.resolve()) + assert bundle.python_roots == [str(python_root.resolve())] + assert bundle.ptrace_tracer_path == str(tracer_path.resolve()) + + with tarfile.open(output_path, "r:gz") as archive: + members = set(archive.getnames()) + + assert "python/roar/__main__.py" in members + assert "python/roar/cli/__init__.py" in members + assert "python/site-packages/click/__init__.py" in members + assert "python/site-packages/yaml/__init__.py" in members + assert "bin/roar-tracer" in members diff --git a/tests/backends/test_osmo_workflow_preparation.py b/tests/backends/test_osmo_workflow_preparation.py new file mode 100644 index 00000000..406f4913 --- /dev/null +++ b/tests/backends/test_osmo_workflow_preparation.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from roar.backends.osmo import prepare_osmo_workflow_for_lineage + + +def test_prepare_osmo_workflow_for_lineage_preserves_template_values(tmp_path: Path) -> None: + input_path = tmp_path / "ray.yaml" + output_path = tmp_path / "prepared.yaml" + input_path.write_text( + """ +workflow: + name: {{workflow_name}} + resources: {{resources}} + tasks: + - name: master + image: python:3.11-slim + +default-values: + workflow_name: roar-osmo-ray + resources: + default: + cpu: 1 +""".strip() + + "\n", + encoding="utf-8", + ) + + prepared = prepare_osmo_workflow_for_lineage( + input_path=input_path, + output_path=output_path, + lineage_dataset_name="roar-lineage", + lineage_bundle_filename="roar-fragments.json", + ) + + rendered = output_path.read_text(encoding="utf-8") + assert prepared.selected_tasks == ["master"] + assert prepared.modified_tasks == ["master"] + assert "name: {{ workflow_name }}" in rendered + assert "resources: {{ resources }}" in rendered + assert "name: roar-lineage" in rendered + assert "path: roar-fragments.json" in rendered + + +def test_prepare_osmo_workflow_for_lineage_requires_task_selection_for_multi_task_workflow( + tmp_path: Path, +) -> None: + input_path = tmp_path / "multi.yaml" + output_path = tmp_path / "prepared.yaml" + input_path.write_text( + """ +workflow: + name: sample + tasks: + - name: first + - name: second +""".strip() + + "\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="specify --task"): + prepare_osmo_workflow_for_lineage( + input_path=input_path, + output_path=output_path, + lineage_dataset_name="roar-lineage", + lineage_bundle_filename="roar-fragments.json", + ) + + +def test_prepare_osmo_workflow_for_lineage_only_updates_requested_tasks(tmp_path: Path) -> None: + input_path = tmp_path / "multi.yaml" + output_path = tmp_path / "prepared.yaml" + input_path.write_text( + """ +workflow: + name: sample + tasks: + - name: first + - name: second + outputs: + - dataset: + name: existing + path: existing.txt +""".strip() + + "\n", + encoding="utf-8", + ) + + prepared = prepare_osmo_workflow_for_lineage( + input_path=input_path, + output_path=output_path, + lineage_dataset_name="roar-lineage", + lineage_bundle_filename="roar-fragments.json", + task_names=["second"], + ) + + rendered = output_path.read_text(encoding="utf-8") + assert prepared.selected_tasks == ["second"] + assert prepared.modified_tasks == ["second"] + assert "name: first" in rendered + first_section, second_section = rendered.split("- name: second", maxsplit=1) + assert "roar-lineage" not in first_section + assert "name: existing" in second_section + assert "name: roar-lineage" in second_section + + +def test_prepare_osmo_workflow_for_lineage_can_wrap_all_tasks_by_default(tmp_path: Path) -> None: + input_path = tmp_path / "multi.yaml" + output_path = tmp_path / "prepared.yaml" + input_path.write_text( + """ +workflow: + name: sample + tasks: + - name: first + command: ["python"] + args: ["first.py"] + - name: second + command: ["python"] + args: ["second.py"] +""".strip() + + "\n", + encoding="utf-8", + ) + + prepared = prepare_osmo_workflow_for_lineage( + input_path=input_path, + output_path=output_path, + lineage_dataset_name="roar-lineage", + lineage_bundle_filename="roar-fragments.json", + inject_runtime_wrapper=True, + runtime_install_requirement="roar-cli==9.9.9", + default_to_all_tasks=True, + ) + + rendered = output_path.read_text(encoding="utf-8") + assert prepared.selected_tasks == ["first", "second"] + assert prepared.wrapped_tasks == ["first", "second"] + assert rendered.count("path: /tmp/roar-osmo-wrapper.sh") == 2 + assert rendered.count("name: roar-lineage") == 2 + + +def test_prepare_osmo_workflow_for_lineage_can_inject_runtime_wrapper(tmp_path: Path) -> None: + input_path = tmp_path / "basic.yaml" + output_path = tmp_path / "prepared.yaml" + input_path.write_text( + """ +workflow: + name: sample + tasks: + - name: basic + command: ["python"] + args: ["task.py", "{{output}}/result.txt"] + files: + - localpath: ./task.py + path: /workspace/task.py +""".strip() + + "\n", + encoding="utf-8", + ) + + prepared = prepare_osmo_workflow_for_lineage( + input_path=input_path, + output_path=output_path, + lineage_dataset_name="roar-lineage", + lineage_bundle_filename="roar-fragments.json", + inject_runtime_wrapper=True, + ) + + rendered = output_path.read_text(encoding="utf-8") + assert prepared.modified_tasks == ["basic"] + assert prepared.wrapped_tasks == ["basic"] + assert "command:\n - bash\n - /tmp/roar-osmo-wrapper.sh" in rendered + assert ( + " - basic\n" + " - '{{output}}/roar-fragments.json'\n" + " - python\n" + " - task.py\n" + " - '{{output}}/result.txt'" + ) in rendered + assert "path: /tmp/roar-osmo-wrapper.sh" in rendered + assert "-m roar run --tracer ptrace --no-tracer-fallback" in rendered + assert "-m roar osmo export-lineage-bundle" in rendered + + +def test_prepare_osmo_workflow_for_lineage_can_stage_runtime_bundle(tmp_path: Path) -> None: + input_path = tmp_path / "basic.yaml" + output_path = tmp_path / "prepared.yaml" + input_path.write_text( + """ +workflow: + name: sample + tasks: + - name: basic + command: ["python"] + args: ["task.py"] +""".strip() + + "\n", + encoding="utf-8", + ) + + prepared = prepare_osmo_workflow_for_lineage( + input_path=input_path, + output_path=output_path, + lineage_dataset_name="roar-lineage", + lineage_bundle_filename="roar-fragments.json", + inject_runtime_wrapper=True, + runtime_bundle_local_path="./roar-osmo-runtime.tar.gz", + ) + + rendered = output_path.read_text(encoding="utf-8") + assert prepared.runtime_bundle_local_path == "./roar-osmo-runtime.tar.gz" + assert prepared.runtime_bundle_remote_path == "/tmp/roar-osmo-runtime.tar.gz" + assert "localpath: ./roar-osmo-runtime.tar.gz" in rendered + assert "path: /tmp/roar-osmo-runtime.tar.gz" in rendered + assert 'tar -xzf "$runtime_bundle" -C "$runtime_root"' in rendered + assert 'export PATH="$runtime_root/bin:${PATH:-}"' in rendered + assert ( + 'export PYTHONPATH="$runtime_root/python:$runtime_root/python/site-packages:${PYTHONPATH:-}"' + in rendered + ) + + +def test_prepare_osmo_workflow_for_lineage_can_install_roar_runtime(tmp_path: Path) -> None: + input_path = tmp_path / "basic.yaml" + output_path = tmp_path / "prepared.yaml" + input_path.write_text( + """ +workflow: + name: sample + tasks: + - name: basic + command: ["python"] + args: ["task.py"] +""".strip() + + "\n", + encoding="utf-8", + ) + + prepared = prepare_osmo_workflow_for_lineage( + input_path=input_path, + output_path=output_path, + lineage_dataset_name="roar-lineage", + lineage_bundle_filename="roar-fragments.json", + inject_runtime_wrapper=True, + runtime_install_requirement="roar-cli==9.9.9", + ) + + rendered = output_path.read_text(encoding="utf-8") + assert prepared.runtime_install_requirement == "roar-cli==9.9.9" + assert 'install_root="/tmp/roar-osmo-python"' in rendered + assert 'if ! "$python_bin" -m pip --version >/dev/null 2>&1; then' in rendered + assert 'if ! "$python_bin" -m ensurepip --user >/dev/null 2>&1; then' in rendered + assert 'pip_command="$PYTHONUSERBASE/bin/pip3"' in rendered + assert ( + '"$python_bin" -m pip install --disable-pip-version-check --no-input --target "$install_root" "roar-cli==9.9.9"' + in rendered + ) + assert 'export PYTHONPATH="$install_root:${PYTHONPATH:-}"' in rendered + assert "installed roar-cli distribution does not expose roar-tracer" in rendered + assert "find_ptrace_tracer" in rendered + assert "urlopen(tracer_url)" not in rendered + + +def test_prepare_osmo_workflow_for_lineage_can_install_roar_runtime_from_local_artifact( + tmp_path: Path, +) -> None: + input_path = tmp_path / "basic.yaml" + output_path = tmp_path / "prepared.yaml" + wheel_path = tmp_path / "dist" / "roar_cli.whl" + wheel_path.parent.mkdir(parents=True, exist_ok=True) + wheel_path.write_bytes(b"\xfcwheel") + input_path.write_text( + """ +workflow: + name: sample + tasks: + - name: basic + command: ["python"] + args: ["task.py"] +""".strip() + + "\n", + encoding="utf-8", + ) + + prepared = prepare_osmo_workflow_for_lineage( + input_path=input_path, + output_path=output_path, + lineage_dataset_name="roar-lineage", + lineage_bundle_filename="roar-fragments.json", + inject_runtime_wrapper=True, + runtime_install_local_path=str(wheel_path), + ) + + rendered = output_path.read_text(encoding="utf-8") + assert prepared.runtime_install_requirement is None + assert prepared.runtime_install_local_path == str(wheel_path) + assert prepared.runtime_install_remote_path == "/tmp/roar-osmo-install.whl" + assert f"localpath: {wheel_path}" in rendered + assert "path: /tmp/roar-osmo-install.whl" in rendered + assert ( + '"$python_bin" -m pip install --disable-pip-version-check --no-input --target "$install_root" "/tmp/roar-osmo-install.whl"' + in rendered + ) + assert "installed roar-cli distribution does not expose roar-tracer" in rendered + assert "base64.b64decode(payload)" not in rendered diff --git a/tests/backends/test_osmo_workflow_preparation_integration.py b/tests/backends/test_osmo_workflow_preparation_integration.py new file mode 100644 index 00000000..25316803 --- /dev/null +++ b/tests/backends/test_osmo_workflow_preparation_integration.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import tarfile +from pathlib import Path + + +def test_roar_osmo_prepare_workflow_uses_configured_lineage_contract( + temp_git_repo: Path, + roar_cli, +) -> None: + input_path = temp_git_repo / "workflow.yaml" + output_path = temp_git_repo / "prepared" / "workflow.yaml" + input_path.write_text( + """ +workflow: + name: {{workflow_name}} + tasks: + - name: first + image: python:3.11-slim + - name: second + image: python:3.11-slim + +default-values: + workflow_name: roar-osmo-prepare +""".strip() + + "\n", + encoding="utf-8", + ) + + config_path = temp_git_repo / ".roar" / "config.toml" + config_path.write_text( + "[osmo]\n" + 'lineage_bundle_dataset_name = "custom-lineage"\n' + 'lineage_bundle_filename = "custom-bundle.json"\n', + encoding="utf-8", + ) + + result = roar_cli( + "osmo", + "prepare-workflow", + "workflow.yaml", + "prepared/workflow.yaml", + "--task", + "second", + ) + + assert result.returncode == 0 + assert "prepared/workflow.yaml" in result.stdout + + rendered = output_path.read_text(encoding="utf-8") + assert "name: {{ workflow_name }}" in rendered + first_section, second_section = rendered.split("- name: second", maxsplit=1) + assert "custom-lineage" not in first_section + assert "name: custom-lineage" in second_section + assert "path: custom-bundle.json" in second_section + + +def test_roar_osmo_prepare_workflow_can_inject_runtime_wrapper( + temp_git_repo: Path, + roar_cli, +) -> None: + input_path = temp_git_repo / "workflow.yaml" + output_path = temp_git_repo / "prepared" / "workflow.yaml" + input_path.write_text( + """ +workflow: + name: sample + tasks: + - name: basic + command: ["python"] + args: ["task.py", "{{output}}/result.txt"] +""".strip() + + "\n", + encoding="utf-8", + ) + + result = roar_cli( + "osmo", + "prepare-workflow", + "workflow.yaml", + "prepared/workflow.yaml", + "--inject-runtime-wrapper", + ) + + assert result.returncode == 0 + assert "Wrapped tasks: basic" in result.stdout + + rendered = output_path.read_text(encoding="utf-8") + assert "path: /tmp/roar-osmo-wrapper.sh" in rendered + assert "contents: |" in rendered + assert "-m roar run --tracer ptrace --no-tracer-fallback" in rendered + assert "-m roar osmo export-lineage-bundle" in rendered + assert "{{output}}/roar-fragments.json" in rendered + + +def test_roar_osmo_prepare_workflow_can_stage_runtime_bundle( + temp_git_repo: Path, + roar_cli, +) -> None: + input_path = temp_git_repo / "workflow.yaml" + output_path = temp_git_repo / "prepared" / "workflow.yaml" + fake_roar = temp_git_repo / "fake-runtime" / "roar" + fake_site = temp_git_repo / "fake-runtime" / "site-packages" + fake_tracer = temp_git_repo / "fake-runtime" / "bin" / "roar-tracer" + input_path.write_text( + """ +workflow: + name: sample + tasks: + - name: basic + command: ["python"] + args: ["task.py"] +""".strip() + + "\n", + encoding="utf-8", + ) + (fake_roar / "cli").mkdir(parents=True) + (fake_roar / "__main__.py").write_text("print('roar')\n", encoding="utf-8") + (fake_roar / "cli" / "__init__.py").write_text("", encoding="utf-8") + (fake_site / "click").mkdir(parents=True) + (fake_site / "click" / "__init__.py").write_text("", encoding="utf-8") + fake_tracer.parent.mkdir(parents=True) + fake_tracer.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + + result = roar_cli( + "osmo", + "prepare-workflow", + "workflow.yaml", + "prepared/workflow.yaml", + "--inject-runtime-wrapper", + "--stage-roar-runtime", + "--runtime-roar-package-dir", + str(fake_roar), + "--runtime-python-root", + str(fake_site), + "--runtime-tracer", + str(fake_tracer), + ) + + assert result.returncode == 0 + assert "Staged runtime bundle:" in result.stdout + + bundle_path = output_path.parent / "roar-osmo-runtime.tar.gz" + rendered = output_path.read_text(encoding="utf-8") + assert bundle_path.exists() + assert "localpath: roar-osmo-runtime.tar.gz" in rendered + assert "path: /tmp/roar-osmo-runtime.tar.gz" in rendered + assert 'tar -xzf "$runtime_bundle" -C "$runtime_root"' in rendered + + with tarfile.open(bundle_path, "r:gz") as archive: + members = set(archive.getnames()) + assert "python/roar/__main__.py" in members + assert "python/site-packages/click/__init__.py" in members + assert "bin/roar-tracer" in members + + +def test_roar_osmo_prepare_workflow_can_install_roar_runtime( + temp_git_repo: Path, + roar_cli, +) -> None: + input_path = temp_git_repo / "workflow.yaml" + output_path = temp_git_repo / "prepared" / "workflow.yaml" + input_path.write_text( + """ +workflow: + name: sample + tasks: + - name: basic + command: ["python"] + args: ["task.py"] +""".strip() + + "\n", + encoding="utf-8", + ) + + result = roar_cli( + "osmo", + "prepare-workflow", + "workflow.yaml", + "prepared/workflow.yaml", + "--inject-runtime-wrapper", + "--install-roar-runtime", + "--runtime-install-requirement", + "roar-cli==9.9.9", + ) + + assert result.returncode == 0 + + rendered = output_path.read_text(encoding="utf-8") + assert 'install_root="/tmp/roar-osmo-python"' in rendered + assert 'if ! "$python_bin" -m pip --version >/dev/null 2>&1; then' in rendered + assert 'if ! "$python_bin" -m ensurepip --user >/dev/null 2>&1; then' in rendered + assert 'pip_command="$PYTHONUSERBASE/bin/pip3"' in rendered + assert ( + '"$python_bin" -m pip install --disable-pip-version-check --no-input --target "$install_root" "roar-cli==9.9.9"' + in rendered + ) + assert 'export PYTHONPATH="$install_root:${PYTHONPATH:-}"' in rendered + assert "installed roar-cli distribution does not expose roar-tracer" in rendered + assert "find_ptrace_tracer" in rendered + assert "urlopen(tracer_url)" not in rendered + + +def test_roar_osmo_prepare_workflow_can_install_roar_runtime_from_local_artifact( + temp_git_repo: Path, + roar_cli, +) -> None: + input_path = temp_git_repo / "workflow.yaml" + output_path = temp_git_repo / "prepared" / "workflow.yaml" + wheel_path = temp_git_repo / "dist" / "roar_cli.whl" + wheel_path.parent.mkdir(parents=True, exist_ok=True) + wheel_path.write_bytes(b"\xfcwheel") + input_path.write_text( + """ +workflow: + name: sample + tasks: + - name: basic + command: ["python"] + args: ["task.py"] +""".strip() + + "\n", + encoding="utf-8", + ) + + result = roar_cli( + "osmo", + "prepare-workflow", + "workflow.yaml", + "prepared/workflow.yaml", + "--inject-runtime-wrapper", + "--install-roar-runtime", + "--runtime-install-local-path", + str(wheel_path), + ) + + assert result.returncode == 0 + + rendered = output_path.read_text(encoding="utf-8") + assert "Runtime install artifact:" in result.stdout + assert "localpath: ../dist/roar_cli.whl" in rendered + assert "path: /tmp/roar-osmo-install.whl" in rendered + assert ( + '"$python_bin" -m pip install --disable-pip-version-check --no-input --target "$install_root" "/tmp/roar-osmo-install.whl"' + in rendered + ) + assert "installed roar-cli distribution does not expose roar-tracer" in rendered + assert "base64.b64decode(payload)" not in rendered diff --git a/tests/execution/framework/test_execution_framework_layout.py b/tests/execution/framework/test_execution_framework_layout.py index acdd1d83..5fdea705 100644 --- a/tests/execution/framework/test_execution_framework_layout.py +++ b/tests/execution/framework/test_execution_framework_layout.py @@ -8,6 +8,7 @@ import tomli as tomllib from roar.backends.local.plugin import LOCAL_EXECUTION_BACKEND +from roar.backends.osmo.plugin import OSMO_EXECUTION_BACKEND from roar.backends.ray.plugin import RAY_EXECUTION_BACKEND from roar.cli.commands.init import build_default_config_template from roar.execution.framework import iter_execution_backends, plan_execution_command @@ -23,21 +24,45 @@ def test_canonical_execution_framework_imports_are_available() -> None: assert DatasetIdentifierInferer is not None assert ExecutionJobRecorder is not None assert LOCAL_EXECUTION_BACKEND.name == "local" + assert OSMO_EXECUTION_BACKEND.name == "osmo" assert RAY_EXECUTION_BACKEND.name == "ray" assert any(backend.name == "ray" for backend in iter_execution_backends()) + assert any(backend.name == "osmo" for backend in iter_execution_backends()) def test_backend_config_registration_is_available_through_framework() -> None: adapters = iter_execution_backend_config_adapters() assert any(adapter.section_name == "ray" for adapter in adapters) + assert any(adapter.section_name == "osmo" for adapter in adapters) assert "ray.pip_install" in iter_execution_backend_configurable_keys() + assert "osmo.enabled" in iter_execution_backend_configurable_keys() + assert "osmo.auto_prepare_submissions" in iter_execution_backend_configurable_keys() + assert "osmo.force_json_output" in iter_execution_backend_configurable_keys() + assert "osmo.wait_for_completion" in iter_execution_backend_configurable_keys() + assert "osmo.download_declared_outputs" in iter_execution_backend_configurable_keys() + assert "osmo.download_directory" in iter_execution_backend_configurable_keys() + assert "osmo.ingest_lineage_bundles" in iter_execution_backend_configurable_keys() + assert "osmo.lineage_bundle_dataset_name" in iter_execution_backend_configurable_keys() + assert "osmo.lineage_bundle_filename" in iter_execution_backend_configurable_keys() + assert "osmo.runtime_install_requirement" in iter_execution_backend_configurable_keys() + assert "osmo.runtime_install_local_path" in iter_execution_backend_configurable_keys() + assert "osmo.runtime_install_remote_path" in iter_execution_backend_configurable_keys() def test_init_template_includes_backend_registered_sections() -> None: template = build_default_config_template() assert "[ray]" in template + assert "[osmo]" in template assert 'actor_attribution = "per_call"' in template + assert "auto_prepare_submissions = true" in template + assert "force_json_output = true" in template + assert "wait_for_completion = false" in template + assert "download_declared_outputs = false" in template + assert "ingest_lineage_bundles = false" in template + assert 'lineage_bundle_dataset_name = "roar-lineage"' in template + assert 'runtime_install_requirement = ""' in template + assert 'runtime_install_local_path = ""' in template def test_packaged_roar_worker_entrypoint_uses_canonical_runtime_module() -> None: diff --git a/tests/execution/framework/test_execution_planning.py b/tests/execution/framework/test_execution_planning.py index d16fc3bd..8043feaa 100644 --- a/tests/execution/framework/test_execution_planning.py +++ b/tests/execution/framework/test_execution_planning.py @@ -169,7 +169,7 @@ def finalizer(_ctx) -> None: assert planned.finalize_run is finalizer -def test_execution_backends_register_local_and_ray_backends() -> None: +def test_execution_backends_register_local_ray_and_osmo_backends() -> None: from roar.execution.framework.registry import ( is_distributed_submission_command, is_execution_backend_job_environment, @@ -188,7 +188,9 @@ def test_execution_backends_register_local_and_ray_backends() -> None: assert "local" in backend_names assert "ray" in backend_names + assert "osmo" in backend_names assert backend_names.index("ray") < backend_names.index("local") + assert backend_names.index("osmo") < backend_names.index("local") assert match_execution_backend_for_module("ray") is not None assert match_execution_backend_for_module("ray.data").name == "ray" assert match_execution_backend_for_module("numpy") is None diff --git a/tests/integration/test_register_dry_run_cli.py b/tests/integration/test_register_dry_run_cli.py index 02171147..f20a98db 100644 --- a/tests/integration/test_register_dry_run_cli.py +++ b/tests/integration/test_register_dry_run_cli.py @@ -187,7 +187,7 @@ def test_register_honors_logging_config_for_console_and_file( config_text = config_text.replace("file = false", "file = true") else: config_text = ( - f"{config_text.rstrip()}\n\n[logging]\nlevel = \"debug\"\nconsole = true\nfile = true\n" + f'{config_text.rstrip()}\n\n[logging]\nlevel = "debug"\nconsole = true\nfile = true\n' ) config_path.write_text(config_text, encoding="utf-8") diff --git a/tests/integrations/git/test_provider.py b/tests/integrations/git/test_provider.py new file mode 100644 index 00000000..b9fe6051 --- /dev/null +++ b/tests/integrations/git/test_provider.py @@ -0,0 +1,30 @@ +"""Tests for git VCS provider behavior.""" + +from unittest.mock import patch + +from roar.integrations.git.provider import GitVCSProvider + + +def test_get_info_returns_empty_when_git_binary_is_missing(tmp_path) -> None: + provider = GitVCSProvider() + + with patch.object(provider, "is_available", return_value=False): + info = provider.get_info(str(tmp_path)) + + assert info.commit is None + assert info.branch is None + assert info.remote_url is None + assert info.clean is True + assert info.uncommitted_changes == [] + + +def test_get_status_returns_clean_when_git_binary_is_missing(tmp_path) -> None: + provider = GitVCSProvider() + + with patch( + "roar.integrations.git.provider.subprocess.check_output", + side_effect=FileNotFoundError("git"), + ): + clean, changes = provider.get_status(str(tmp_path)) + + assert (clean, changes) == (True, []) diff --git a/tests/integrations/glaas/test_client.py b/tests/integrations/glaas/test_client.py index df239e79..b78a0d32 100644 --- a/tests/integrations/glaas/test_client.py +++ b/tests/integrations/glaas/test_client.py @@ -219,7 +219,9 @@ def test_authenticated_probe_is_cached_after_first_success(self): assert first_result == {"id": 1} assert second_result == {"id": 2} assert mock_urlopen.call_count == 3 - assert mock_urlopen.call_args_list[0][0][0].full_url.endswith("/api/v1/sessions?limit=1") + assert mock_urlopen.call_args_list[0][0][0].full_url.endswith( + "/api/v1/sessions?limit=1" + ) assert mock_urlopen.call_args_list[1][0][0].full_url.endswith("/api/v1/test") assert mock_urlopen.call_args_list[2][0][0].full_url.endswith("/api/v1/test-2") @@ -360,7 +362,9 @@ def test_anonymous_fallback_is_cached_after_first_auth_failure(self): assert first_result == {"id": 1} assert second_result == {"id": 2} assert mock_urlopen.call_count == 3 - assert mock_urlopen.call_args_list[0][0][0].full_url.endswith("/api/v1/sessions?limit=1") + assert mock_urlopen.call_args_list[0][0][0].full_url.endswith( + "/api/v1/sessions?limit=1" + ) assert mock_urlopen.call_args_list[1][0][0].full_url.endswith("/api/v1/test") assert mock_urlopen.call_args_list[1][0][0].get_header("Authorization") is None assert mock_urlopen.call_args_list[2][0][0].full_url.endswith("/api/v1/test-2") diff --git a/tests/live_glaas/test_labels_live.py b/tests/live_glaas/test_labels_live.py index 6bfd4659..dbb4c69c 100644 --- a/tests/live_glaas/test_labels_live.py +++ b/tests/live_glaas/test_labels_live.py @@ -71,9 +71,7 @@ def _clear_remote_label_storage() -> None: def _clear_remote_labels(_serialize_external_label_tests): del _serialize_external_label_tests rows = composite_live._db_query_rows("SELECT 1 AS ok") - assert rows and str(rows[0].get("ok")) == "1", ( - f"Unexpected GLaaS database probe result: {rows}" - ) + assert rows and str(rows[0].get("ok")) == "1", f"Unexpected GLaaS database probe result: {rows}" _clear_remote_label_storage() diff --git a/tests/unit/test_file_classifier_perf.py b/tests/unit/test_file_classifier_perf.py index fa3f3b03..5ebb221b 100644 --- a/tests/unit/test_file_classifier_perf.py +++ b/tests/unit/test_file_classifier_perf.py @@ -9,8 +9,10 @@ """ import os +import subprocess import sys import time +from unittest.mock import patch class TestBuildPackageFileMapPerf: @@ -142,3 +144,20 @@ def test_classify_non_site_packages_not_mislabeled(self, tmp_path): f"json.__file__ classified as {kind!r}; expected stdlib/system/skip/unmanaged. " f"Path: {json_file}" ) + + def test_classify_in_repo_file_tolerates_missing_git_binary(self, tmp_path): + """Missing git should degrade to unmanaged instead of crashing classification.""" + from roar.filters.files import FileClassifier + + repo_file = tmp_path / "task.py" + repo_file.write_text("print('hello')\n", encoding="utf-8") + fc = FileClassifier( + repo_root=str(tmp_path), + sys_prefix=sys.prefix, + sys_base_prefix=sys.base_prefix, + ) + + with patch.object(subprocess, "check_output", side_effect=FileNotFoundError("git")): + kind, pkg = fc.classify(str(repo_file)) + + assert (kind, pkg) == ("unmanaged", None) diff --git a/tests/unit/test_sync_packaged_rust_artifacts.py b/tests/unit/test_sync_packaged_rust_artifacts.py index a5354b0e..bda78440 100644 --- a/tests/unit/test_sync_packaged_rust_artifacts.py +++ b/tests/unit/test_sync_packaged_rust_artifacts.py @@ -3,6 +3,7 @@ import os from pathlib import Path +import scripts.sync_packaged_rust_artifacts as sync_module from scripts.sync_packaged_rust_artifacts import ( ArtifactSpec, SyncLayout, @@ -128,3 +129,50 @@ def test_sync_packaged_rust_artifacts_copies_release_outputs(tmp_path: Path) -> assert (layout.package_bin_dir / "libroar_tracer_preload.so").read_text( encoding="utf-8" ) == "release-library\n" + + +def test_sync_packaged_rust_artifacts_uses_fallback_release_dir_without_rebuild( + tmp_path: Path, monkeypatch +) -> None: + root_dir = tmp_path + rust_manifest = root_dir / "rust" / "Cargo.toml" + portable_release_dir = root_dir / "rust" / "target" / "x86_64-unknown-linux-gnu" / "release" + host_release_dir = root_dir / "rust" / "target" / "release" + package_bin_dir = root_dir / "roar" / "bin" + layout = SyncLayout( + root_dir=root_dir, + rust_manifest=rust_manifest, + release_dir=portable_release_dir, + fallback_release_dirs=(host_release_dir,), + package_bin_dir=package_bin_dir, + artifacts=( + ArtifactSpec( + package_name="roar-tracer", + source_paths=( + rust_manifest, + root_dir / "rust" / "Cargo.lock", + root_dir / "rust" / "tracers" / "ptrace", + ), + binary_names=("roar-tracer",), + ), + ), + portable_target="x86_64-unknown-linux-gnu.2.17", + ) + + _write_file(layout.rust_manifest, "[workspace]\n") + _write_file(layout.root_dir / "rust" / "Cargo.lock", "") + _write_file( + layout.root_dir / "rust" / "tracers" / "ptrace" / "src" / "main.rs", "ptrace source\n" + ) + _write_file(host_release_dir / "roar-tracer", "release-tracer\n") + + def fail_build(*args, **kwargs): + raise AssertionError("expected fallback release artifact to avoid rebuild") + + monkeypatch.setattr(sync_module.subprocess, "run", fail_build) + + sync_packaged_rust_artifacts(layout) + + assert (layout.package_bin_dir / "roar-tracer").read_text( + encoding="utf-8" + ) == "release-tracer\n" diff --git a/uv.lock b/uv.lock index 22eadb80..46df7990 100644 --- a/uv.lock +++ b/uv.lock @@ -1276,6 +1276,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + [[package]] name = "requests" version = "2.32.5" @@ -1304,6 +1368,7 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, { name = "pysqlite3-binary", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "pyyaml" }, { name = "sqlalchemy" }, { name = "tomli", marker = "python_full_version < '3.11'" }, ] @@ -1339,6 +1404,7 @@ requires-dist = [ { name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.10.0" }, { name = "pytest-timeout", marker = "extra == 'dev'", specifier = ">=2.0.0" }, { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5.0" }, + { name = "pyyaml", specifier = ">=6.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.8.0" }, { name = "sqlalchemy", specifier = ">=2.0.0" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.0" },