From 7b8e253b52a750430013f1639814e1956b628c70 Mon Sep 17 00:00:00 2001 From: Vlad Stirbu Date: Mon, 26 Jan 2026 16:14:14 +0200 Subject: [PATCH 1/7] feat: enhance job environment with project and git information --- src/q8s/execution.py | 7 ++++++- src/q8s/plugins/cpu_job.py | 3 ++- src/q8s/plugins/job.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/q8s/execution.py b/src/q8s/execution.py index 108c22e..45e065a 100644 --- a/src/q8s/execution.py +++ b/src/q8s/execution.py @@ -16,6 +16,7 @@ from q8s.plugins.cpu_job import CPUJobTemplatePlugin from q8s.plugins.cuda_job import CUDAJobTemplatePlugin from q8s.plugins.job_template_spec import JobTemplatePluginSpec +from q8s.project import Project from q8s.utils import extract_non_none_value from .workload import Workload @@ -140,6 +141,7 @@ def __init__(self, kubeconfig: str, logger=None, progress: Progress = None): self.__env = load_env() self.jupyter_logger = logger + self.project = Project() @staticmethod def get_id(): @@ -206,7 +208,10 @@ def __create_job_object_from_workload(self, workload: Workload) -> client.V1Job: metadata=client.V1ObjectMeta( name=self.name, namespace=self.namespace, - labels={"qubernetes.dev/job.type": "jupyter"}, + labels={ + # "qubernetes.dev/job.type": "jupyter", + "qubernetes.dev/project.name": self.project.name, + }, ), spec=spec, ) diff --git a/src/q8s/plugins/cpu_job.py b/src/q8s/plugins/cpu_job.py index 12807d8..d4fa7d4 100644 --- a/src/q8s/plugins/cpu_job.py +++ b/src/q8s/plugins/cpu_job.py @@ -27,10 +27,11 @@ def makejob( volume_name = f"app-volume-{name}" env_var = list(env) + if workload.is_src_project: env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src")) - self.patch_environment_with_git_info(env_var) + env_var = self.patch_environment(env_var) container = client.V1Container( name="quantum-routine", diff --git a/src/q8s/plugins/job.py b/src/q8s/plugins/job.py index 496cb0b..4e01dad 100644 --- a/src/q8s/plugins/job.py +++ b/src/q8s/plugins/job.py @@ -3,6 +3,7 @@ from kubernetes import client from q8s.plugins.utils.git_info import get_git_info +from q8s.project import Project class JobPlugin: @@ -18,6 +19,21 @@ class JobPlugin: - GIT_PYTHON_REFRESH: Set to "quiet" to suppress GitPython refresh warnings. """ + def patch_environment(self, env: list[client.V1EnvVar]) -> list[client.V1EnvVar]: + """ + Patch environment variables for the job with Git and Project information + + Args: + env (list[client.V1EnvVar]): Original environment variables + Returns: + list[client.V1EnvVar]: Patched environment variables + """ + + env = self.patch_environment_with_git_info(env) + env = self.patch_environment_with_project_info(env) + + return env + def patch_environment_with_git_info( self, env: list[client.V1EnvVar] ) -> list[client.V1EnvVar]: @@ -52,3 +68,21 @@ def patch_environment_with_git_info( env.append(client.V1EnvVar(name="GIT_PYTHON_REFRESH", value="quiet")) return env + + def patch_environment_with_project_info( + self, env: list[client.V1EnvVar] + ) -> list[client.V1EnvVar]: + """ + Patch environment variables for the job with Project information + + Args: + env (list[client.V1EnvVar]): Original environment variables + Returns: + list[client.V1EnvVar]: Patched environment variables + """ + + project = Project() + + env.append(client.V1EnvVar(name="Q8S_PROJECT_NAME", value=project.name)) + + return env From 706b1482d440b8385d0b84f9326a19275533b093 Mon Sep 17 00:00:00 2001 From: Vlad Stirbu Date: Mon, 26 Jan 2026 16:44:02 +0200 Subject: [PATCH 2/7] feat: add job creator metadata and enhance environment variable handling in CPU and CUDA job plugins --- src/q8s/execution.py | 1 + src/q8s/plugins/cpu_job.py | 5 ++--- src/q8s/plugins/cuda_job.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/q8s/execution.py b/src/q8s/execution.py index 45e065a..0a5ab28 100644 --- a/src/q8s/execution.py +++ b/src/q8s/execution.py @@ -210,6 +210,7 @@ def __create_job_object_from_workload(self, workload: Workload) -> client.V1Job: namespace=self.namespace, labels={ # "qubernetes.dev/job.type": "jupyter", + "qubernetes.dev/created.by": "q8sctl", "qubernetes.dev/project.name": self.project.name, }, ), diff --git a/src/q8s/plugins/cpu_job.py b/src/q8s/plugins/cpu_job.py index d4fa7d4..60a14dc 100644 --- a/src/q8s/plugins/cpu_job.py +++ b/src/q8s/plugins/cpu_job.py @@ -26,13 +26,12 @@ def makejob( volume_name = f"app-volume-{name}" - env_var = list(env) + env_var = self.patch_environment(list(env)) + env_var.append(client.V1EnvVar(name="Q8S_JOB_NAME", value=name)) if workload.is_src_project: env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src")) - env_var = self.patch_environment(env_var) - container = client.V1Container( name="quantum-routine", image=container_image, diff --git a/src/q8s/plugins/cuda_job.py b/src/q8s/plugins/cuda_job.py index b68f25d..40331f8 100644 --- a/src/q8s/plugins/cuda_job.py +++ b/src/q8s/plugins/cuda_job.py @@ -33,12 +33,12 @@ def makejob( volume_name = f"app-volume-{name}" - env_var = list(env) + env_var = self.patch_environment(list(env)) + env_var.append(client.V1EnvVar(name="Q8S_JOB_NAME", value=name)) + if workload.is_src_project: env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src")) - self.patch_environment_with_git_info(env_var) - container = client.V1Container( name="quantum-routine", image=container_image, From 469bccf29e2c6f99511859b0f7d8809426e685c5 Mon Sep 17 00:00:00 2001 From: Vlad Stirbu Date: Mon, 26 Jan 2026 17:31:12 +0200 Subject: [PATCH 3/7] feat: add mock patch for project loading in CPU and CUDA job template tests --- tests/test_job_template_spec.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_job_template_spec.py b/tests/test_job_template_spec.py index 62c1488..c89f72a 100644 --- a/tests/test_job_template_spec.py +++ b/tests/test_job_template_spec.py @@ -6,10 +6,12 @@ from q8s.enums import Target from q8s.plugins.cpu_job import CPUJobTemplatePlugin from q8s.plugins.cuda_job import CUDAJobTemplatePlugin +from tests.test_project import mocked_configuration class TestCPUandGPUJobTemplatePlugin(unittest.TestCase): + @unittest.mock.patch("q8s.project.load", return_value=mocked_configuration) @patch("q8s.plugins.job_template_spec.client.V1Container") @patch("q8s.plugins.job_template_spec.client.V1PodTemplateSpec") @patch("q8s.plugins.job_template_spec.client.V1PodSpec") @@ -30,6 +32,7 @@ def test_makejob_cpu( mock_v1_pod_spec, mock_v1_pod_template_spec, mock_v1_container, + mock_load_project, ): plugin = CPUJobTemplatePlugin() name = "test-job" @@ -64,6 +67,7 @@ def test_makejob_cpu( mock_v1_volume.assert_called_once() mock_v1_config_map_volume_source.assert_called_once() + @unittest.mock.patch("q8s.project.load", return_value=mocked_configuration) @patch("q8s.plugins.job_template_spec.client.V1Container") @patch("q8s.plugins.job_template_spec.client.V1PodTemplateSpec") @patch("q8s.plugins.job_template_spec.client.V1PodSpec") @@ -84,6 +88,7 @@ def test_makejob_gpu( mock_v1_pod_spec, mock_v1_pod_template_spec, mock_v1_container, + mock_load_project, ): plugin = CUDAJobTemplatePlugin() name = "test-job" From c121467772ea5a484ee655f1aa66a0c31ad53b28 Mon Sep 17 00:00:00 2001 From: Vlad Stirbu Date: Tue, 27 Jan 2026 18:41:43 +0200 Subject: [PATCH 4/7] feat: add multirun command with Hydra configuration support and update job name assignment --- pyproject.toml | 1 + src/q8s/cli.py | 103 ++++++++++++++++++++++++++++++++++++++++ src/q8s/execution.py | 4 +- tests/test_execution.py | 2 +- 4 files changed, 107 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a0d4867..399b692 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "typer>=0.15.4", "GitPython>=3.1.43", "python-dxf>=12.1.1", + "hydra-core>=1.2.0", ] requires-python = ">= 3.10" diff --git a/src/q8s/cli.py b/src/q8s/cli.py index 164563b..09b55a1 100644 --- a/src/q8s/cli.py +++ b/src/q8s/cli.py @@ -1,9 +1,13 @@ +import base64 import importlib +import os import sys from pathlib import Path from subprocess import Popen +import hydra import typer +from omegaconf import DictConfig, OmegaConf from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn from typing_extensions import Annotated @@ -93,6 +97,105 @@ def build( project.update_images_cache() +def _multirun_with_cfg( + *, + cfg: DictConfig, + k8sctx: K8sContext, + workload: Workload, + script_args: list[str], +) -> None: + cfg_str = OmegaConf.to_yaml(cfg=cfg, resolve=True) + b64_cfg = base64.b64encode(cfg_str.encode("utf-8")).decode("ascii") + workload.set_args([b64_cfg]) + + k8sctx.execute_workload(workload=workload, submit=True) + + +@app.command( + context_settings={ + "allow_extra_args": True, + "ignore_unknown_options": True, + } +) +def multirun( + file: Annotated[Path, typer.Argument(help="Python file to be executed")], + config: Annotated[Path, typer.Option(help="Hydra config file")] = None, + target: Annotated[ + Target, typer.Option(help="Execution target", case_sensitive=False) + ] = Target.gpu, + kubeconfig: Annotated[ + Path, typer.Option(help="Kubernetes configuration", envvar="KUBECONFIG") + ] = None, + image: Annotated[str, typer.Option(help="Docker image")] = None, + registry_pat: Annotated[ + str, + typer.Option( + help="Registry personal access token (PAT)", + envvar="REGISTRY_PAT", + ), + ] = None, + args: Annotated[list[str], typer.Argument(help="Additional arguments")] = None, +): + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + TimeElapsedColumn(), + expand=True, + ) as progress: + task_project = progress.add_task( + description="[cyan]Loading project...", total=1 + ) + project = Project() + progress.advance(task_project) + + if image is None: + image = project.cached_images(target.value) + + if kubeconfig is None: + kubeconfig = project.kubeconfig + + if kubeconfig.exists() is False: + typer.echo(f"kubeconfig file {kubeconfig} does not exist") + raise typer.Exit(code=1) + + if kubeconfig is None: + typer.echo("KUBECONFIG not set") + raise typer.Exit(code=1) + + k8s_context = K8sContext(kubeconfig.as_posix(), progress=progress) + k8s_context.set_target(target) + k8s_context.set_registry_pat(registry_pat) + k8s_context.set_container_image(image) + + workload = Workload.from_entry_script(entry_script=file) + + @hydra.main( + version_base=None, + config_path=os.getcwd(), + config_name=config.stem, + ) + def _hydra_app(cfg: DictConfig): + _multirun_with_cfg( + cfg=cfg, + k8sctx=k8s_context, + workload=workload, + script_args=args or [], + ) + + try: + old_argv = sys.argv + sys.argv = [ + "q8sctl", + "-m", + "hydra.job.chdir=False", + "hydra.run.dir=.", + ] + + _hydra_app() + finally: + sys.argv = old_argv + + @app.command( context_settings={ "allow_extra_args": True, diff --git a/src/q8s/execution.py b/src/q8s/execution.py index 0a5ab28..c253994 100644 --- a/src/q8s/execution.py +++ b/src/q8s/execution.py @@ -136,8 +136,6 @@ def __init__(self, kubeconfig: str, logger=None, progress: Progress = None): self.core_api_instance = client.CoreV1Api() self.batch_api_instance = client.BatchV1Api() - self.name = f"qubernetes-job-{K8sContext.get_id()}" - self.__env = load_env() self.jupyter_logger = logger @@ -496,6 +494,8 @@ def execute_workload( Execute the given workload. """ + self.name = f"qubernetes-job-{K8sContext.get_id()}" + try: self.__create_job_object_from_workload(workload=workload) diff --git a/tests/test_execution.py b/tests/test_execution.py index 85a6e0d..1d79b8a 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -124,7 +124,7 @@ def _make_context(self): progress.add_task.return_value = "task-id" ctx._K8sContext__progress = progress ctx.jupyter_logger = Mock() - ctx.name = "qubernetes-job-test" + K8sContext.get_id = Mock(return_value="test") return ctx, progress def test_execute_workload_submit_false_returns_logs(self): From 2b35d481376dcdcfe1019fae770c7889feec27a8 Mon Sep 17 00:00:00 2001 From: Vlad Stirbu Date: Tue, 27 Jan 2026 18:47:01 +0200 Subject: [PATCH 5/7] fix: update project version to 0.14.0-dev0 in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 399b692..76e961d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "q8s" -version = "0.13.0" +version = "0.14.0-dev0" description = "Kernel extension for executing quantum programs in simulators on q8s clusters" authors = [{ name = "Vlad Stirbu", email = "vstirbu@gmail.com" }] readme = "README.md" From e54e0e8c65493cc42fd4767859f22226a8bd56ba Mon Sep 17 00:00:00 2001 From: Vlad Stirbu Date: Wed, 28 Jan 2026 07:17:34 +0200 Subject: [PATCH 6/7] feat: add workloads runtime utilities library --- pyproject.toml | 3 ++- src/q8s/runtime/__init__.py | 0 src/q8s/runtime/decorators.py | 51 +++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 src/q8s/runtime/__init__.py create mode 100644 src/q8s/runtime/decorators.py diff --git a/pyproject.toml b/pyproject.toml index 76e961d..7a13504 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "q8s" version = "0.14.0-dev0" -description = "Kernel extension for executing quantum programs in simulators on q8s clusters" +description = "CLI and Jupyter kernel extension for executing quantum programs in simulators on Kubernetes clusters" authors = [{ name = "Vlad Stirbu", email = "vstirbu@gmail.com" }] readme = "README.md" license = { file = "LICENSE" } @@ -29,6 +29,7 @@ dependencies = [ "GitPython>=3.1.43", "python-dxf>=12.1.1", "hydra-core>=1.2.0", + "omegaconf", ] requires-python = ">= 3.10" diff --git a/src/q8s/runtime/__init__.py b/src/q8s/runtime/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/q8s/runtime/decorators.py b/src/q8s/runtime/decorators.py new file mode 100644 index 0000000..8effd71 --- /dev/null +++ b/src/q8s/runtime/decorators.py @@ -0,0 +1,51 @@ +import base64 +import sys +import traceback +from dataclasses import is_dataclass +from functools import wraps +from typing import Type, TypeVar + +from omegaconf import OmegaConf + +T = TypeVar("T") + + +def with_app_config(config_cls: Type[T]): + """ + Decorator that: + - reads base64-encoded config from sys.argv[1] + - merges it with a structured OmegaConf dataclass + - resolves the config + - passes the config instance into the function + """ + + if not is_dataclass(config_cls): + raise TypeError("config_cls must be a dataclass") + + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + cfg = None + trace: str = None + + if len(sys.argv) < 2: + raise RuntimeError( + "Missing base64-encoded config argument (expected sys.argv[1])" + ) + + try: + raw_input = base64.b64decode(sys.argv[1]).decode("utf-8") + + cfg = OmegaConf.merge( + OmegaConf.structured(config_cls), + OmegaConf.create(raw_input), + ) + OmegaConf.resolve(cfg) + except Exception: + trace = traceback.format_exc() + + return func(cfg, trace, *args, **kwargs) + + return wrapper + + return decorator From b25635278442324e5bc4918553b2f90d930bff35 Mon Sep 17 00:00:00 2001 From: Vlad Stirbu Date: Wed, 28 Jan 2026 07:28:37 +0200 Subject: [PATCH 7/7] fix: update project version to 0.14.0-dev1 in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7a13504..45c2d6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "q8s" -version = "0.14.0-dev0" +version = "0.14.0-dev1" description = "CLI and Jupyter kernel extension for executing quantum programs in simulators on Kubernetes clusters" authors = [{ name = "Vlad Stirbu", email = "vstirbu@gmail.com" }] readme = "README.md"