diff --git a/.github/workflows/build_and_deploy.yaml b/.github/workflows/build_and_deploy.yaml index d4a7244..8e26984 100644 --- a/.github/workflows/build_and_deploy.yaml +++ b/.github/workflows/build_and_deploy.yaml @@ -1,14 +1,13 @@ name: Build and Deploy to PyPI -on: [workflow_dispatch] +on: + workflow_dispatch: + push: + tags: + - "*" jobs: build_and_deploy: - strategy: - matrix: - os: [ubuntu-22.04, ubuntu-latest, macos-14, macos-latest] - python-version: ["3.11", "3.12", "3.13"] - runs-on: ${{ matrix.os }} - continue-on-error: true + runs-on: ubuntu-latest steps: - name: Checkout Source Code @@ -17,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: ${{ matrix.python-version }} + python-version: "3.11" - name: Install Dependencies run: make setup-build diff --git a/.github/workflows/upload_coverage.yaml b/.github/workflows/upload_coverage.yaml index 428c452..67c3067 100644 --- a/.github/workflows/upload_coverage.yaml +++ b/.github/workflows/upload_coverage.yaml @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13"] + python-version: [ "3.11", "3.12", "3.13" ] steps: - name: Checkout Source Code uses: actions/checkout@v6 @@ -39,7 +39,7 @@ jobs: run: make setup-test - name: Run tests - run: pytest + run: poetry run pytest - name: Upload results to Codecov if: matrix.python-version == '3.11' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d842cee..29c0af2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,6 +25,8 @@ repos: rev: 8.0.1 hooks: - id: isort + pass_filenames: false + args: [eclypse] - repo: https://github.com/commitizen-tools/commitizen rev: v4.13.9 @@ -41,7 +43,11 @@ repos: args: [--fix, --select, "F401"] name: ruff pycln - id: ruff-check + pass_filenames: false + args: [eclypse] - id: ruff-format + pass_filenames: false + args: [eclypse] - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.20.0 @@ -49,6 +55,5 @@ repos: - id: mypy pass_filenames: false args: [eclypse] - exclude: ^(docs/|tests/) exclude: "examples/.*" diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index d1f211b..0000000 --- a/.pylintrc +++ /dev/null @@ -1,73 +0,0 @@ -[MAIN] - -# Add files or directories matching the regex patterns to the ignore-list. The -# regex matches against paths and can be in Posix or Windows format. -ignore-paths=tests,examples,docs - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. -jobs=0 - -# Minimum supported python version -py-version=3.11 - -# Specify a score threshold under which the program will exit with error. -fail-under=10.0 - -# Clear in-memory caches upon conclusion of linting. Useful if running pylint in -# a server-like mode. -clear-cache-post-run=no - -[MESSAGES CONTROL] - -disable=C0114,C0115,C0116,R0401,W0718,E0401 - -[REPORTS] - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=no - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=20 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=yes - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=120 - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines="^\s*(# )??$" - -[DESIGN] - -# Maximum number of arguments for function / method (see R0913). -max-args=20 - -# Maximum number of locals for function / method body (see R0914). -max-locals=30 - -# Maximum number of attributes for a class (see R0902). -max-attributes=15 - -# Maximum number of positional arguments for function / method (see R0917). -max-positional-arguments=20 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=1 - -# R0912 -max-branches=15 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b83c8ea..e6c10f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,7 +104,7 @@ We use `pytest` for the test suite. Useful command: ```bash -make test +pytest ``` ## Pre-commit Hooks diff --git a/Makefile b/Makefile index c606303..109269a 100644 --- a/Makefile +++ b/Makefile @@ -13,12 +13,13 @@ setup: python -m pip install --upgrade pip pip install poetry poetry config virtualenvs.create false + echo "Poetry virtualenv creation disabled for CI. To re-enable, run `poetry config virtualenvs.create true`." setup-build: setup poetry install --with=dev,deploy --no-root setup-test: setup - poetry install --with=test --extras remote --extras tboard --no-root + poetry install --with=test --no-root format: isort eclypse diff --git a/README.md b/README.md index 002b4ee..1148995 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,8 @@ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Checked with mypy](http://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/) -[![Checked with pylint](https://img.shields.io/badge/pylint-10/10-green)](https://pylint.pycqa.org/en/latest/) [![Import sorted with isort](https://img.shields.io/badge/isort-checked-brightgreen)](https://pycqa.github.io/isort/) -[![Import cleaned with pycln](https://img.shields.io/badge/pycln-checked-brightgreen)](https://github.com/hadialqattan/pycln) -[![Code style: black](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black) -[![Doc style: docformatter](https://img.shields.io/badge/doc%20style-docformatter-black)](https://github.com/PyCQA/docformatter) +[![Doc style: docformatter](https://img.shields.io/badge/doc%20style-docformatter-blue)](https://github.com/PyCQA/docformatter) **ECLYPSE** (**E**dge-**CL**oud p**Y**thon **P**latform for **S**imulated runtime **E**nvironments) is the first simulation library entirely written in Python, for experimenting with deployment strategies in varying infrastructure conditions. It provides an interface to simulate deployments of service-based applications onto life-like infrastructures, without and with an actual application implementation to be deployed. diff --git a/docs/Makefile b/docs/Makefile index 663e009..3faee1e 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line, and also # from the environment for the first two. -SPHINXOPTS ?= +SPHINXOPTS ?= -W SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = _build @@ -19,9 +19,7 @@ help: livehtml: sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)"/html $(SPHINXOPTS) $(O) -# check docstrings using pylint in ../eclypse module check: -# pylint --rcfile=../.pylintrc --enable=C0114,C0115,C0116 --disable=R0801,E0611 ../eclypse ruff check --select D,E501 ../eclypse .PHONY: help Makefile diff --git a/docs/conf.py b/docs/conf.py index 4cf658e..346dbc0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,7 +35,6 @@ "sphinx.ext.viewcode", "sphinx.ext.autosectionlabel", "sphinx.ext.coverage", - "sphinx.ext.napoleon", "sphinxcontrib.icon", "sphinx_copybutton", "sphinx_favicon", @@ -48,7 +47,31 @@ myst_enable_extensions = ["colon_fence"] templates_path = ["_templates"] -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "README.md"] +exclude_patterns = [ + "_build", + "Thumbs.db", + ".DS_Store", + "README.md", + "source/api/reference/graph/application/Application/" + "eclypse.graph.application.Application.has_logic.rst", + "source/api/reference/graph/infrastructure/Infrastructure/" + "eclypse.graph.infrastructure.Infrastructure.contains.rst", + "source/api/reference/graph/infrastructure/Infrastructure/" + "eclypse.graph.infrastructure.Infrastructure.has_strategy.rst", + "source/api/reference/workflow/trigger/bucket/TriggerBucket/" + "eclypse.workflow.trigger.bucket.TriggerBucket.init.rst", + "source/api/reference/workflow/trigger/cascade/RandomCascadeTrigger/" + "eclypse.workflow.trigger.cascade.RandomCascadeTrigger.init.rst", + "source/api/reference/workflow/trigger/trigger/RandomTrigger/" + "eclypse.workflow.trigger.trigger.RandomTrigger.init.rst", + "source/api/reference/workflow/trigger/trigger/ScheduledTrigger/" + "eclypse.workflow.trigger.trigger.ScheduledTrigger.init.rst", + "source/api/reference/workflow/trigger/trigger/Trigger/" + "eclypse.workflow.trigger.trigger.Trigger.init.rst", + "source/api/reference/workflow/event/decorator/" + "eclypse.workflow.event.decorator.event.rst", +] +suppress_warnings = ["toc.not_included"] coverage_show_missing_items = True # Automatically extract typehints when specified and place them in @@ -156,12 +179,9 @@ def patch_autosummary_name_collisions(): """Resolve package-level name collisions for autosummary generation. - The ``eclypse.workflow.event`` package re-exports the ``event`` decorator, - which shadows the ``event`` submodule when autosummary resolves dotted - names. During the docs build we point the package attribute to the submodule - so the generated module page documents ``eclypse.workflow.event.event`` - rather than the decorator function. The decorator remains documented through - ``eclypse.workflow.event.decorator``. + The ``eclypse.workflow.event`` package contains an ``event`` submodule. During + the docs build we point the package attribute to the submodule so the + generated module page documents ``eclypse.workflow.event.event`` reliably. Recent autosummary releases also expect package-level attributes for relative submodule entries such as ``simulation`` under ``eclypse`` and @@ -177,7 +197,6 @@ def patch_autosummary_name_collisions(): metrics_pkg.defaults = import_module("eclypse.report.metrics.defaults") event_pkg = import_module("eclypse.workflow.event") - event_pkg.decorator_event = event_pkg.event event_pkg.event = import_module("eclypse.workflow.event.event") diff --git a/docs/index.rst b/docs/index.rst index a67154b..cecdf97 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,7 +21,7 @@ ===================== ECLYPSE documentation ===================== -**ECLYPSE** (Edge-Cloud raY-based Platform for Simulated Environments) stands as a groundbreaking simulation library, crafted entirely in Python. +**ECLYPSE** (Edge-CLoud pYthon Platform for Simulated runtime Environments) stands as a groundbreaking simulation library, crafted entirely in Python. It offers a practical interface for experimenting with deployment strategies across different infrastructure settings. One of its key strengths lies in its ability to simulate the deployment of service-based applications in environments that closely mimic real-world conditions, with or without actual application implementation. diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 2c5d4ce..0000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -jinja2>=3.1.5 -pydata-sphinx-theme==0.16.0 -sphinx==7.2.6 -sphinx-copybutton==0.5.2 -sphinx-icon==0.2.2 diff --git a/docs/source/overview/advanced/reporting.rst b/docs/source/overview/advanced/reporting.rst index 538f387..b377fec 100644 --- a/docs/source/overview/advanced/reporting.rst +++ b/docs/source/overview/advanced/reporting.rst @@ -40,8 +40,8 @@ There are 7 decorators corresponding to different metric types: - :py:func:`~eclypse.report.metrics.metric.service` - :py:func:`~eclypse.report.metrics.metric.interaction` -See :ref:`event-decorator` for details on defining and triggering events. A -metric lets you specify: +See :ref:`event-decorator` for details on scheduled event helpers. A metric lets +you specify: - **What data to collect** - **How often to report** (using :doc:`triggers `) @@ -61,6 +61,28 @@ Example: .. note:: Metrics are executed like events, and use the same underlying logic, including support for cascade triggers and trigger conditions. +Custom metric recipe +~~~~~~~~~~~~~~~~~~~~ + +Use the metric decorator that matches the object you want to inspect, choose +when it activates, and return either a scalar or a mapping. + +.. code-block:: python + :caption: Custom service metric + + from eclypse.report.metrics import metric + + @metric.service(activates_on="step", report=["csv"]) + def requested_cpu(service_id, requirements, placement, infrastructure): + return { + "service": service_id, + "value": requirements["cpu"], + } + +The reporting pipeline records the event metadata and stores the returned +``value``. For multi-field mappings, keep a stable ``value`` key for the main +measurement and use the other keys as context. + Metric and callback types ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -162,6 +184,14 @@ Example usage: report = Report("./output") df = report.service(application_ids="app1", service_ids="srv2") +For a quick run summary, call +:py:meth:`~eclypse.report.report.Report.describe`: + +.. code-block:: python + + print(report.describe()) + # 84 rows x 10 steps x 12 metrics | 5 applications | ... + Each accessor method supports filtering by: - `report_range` (e.g., only events between 10 and 100) diff --git a/docs/source/overview/advanced/triggers.rst b/docs/source/overview/advanced/triggers.rst index 7cb81aa..0ff8033 100644 --- a/docs/source/overview/advanced/triggers.rst +++ b/docs/source/overview/advanced/triggers.rst @@ -5,6 +5,9 @@ Triggers are conditions that determine **when an event should fire** during the Each trigger must implement a :py:meth:`~eclypse.workflow.trigger.trigger.Trigger.trigger` method that returns ``True`` if the event should be executed at that moment. +Triggers that need internal state can also override +:py:meth:`~eclypse.workflow.trigger.trigger.Trigger.prepare` and +:py:meth:`~eclypse.workflow.trigger.trigger.Trigger.reset`. ECLYPSE provides both: @@ -64,11 +67,11 @@ The tables below compares all available trigger types: - ``probability: float`` - ``seed: int (optional)`` -Define triggers in the @event decorator ----------------------------------------- +Define triggers in scheduled decorators +--------------------------------------- You can define cascade triggers more compactly, using the ``activates_on`` parameter -in the :ref:`@event decorator `: +in the :ref:`scheduled event decorators `: .. list-table:: :header-rows: 1 @@ -84,6 +87,49 @@ in the :ref:`@event decorator `: * - ``("event_name", 0.2)`` - RandomCascadeTrigger("event_name", 0.2) +Schedule helpers +---------------- + +For the most common scheduling cases, import the helper decorators directly +from :mod:`eclypse.workflow`: + +.. code-block:: python + + from eclypse.workflow import after, every, once_at + + @every(ms=500, event_type="simulation") + def heartbeat(triggering_event): + return {"value": triggering_event.n_triggers} + + @after(sim_seconds=10) + def warmup_complete(): + return {"value": True} + + @once_at(sim_seconds=60) + def final_checkpoint(): + return {"value": True} + +``@every`` creates a :class:`~eclypse.workflow.trigger.trigger.PeriodicTrigger`. +``@after`` and ``@once_at`` create +:class:`~eclypse.workflow.trigger.trigger.ScheduledTrigger` instances and +default to one firing. + +Trigger lifecycle +----------------- + +The simulator prepares every registered trigger bucket before the run starts. +The state machine is: + +#. ``prepare()``: allocate state such as scheduled timestamps or random-number + generators. +#. ``trigger(...)``: evaluate whether the event should fire. +#. event execution. +#. ``reset()``: update post-execution state before the next evaluation. + +If you implement a custom trigger that depends on prepared state, raise a clear +error from ``trigger()`` when ``prepare()`` has not been called. This mirrors the +built-in scheduled and random triggers. + Having multiple triggers ------------------------ @@ -95,23 +141,36 @@ When an event is associated with **multiple triggers**, the ``activates_on`` par .. code-block:: python :caption: **Example:** Using multiple triggers with different conditions - @event(event_type="application", - triggers=[ - PeriodicTrigger(500), - CascadeTrigger("check_resources") + from eclypse.workflow import every + from eclypse.workflow.trigger import CascadeTrigger + + @every( + ms=500, + event_type="application", + triggers=[ + CascadeTrigger("check_resources"), ], trigger_condition="any" # event fires on either ) def log_app_health(application, placement, infrastructure, **event_data): ... -You can also set this field when manually instantiating an -:class:`~eclypse.workflow.event.event.EclypseEvent`: +For more particular workflows, subclass +:class:`~eclypse.workflow.event.event.EclypseEvent` and pass custom triggers to +``super().__init__``: .. code-block:: python - event = EclypseEvent( - name="monitor", - triggers=[PeriodicTrigger(1000), CascadeTrigger("step")], - trigger_condition="all" # fires only if both trigger - ) + from eclypse.workflow.event import EclypseEvent + from eclypse.workflow.trigger import CascadeTrigger, PeriodicTrigger + + class Monitor(EclypseEvent): + def __init__(self): + super().__init__( + name="monitor", + triggers=[PeriodicTrigger(1000), CascadeTrigger("step")], + trigger_condition="all", # fires only if both trigger + ) + + def __call__(self, triggering_event): + return {"source": triggering_event.name} diff --git a/docs/source/overview/concepts/events.rst b/docs/source/overview/concepts/events.rst index ae3ed48..84975a4 100644 --- a/docs/source/overview/concepts/events.rst +++ b/docs/source/overview/concepts/events.rst @@ -161,32 +161,36 @@ Regular events, callbacks, and metrics can all use the same event types. For exa .. _event-decorator: -@event() decorator -~~~~~~~~~~~~~~~~~~ +Scheduled event decorators +~~~~~~~~~~~~~~~~~~~~~~~~~~ -The simplest way to define an event and its parameters is the -:py:func:`@event() ` decorator. +The public decorator API exposes the common scheduling shapes directly: -This flexible decorator allows you to register both functions and classes as **simulation events**, giving full control over when and how they are triggered. You can apply the decorator to: +- :py:func:`@every(ms=...) ` for periodic events, +- :py:func:`@after(sim_seconds=...) ` for delayed events, +- :py:func:`@once_at(sim_seconds=...) ` for one-shot events. -- A **function**, which becomes the logic of the event -- A **class** (with a ``__call__`` method), to maintain internal state +These decorators allow you to register both functions and callable classes as +**simulation events**. You can apply them to: + +- a **function**, which becomes the logic of the event, +- a **class** with a ``__call__`` method, to maintain internal state. .. code-block:: python :caption: Example: Decorating a *function* - from eclypse.workflow.event import event + from eclypse.workflow.event import every - @event(name="step_logger", event_type="simulation", activates_on=["step"]) + @every(ms=500, name="step_logger", event_type="simulation") def log_step(): print("Simulation step") .. code-block:: python :caption: Example: Decorating a *class* - from eclypse.workflow.event import event + from eclypse.workflow.event import once_at - @event(name="step_counter", event_type="simulation", activates_on=["step"]) + @once_at(sim_seconds=10, name="step_counter", event_type="simulation") class StepCounter: def __init__(self): self.counter = 0 @@ -198,14 +202,16 @@ This flexible decorator allows you to register both functions and classes as **s Callbacks --------- -Callbacks use the same decorator as regular events, but with a different role: +Callbacks use the same scheduled decorators as regular events, but with a +different role: .. code-block:: python :caption: Example: Defining a callback - from eclypse.workflow.event import EventRole, event + from eclypse.workflow.event import EventRole, after - @event( + @after( + sim_seconds=0, name="after_step", event_type="simulation", activates_on=["step"], @@ -220,6 +226,28 @@ Callbacks are best suited for: - deriving transient information from the triggering event, - reacting to another event without necessarily reporting the result. +If the scheduling decorators do not fit a particular workflow, define a custom +event by subclassing :class:`~eclypse.workflow.event.event.EclypseEvent` and +overriding ``__call__``. This gives full control over construction, state, and +trigger configuration: + +.. code-block:: python + :caption: Example: Custom event class + + from eclypse.workflow.event import EclypseEvent + from eclypse.workflow.trigger import CascadeTrigger + + class StepAudit(EclypseEvent): + def __init__(self): + super().__init__( + name="step_audit", + event_type="simulation", + triggers=[CascadeTrigger("step")], + ) + + def __call__(self, triggering_event): + return {"source": triggering_event.name} + .. _event-metrics: Metrics diff --git a/docs/source/overview/concepts/placement-strategy.rst b/docs/source/overview/concepts/placement-strategy.rst index 6f8f7a8..a219d64 100644 --- a/docs/source/overview/concepts/placement-strategy.rst +++ b/docs/source/overview/concepts/placement-strategy.rst @@ -1,7 +1,7 @@ Placement Strategy ================== -In ECLYPSE, a :class:`~eclypse.placement.strategies.strategy.PlacementStrategy` +In ECLYPSE, a :class:`~eclypse.placement.strategies.PlacementStrategy` defines how application services are assigned to infrastructure nodes. Placement can be performed globally across the infrastructure, or separately for each application. @@ -19,7 +19,7 @@ There are two ways to choose which strategy to use: :link-type: ref Subclass the abstract - :class:`~eclypse.placement.strategies.strategy.PlacementStrategy` base + :class:`~eclypse.placement.strategies.PlacementStrategy` base class or one of the built-in specialisations. .. grid-item:: @@ -36,9 +36,9 @@ Extend the PlacementStrategy class ---------------------------------- To define a custom placement strategy, subclass the base class -:class:`~eclypse.placement.strategies.strategy.PlacementStrategy` and override +:class:`~eclypse.placement.strategies.PlacementStrategy` and override the -:py:meth:`~eclypse.placement.strategies.strategy.PlacementStrategy.place` +:py:meth:`~eclypse.placement.strategies.PlacementStrategy.place` method. This method must return a mapping from **service IDs** to **node IDs**, representing where each service in the application should be deployed. @@ -62,7 +62,7 @@ This method must return a mapping from **service IDs** to **node IDs**, represen .. important:: The `infrastructure` parameter passed to the - :py:meth:`~eclypse.placement.strategies.strategy.PlacementStrategy.place` + :py:meth:`~eclypse.placement.strategies.PlacementStrategy.place` method is **already filtered** to include only the available portion of the infrastructure. @@ -80,29 +80,31 @@ ECLYPSE provides a collection of predefined placement strategies that can be use The available default strategies are: -- :class:`~eclypse.placement.strategies.round_robin.RoundRobinStrategy` — assigns services to nodes in a round-robin fashion. -- :class:`~eclypse.placement.strategies.random.RandomStrategy` — randomly selects a node for each service. -- :class:`~eclypse.placement.strategies.static.StaticStrategy` — expects service-to-node mappings to be provided statically. -- :class:`~eclypse.placement.strategies.first_fit.FirstFitStrategy` — places services on the first node that satisfies their requirements. -- :class:`~eclypse.placement.strategies.best_fit.BestFitStrategy` — selects the node with the tightest fit (smallest surplus) for each service. +- :class:`~eclypse.placement.strategies.RoundRobinStrategy` — assigns services to nodes in a round-robin fashion. +- :class:`~eclypse.placement.strategies.RandomStrategy` — randomly selects a node for each service. +- :class:`~eclypse.placement.strategies.StaticStrategy` — expects service-to-node mappings to be provided statically. +- :class:`~eclypse.placement.strategies.FirstFitStrategy` — places services on the first node that satisfies their requirements. +- :class:`~eclypse.placement.strategies.BestFitStrategy` — selects the node with the tightest fit (smallest surplus) for each service. Attaching a Placement Strategy ------------------------------ -To use a placement strategy during simulation, it must be associated with either an application or the infrastructure. +To use a placement strategy during simulation, provide it either as the +simulation default or for a specific application registration. There are two ways to attach a strategy: -- **Via the infrastructure:** - You can pass the strategy when instantiating the :class:`~eclypse.graph.infrastructure.Infrastructure` object: +- **Via ``SimulationConfig.default_strategy``:** + Use this when most applications should share the same placement policy: .. code-block:: python - from eclypse.graph.infrastructure import Infrastructure from eclypse.placement.strategies import RandomStrategy + from eclypse.simulation import Simulation, SimulationConfig - strategy = RandomStrategy() - infr = Infrastructure(..., placement_strategy=strategy) + config = SimulationConfig(default_strategy=RandomStrategy()) + sim = Simulation(infrastructure, config) + sim.register(application) - **Via the application registration:** You can associate a strategy when registering the application with the simulation using the :py:meth:`~eclypse.simulation.simulation.Simulation.register` method: @@ -117,6 +119,8 @@ There are two ways to attach a strategy: .. important:: - If **no strategy is attached** to either the application or the infrastructure, the simulation will raise an error. + If no strategy is provided through ``SimulationConfig.default_strategy`` or + ``Simulation.register(...)``, the simulation raises an error. - If **both** are provided, the strategy associated with the **application** takes precedence over the one defined in the infrastructure. + If both are provided, the strategy passed to ``register`` takes precedence + for that application. diff --git a/docs/source/overview/concepts/simulation-configuration.rst b/docs/source/overview/concepts/simulation-configuration.rst index 0d05522..3c2843d 100644 --- a/docs/source/overview/concepts/simulation-configuration.rst +++ b/docs/source/overview/concepts/simulation-configuration.rst @@ -23,7 +23,7 @@ The usual flow is: #. create a :class:`~eclypse.simulation.config.SimulationConfig`, #. create a :class:`~eclypse.simulation.simulation.Simulation`, #. register one or more applications with a placement strategy, -#. start the simulation and wait for completion. +#. run the simulation and wait for completion. .. code-block:: python @@ -42,8 +42,7 @@ The usual flow is: simulation = Simulation(example_infra, config) simulation.register(example_app, RandomStrategy(seed=42)) - simulation.start() - simulation.wait() + simulation.run() application_frame = simulation.report.application() @@ -61,10 +60,12 @@ The table below summarises every public parameter of - Default - Meaning * - ``step_every_ms`` - - ``"manual"`` + - ``"auto"`` - Controls how the driving ``enact`` event is scheduled. Use - ``"manual"`` for fully manual stepping, ``"auto"`` for continuous - progression, or a numeric value for a fixed periodic step. + ``"auto"`` for continuous local progression, ``"manual"`` or ``None`` + for fully manual stepping, or a numeric value for a fixed periodic step. + In remote mode, ``"auto"`` resolves to manual stepping unless a numeric + cadence is provided. * - ``timeout`` - ``None`` - Stops the simulation after the given number of seconds. @@ -82,6 +83,12 @@ The table below summarises every public parameter of * - ``include_default_metrics`` - ``False`` - Adds the built-in metrics shipped by ECLYPSE to the event set. + * - ``default_strategy`` + - ``None`` + - Placement strategy used for applications registered without an explicit + strategy. Per-application strategies passed to + :meth:`~eclypse.simulation.simulation.Simulation.register` take + precedence. * - ``seed`` - random - Seed used for deterministic sampling and reproducible scenarios. @@ -131,11 +138,11 @@ This configuration asks the simulation to: - stop after one minute, - or stop earlier if 120 steps are reached first. -If you want manual progression instead, keep the default: +If you want manual progression instead, opt in explicitly: .. code-block:: python - config = SimulationConfig(step_every_ms="manual") + config = SimulationConfig(step_every_ms=None) and then advance the run explicitly with :py:meth:`~eclypse.simulation.simulation.Simulation.step`. @@ -247,8 +254,7 @@ applications together with a placement strategy: simulation = Simulation(example_infra, config) simulation.register(example_app, RandomStrategy(seed=42)) - simulation.start() - simulation.wait() + simulation.run() When the run finishes, the :attr:`~eclypse.simulation.simulation.Simulation.report` property exposes a :class:`~eclypse.report.report.Report` object: diff --git a/docs/source/overview/concepts/topology.rst b/docs/source/overview/concepts/topology.rst index 39f6d19..760daaf 100644 --- a/docs/source/overview/concepts/topology.rst +++ b/docs/source/overview/concepts/topology.rst @@ -33,7 +33,6 @@ The two classes share many structural similarities, but differ in purpose and in edge_assets=[...], resource_init="min", seed=42, - placement_strategy=..., path_assets_aggregators=..., path_algorithm=..., ) @@ -45,7 +44,6 @@ The two classes share many structural similarities, but differ in purpose and in - ``node_assets`` / ``edge_assets``: available capabilities (:doc:`asset ` values) of nodes and links - ``resource_init``: initialisation of resources (*min* or *max*) - ``seed``: random seed for reproducibility - - ``placement_strategy``: global :doc:`placement strategy ` for all applications - ``path_assets_aggregators``: aggregators for *each link asset* evaluation across paths - ``path_algorithm``: path logic to retrieve and check the paths among nodes @@ -199,39 +197,69 @@ assets and flows. .. code-block:: python from eclypse.builders.infrastructure import ( - b_cube, - fat_tree, - hierarchical, - random, - star, + get_b_cube, + get_continuum_tiered, + get_fat_tree, + get_backbone, + get_caida, + get_gabriel, get_orion_cev, + get_sndlib, + get_topohub, + get_topology_zoo, + get_hierarchical, + get_mec_5g, + get_multi_region_wan, + get_random, + get_scale_free, + get_small_world, + get_star, ) - **Available infrastructure builders:** - - - :py:func:`~eclypse.builders.infrastructure.generators.b_cube` - - :py:func:`~eclypse.builders.infrastructure.generators.fat_tree` - - :py:func:`~eclypse.builders.infrastructure.generators.hierarchical` - - :py:func:`~eclypse.builders.infrastructure.generators.random` - - :py:func:`~eclypse.builders.infrastructure.generators.star` - - :py:func:`~eclypse.builders.infrastructure.orion_cev.get_orion_cev`: returns the ORION-CEV reference infrastructure + ECLYPSE includes several off-the-shelf infrastructure builders across + generic generators, architecture patterns, and named references. For the + full list, see :mod:`eclypse.builders.infrastructure`. + + .. list-table:: + :header-rows: 1 + + * - Category + - Builders + * - Generic generators + - ``get_star``, ``get_random``, ``get_hierarchical``, + ``get_fat_tree``, ``get_b_cube``, ``get_small_world``, + ``get_scale_free`` + * - Architecture patterns + - ``get_continuum_tiered``, ``get_mec_5g``, + ``get_multi_region_wan``, ``get_industrial_tsn``, + ``get_factory_cells``, ``get_vehicular_edge`` + * - References + - ``get_orion_cev``, ``get_topohub``, ``get_topology_zoo``, + ``get_sndlib``, ``get_backbone``, ``get_caida``, + ``get_gabriel`` **Example:** .. code-block:: python - from eclypse.builders.infrastructure import fat_tree + from eclypse.builders.infrastructure import get_fat_tree - infra = fat_tree(k=4) + infra = get_fat_tree(k=4) .. tab-item:: Application :sync: app - ECLYPSE currently includes a builder for the **SockShop** application - from the `Microservices Demo `_ - project, using + ECLYPSE includes several built-in application builders, all collected in + the :mod:`eclypse.builders.application` package. Sock Shop remains the + reference example used throughout this section, using :func:`~eclypse.builders.application.sock_shop.application.get_sock_shop`. + For simulation-only task DAGs, ECLYPSE also provides workflow builders in + :mod:`eclypse.builders.workflow`. These builders use WfCommons to + generate workflows and normalise file-size-derived ``storage`` and + dependency ``bandwidth`` values from bytes to MiB before assigning them + to the default ECLYPSE assets. + .. code-block:: python from eclypse.builders.application import get_sock_shop @@ -239,7 +267,8 @@ assets and flows. app = get_sock_shop() This application contains multiple interconnected services and - representative communication flows. + representative communication flows. For the full list of built-in + applications, see :mod:`eclypse.builders.application`. .. tip:: diff --git a/docs/source/overview/concepts/update-policy.rst b/docs/source/overview/concepts/update-policy.rst index 45aace3..a42cb90 100644 --- a/docs/source/overview/concepts/update-policy.rst +++ b/docs/source/overview/concepts/update-policy.rst @@ -30,14 +30,29 @@ ECLYPSE also provides a catalogue of off-the-shelf policies in :mod:`eclypse.policies`. The module groups reusable policies into a few common families: -- **failure**: availability flapping, node failures, and latency spikes -- **noise**: bounded random walks, momentum walks, and impulse shocks +- **failure**: node and edge failures, availability flapping, correlated + failures, partitions, brownouts, resource exhaustion, and latency spikes +- **noise**: bounded and momentum random walks, additive or multiplicative + jitter, Gaussian jitter, correlated noise, seasonal noise, dropout, and + impulse shocks - **distribution**: uniform, normal, lognormal, triangular, beta, gamma, - truncated-normal, and categorical multiplicative perturbations -- **degrade**: progressive increase or reduction of selected assets through - explicit ``increase()`` and ``reduce()`` policies -- **replay**: replay of node or edge values from records, dataframes, or parquet files -- **schedule**: wrappers such as ``every()``, ``after()``, ``between()``, and ``once_at()`` + truncated-normal, categorical, constant, Bernoulli, Poisson, exponential, + Weibull, Pareto, empirical, and weighted discrete multiplicative + perturbations +- **degrade**: progressive increase or reduction, direct assignment, scaling, + decay, clamping, restoring, and ramping of selected assets +- **replay**: replay of node, edge, graph, and event values from records, + dataframes, CSV files, or parquet files, with optional cyclic replay +- **schedule**: wrappers such as ``every()``, ``after()``, ``between()``, + ``once_at()``, ``at()``, ``until()``, ``repeat()``, ``with_probability()``, + ``jittered_every()``, and ``cooldown()`` +- **compose**: reusable policy composition with ``chain()``, ``all_of()``, + ``one_of()``, ``weighted_choice()``, and ``conditional()`` +- **workload**: arrival processes, traffic matrices, and diurnal load +- **topology**: graph mutation policies for adding, removing, rewiring, and + churn +- **constraints**: invariant-enforcing policies such as clamping, + normalisation, rounding, and capacity floors For most simulations, the easiest workflow is to compose a few built-in policies and only fall back to a custom callable when the behaviour is @@ -128,6 +143,21 @@ Scheduling wrappers let you activate a policy only during part of the run. edge_assets=["latency"], ), ), + policies.with_probability( + 0.2, + policies.failure.brownout( + factor=0.75, + node_assets=["cpu", "ram"], + ), + ), + policies.jittered_every( + 10, + policies.noise.additive_jitter( + edge_ranges={"latency": (-1.0, 2.0)}, + lower=0.0, + ), + jitter=2, + ), ] Replay Policies @@ -148,6 +178,59 @@ or synthetic measurements over time. time_column="time", value_columns=["user_count"], start_step=0, + cyclic=True, + ) + +.. code-block:: python + :caption: **Example:** Replay node and edge values together + + from eclypse import policies + + replay_trace = policies.replay.replay_graph( + node_records=[ + {"time": 0, "node_id": "edge-1", "users": 10}, + {"time": 1, "node_id": "edge-1", "users": 18}, + ], + edge_records=[ + {"time": 0, "source": "edge-1", "target": "cloud", "latency": 12}, + {"time": 1, "source": "edge-1", "target": "cloud", "latency": 20}, + ], + node_value_columns=["users"], + edge_value_columns=["latency"], + cyclic=True, + ) + +Composition, Workloads, Topology, and Constraints +------------------------------------------------- + +The higher-level families help keep scenario code small when multiple effects +must be combined. + +.. code-block:: python + :caption: **Example:** Compose workload, topology, and constraints + + from eclypse import policies + + update_policy = policies.compose.chain( + policies.workload.arrival_process( + rate=20, + node_assets="users", + node_filter=lambda node_id, data: data.get("tier") == "edge", + ), + policies.workload.traffic_matrix( + {("edge-1", "cloud"): 120.0}, + asset="traffic", + ), + policies.constraints.ensure_capacity_floor( + 1.0, + edge_assets="bandwidth", + ), + policies.topology.churn( + add_probability=0.1, + candidate_nodes={ + "burst-edge": {"cpu": 16, "ram": 32, "availability": 1.0}, + }, + ), ) Writing Custom Policies diff --git a/docs/source/overview/examples/echo.rst b/docs/source/overview/examples/echo.rst index 9067fa8..1b90b17 100644 --- a/docs/source/overview/examples/echo.rst +++ b/docs/source/overview/examples/echo.rst @@ -14,6 +14,12 @@ The full code lives in the `examples/echo `_ directory. +Run it from the repository root with: + +.. code-block:: bash + + poetry run echo + Application ----------- diff --git a/docs/source/overview/examples/grid_analysis.rst b/docs/source/overview/examples/grid_analysis.rst new file mode 100644 index 0000000..b084ae1 --- /dev/null +++ b/docs/source/overview/examples/grid_analysis.rst @@ -0,0 +1,45 @@ +Grid analysis +============= + +The grid analysis example runs a Ray Tune sweep over infrastructure topology, +load, failure policy, random seed, and placement strategy choices. + +Use it when you want to understand: + +- how to wrap an ECLYPSE simulation in a parameter-search function, +- how to compare placement strategies across many generated infrastructures, +- how to write custom infrastructure assets and policies for experiments. + +The full code lives in the +`examples/grid_analysis `_ +directory. + +Run it from the repository root with: + +.. code-block:: bash + + poetry run grid-analysis + +Simulation sweep +---------------- + +.. dropdown:: Main sweep code + + .. literalinclude:: ../../../../examples/grid_analysis/main.py + :language: python + +Infrastructure +-------------- + +.. dropdown:: Infrastructure code + + .. literalinclude:: ../../../../examples/grid_analysis/infrastructure.py + :language: python + +Placement strategy +------------------ + +.. dropdown:: Strategy code + + .. literalinclude:: ../../../../examples/grid_analysis/strategy.py + :language: python diff --git a/docs/source/overview/examples/image_prediction.rst b/docs/source/overview/examples/image_prediction.rst new file mode 100644 index 0000000..1163db8 --- /dev/null +++ b/docs/source/overview/examples/image_prediction.rst @@ -0,0 +1,46 @@ +Image prediction +================ + +The image prediction example shows a remote application with trainer, +predictor, and end-user services. It combines service implementations, custom +metrics, and a degradation policy in emulation mode. + +Use it when you want to understand: + +- how to attach concrete service implementations to an application, +- how to configure a remote simulation, +- how to collect custom metrics from remote service behaviour. + +The full code lives in the +`examples/image_prediction `_ +directory. + +Run it from the repository root with: + +.. code-block:: bash + + poetry run image-prediction + +Application +----------- + +.. dropdown:: Application code + + .. literalinclude:: ../../../../examples/image_prediction/application.py + :language: python + +Simulation +---------- + +.. dropdown:: Simulation code + + .. literalinclude:: ../../../../examples/image_prediction/main.py + :language: python + +Metrics +------- + +.. dropdown:: Metrics code + + .. literalinclude:: ../../../../examples/image_prediction/metrics.py + :language: python diff --git a/docs/source/overview/examples/index.rst b/docs/source/overview/examples/index.rst index 51e4b78..2e79227 100644 --- a/docs/source/overview/examples/index.rst +++ b/docs/source/overview/examples/index.rst @@ -7,8 +7,11 @@ Examples :hidden: echo + grid_analysis + image_prediction off_the_shelf sock_shop + user_distribution The examples section complements the getting-started guides with runnable scenarios you can inspect and adapt. @@ -48,3 +51,32 @@ subdirectory of the repository. A microservices application that simulates an online shop and is provided in both MPI and REST variants. + + .. grid-item:: + + .. card:: :octicon:`graph;1em;info` **Grid analysis** + :link-type: doc + :link: grid_analysis + + A Ray Tune sweep over infrastructure topologies, update policies, and + placement strategies. + + + .. grid-item:: + + .. card:: :octicon:`image;1em;info` **Image prediction** + :link-type: doc + :link: image_prediction + + A remote image-prediction pipeline with custom services, metrics, and + degradation policies. + + + .. grid-item:: + + .. card:: :octicon:`people;1em;info` **User distribution** + :link-type: doc + :link: user_distribution + + A custom metric and update-policy example driven by changing user + distributions. diff --git a/docs/source/overview/examples/off_the_shelf.rst b/docs/source/overview/examples/off_the_shelf.rst index 08f8ded..557b377 100644 --- a/docs/source/overview/examples/off_the_shelf.rst +++ b/docs/source/overview/examples/off_the_shelf.rst @@ -4,8 +4,8 @@ Off-the-shelf This example shows a complete local simulation built only from reusable ECLYPSE components: -- the :func:`~eclypse.builders.application.get_sock_shop` application builder -- the :func:`~eclypse.builders.infrastructure.hierarchical` infrastructure builder +- the :func:`~eclypse.builders.application.get_hotel_reservation` application builder +- an infrastructure builder from :mod:`eclypse.builders.infrastructure` - built-in update policies from :mod:`eclypse.policies` - a built-in placement strategy @@ -16,12 +16,21 @@ The full code lives in the `examples/off_the_shelf `_ directory. +Run it from the repository root with: + +.. code-block:: bash + + poetry run off-the-shelf + Application ----------- -The application is the standard Sock Shop graph created through the built-in -builder, with built-in uniform-distribution and degradation policies that -progressively make placement harder. +The application is the built-in hotel reservation graph, paired with built-in +uniform-distribution and degradation policies that progressively make +placement harder. + +ECLYPSE also provides several other ready-made application builders collected +in :mod:`eclypse.builders.application`. .. dropdown:: Application code @@ -34,9 +43,10 @@ Infrastructure The infrastructure is a generated hierarchical topology using the default assets and a built-in policy mix for flapping availability, uniform -perturbations, periodic latency spikes, and scheduled degradation. Together with -``BestFitStrategy``, this makes the example exercise repeated placement under a -changing environment. +perturbations, periodic latency spikes, and scheduled degradation. ECLYPSE also +provides several other off-the-shelf infrastructure builders collected in +:mod:`eclypse.builders.infrastructure`. Together with ``BestFitStrategy``, this +example exercises repeated placement under a changing environment. .. dropdown:: Infrastructure code diff --git a/docs/source/overview/examples/sock_shop.rst b/docs/source/overview/examples/sock_shop.rst index 28a4abc..24862c9 100644 --- a/docs/source/overview/examples/sock_shop.rst +++ b/docs/source/overview/examples/sock_shop.rst @@ -19,11 +19,18 @@ because it highlights the practical differences between MPI and REST communication in ECLYPSE. The full code lives in the -`examples/sock_shop/mpi `_ +`examples/sock_shop/mpi.py `_ and -`examples/sock_shop/rest `_ +`examples/sock_shop/rest.py `_ directories. +Run either variant from the repository root with: + +.. code-block:: bash + + poetry run sock-shop-mpi + poetry run sock-shop-rest + .. warning:: Both interfaces are asynchronous. When you call the low-level request APIs diff --git a/docs/source/overview/examples/user_distribution.rst b/docs/source/overview/examples/user_distribution.rst new file mode 100644 index 0000000..7045c64 --- /dev/null +++ b/docs/source/overview/examples/user_distribution.rst @@ -0,0 +1,46 @@ +User distribution +================= + +The user distribution example customises a generated infrastructure with a +``user_count`` asset and updates latency and placement conditions as the user +distribution evolves. + +Use it when you want to understand: + +- how to add a custom infrastructure asset, +- how to write metrics around domain-specific infrastructure state, +- how update policies can drive a longer-running simulation. + +The full code lives in the +`examples/user_distribution `_ +directory. + +Run it from the repository root with: + +.. code-block:: bash + + poetry run user-distribution + +Infrastructure +-------------- + +.. dropdown:: Infrastructure code + + .. literalinclude:: ../../../../examples/user_distribution/infrastructure.py + :language: python + +Metrics +------- + +.. dropdown:: Metrics code + + .. literalinclude:: ../../../../examples/user_distribution/metric.py + :language: python + +Simulation +---------- + +.. dropdown:: Simulation code + + .. literalinclude:: ../../../../examples/user_distribution/main.py + :language: python diff --git a/docs/source/overview/getting-started/minimal-local-run.rst b/docs/source/overview/getting-started/minimal-local-run.rst index 2935a46..908646b 100644 --- a/docs/source/overview/getting-started/minimal-local-run.rst +++ b/docs/source/overview/getting-started/minimal-local-run.rst @@ -19,22 +19,22 @@ Build the scenario The following example uses: -- :func:`~eclypse.builders.infrastructure.generators.hierarchical` to build a +- :func:`~eclypse.builders.infrastructure.get_hierarchical` to build a small infrastructure, - :func:`~eclypse.builders.application.sock_shop.application.get_sock_shop` to build a reference application, -- :class:`~eclypse.placement.strategies.random.RandomStrategy` for placement. +- :class:`~eclypse.placement.strategies.RandomStrategy` for placement. .. code-block:: python from eclypse.builders.application import get_sock_shop - from eclypse.builders.infrastructure import hierarchical + from eclypse.builders.infrastructure import get_hierarchical from eclypse.placement.strategies import RandomStrategy from eclypse.simulation import Simulation, SimulationConfig seed = 22 - infrastructure = hierarchical( + infrastructure = get_hierarchical( n=20, include_default_assets=True, seed=seed, @@ -49,10 +49,11 @@ The following example uses: seed=seed, max_steps=20, include_default_metrics=True, + default_strategy=RandomStrategy(seed=seed), ) simulation = Simulation(infrastructure, simulation_config=config) - simulation.register(application, RandomStrategy(seed=seed)) + simulation.register(application) Run the simulation ------------------ @@ -62,8 +63,7 @@ simulation and wait for it to finish: .. code-block:: python - simulation.start() - simulation.wait() + simulation.run() Inspect the report ------------------ diff --git a/docs/source/overview/getting-started/remote-emulation.rst b/docs/source/overview/getting-started/remote-emulation.rst index f622756..284adac 100644 --- a/docs/source/overview/getting-started/remote-emulation.rst +++ b/docs/source/overview/getting-started/remote-emulation.rst @@ -20,13 +20,13 @@ Minimal setup .. code-block:: python from eclypse.builders.application import get_sock_shop - from eclypse.builders.infrastructure import hierarchical + from eclypse.builders.infrastructure import get_hierarchical from eclypse.placement.strategies import RandomStrategy from eclypse.simulation import Simulation, SimulationConfig seed = 22 - infrastructure = hierarchical( + infrastructure = get_hierarchical( n=30, include_default_assets=True, seed=seed, @@ -44,12 +44,12 @@ Minimal setup max_steps=100, step_every_ms=500, include_default_metrics=True, + default_strategy=RandomStrategy(seed=seed), ) simulation = Simulation(infrastructure, simulation_config=config) - simulation.register(application, RandomStrategy(seed=seed)) - simulation.start() - simulation.wait() + simulation.register(application) + simulation.run() Choosing the communication interface ------------------------------------ diff --git a/eclypse/builders/__init__.py b/eclypse/builders/__init__.py index 9a97f25..4fa2167 100644 --- a/eclypse/builders/__init__.py +++ b/eclypse/builders/__init__.py @@ -1,6 +1,7 @@ -"""Package for the application and infrastructure builders.""" +"""Package for the application, infrastructure, and workflow builders.""" from . import application from . import infrastructure +from . import workflow -__all__ = ["application", "infrastructure"] +__all__ = ["application", "infrastructure", "workflow"] diff --git a/eclypse/builders/application/__init__.py b/eclypse/builders/application/__init__.py index 664e3c4..a900d67 100644 --- a/eclypse/builders/application/__init__.py +++ b/eclypse/builders/application/__init__.py @@ -1,5 +1,23 @@ """Application builders.""" +from .anomaly_detection.application import get_anomaly_detection +from .crud_api.application import get_crud_api +from .deathstarbench.hotel_reservation.application import get_hotel_reservation +from .deathstarbench.media_service.application import get_media_service +from .deathstarbench.social_network.application import get_social_network +from .keyword_spotting.application import get_keyword_spotting from .sock_shop.application import get_sock_shop +from .thumbnailer.application import get_thumbnailer +from .video_analytics_serving.application import get_video_analytics_serving -__all__ = ["get_sock_shop"] +__all__ = [ + "get_anomaly_detection", + "get_crud_api", + "get_hotel_reservation", + "get_keyword_spotting", + "get_media_service", + "get_social_network", + "get_sock_shop", + "get_thumbnailer", + "get_video_analytics_serving", +] diff --git a/eclypse/builders/application/_helpers.py b/eclypse/builders/application/_helpers.py new file mode 100644 index 0000000..ddfd682 --- /dev/null +++ b/eclypse/builders/application/_helpers.py @@ -0,0 +1,273 @@ +"""Helper functions shared by application builders.""" + +from __future__ import annotations + +from importlib import import_module +from typing import ( + TYPE_CHECKING, + Any, + Literal, +) + +from eclypse.builders._helpers import prune_assets +from eclypse.graph import Application +from eclypse.utils.defaults import SUPPORTED_COMMUNICATION_INTERFACES + +if TYPE_CHECKING: + from collections.abc import Callable + + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + CommunicationInterface, + InitPolicy, + UpdatePolicies, + ) + + AddFunction = Callable[..., Any] + IdentifierFactory = Callable[[str], Any] + EdgeRequirements = tuple[str, str, dict[str, Any]] + NodeRequirements = dict[str, dict[str, Any]] + + +def resolve_flows( + flows: Literal["default"] | list[list[str]], + default_flows: list[list[str]], +) -> list[list[str]]: + """Resolve the application flows passed to a builder. + + Args: + flows (Literal["default"] | list[list[str]]): + Application flows requested by the caller. Builders pass + ``"default"`` to select their bundled flow definitions. + default_flows (list[list[str]]): + Built-in flows exposed by the builder. + + Returns: + list[list[str]]: The flows to install on the application. + """ + if flows == "default": + return default_flows + return flows + + +def build_application( + application_id: str, + update_policies: UpdatePolicies, + node_assets: dict[str, Asset] | None, + edge_assets: dict[str, Asset] | None, + include_default_assets: bool, + requirement_init: InitPolicy, + flows: list[list[str]], + seed: int | None, +) -> Application: + """Create an application instance for a builder. + + Args: + application_id (str): Identifier assigned to the application. + update_policies (Callable | list[Callable] | None): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Optional assets attached to application nodes. + edge_assets (dict[str, Asset] | None): + Optional assets attached to application edges. + include_default_assets (bool): + Whether default graph assets should be included in the application. + requirement_init (InitPolicy): + Initialisation strategy applied to node and edge requirements. + flows (list[list[str]]): + Application flows to install on the graph. + seed (int | None): + Seed forwarded to the application random generator. + + Returns: + Application: The newly created application instance. + """ + return Application( + application_id=application_id, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=edge_assets, + include_default_assets=include_default_assets, + requirement_init=requirement_init, + flows=flows, + seed=seed, + ) + + +def resolve_builder_functions( + app: Application, + communication_interface: CommunicationInterface | None, + package_name: str, + service_names: list[str], + store_step: bool = False, +) -> tuple[AddFunction, IdentifierFactory]: + """Resolve the node-creation functions for an application builder. + + Args: + app (Application): Application being populated by the builder. + communication_interface (CommunicationInterface | None): + Communication backend requested by the caller. When ``None``, the + builder returns graph nodes instead of executable services. + package_name (str): Package that owns the service implementations. + service_names (list[str]): Service classes to resolve for the builder. + store_step (bool): Whether instantiated services should store their + step outputs in the internal step queue. + + Returns: + tuple[AddFunction, IdentifierFactory]: + A pair containing the application add function and an identifier + factory that returns either service ids or instantiated services. + + Raises: + ValueError: If ``communication_interface`` is not supported. + """ + if communication_interface is None: + return app.add_node, lambda service_id: service_id + + if communication_interface not in SUPPORTED_COMMUNICATION_INTERFACES: + raise ValueError( + f"Unknown communication interface: {communication_interface}", + ) + + services = import_module( + f".{communication_interface}_services", + package=package_name, + ) + classes = { + service_name: getattr(services, service_name) for service_name in service_names + } + return app.add_service, lambda service_id: classes[service_id]( + service_id, + store_step=store_step, + ) + + +def populate_application_topology( + app: Application, + add_fn: AddFunction, + id_fn: IdentifierFactory, + node_requirements: NodeRequirements, + edge_requirements: list[EdgeRequirements], +) -> None: + """Populate the nodes and edges of an application. + + Args: + app (Application): Application being populated. + add_fn (AddFunction): Function used to add nodes or services. + id_fn (IdentifierFactory): Factory returning the object to add for each + service identifier. + node_requirements (dict[str, dict[str, Any]]): + Resource and QoS requirements keyed by service name. + edge_requirements (list[EdgeRequirements]): + Communication requirements keyed by source-target service pairs. + Each item contains source id, target id, and edge attributes. + """ + for service_id, requirements in node_requirements.items(): + add_fn( + id_fn(service_id), + **prune_assets(app.node_assets, **requirements), + ) + + for source, target, requirements in edge_requirements: + edge_data = dict(requirements) + symmetric = edge_data.pop("symmetric", True) + app.add_edge( + source, + target, + symmetric=symmetric, + **prune_assets(app.edge_assets, **edge_data), + ) + + +def build_application_from_specs( + application_id: str, + communication_interface: CommunicationInterface | None, + update_policies: UpdatePolicies, + node_assets: dict[str, Asset] | None, + edge_assets: dict[str, Asset] | None, + include_default_assets: bool, + requirement_init: InitPolicy, + flows: Literal["default"] | list[list[str]], + default_flows: list[list[str]], + service_names: list[str], + node_requirements: NodeRequirements, + edge_requirements: list[EdgeRequirements], + seed: int | None, + package_name: str, + store_step: bool = False, +) -> Application: + """Build and populate an application from declarative builder specs. + + Args: + application_id (str): Identifier assigned to the generated application. + communication_interface (CommunicationInterface | None): + Communication backend used to instantiate executable services. When + ``None``, the builder returns a graph-only application. + update_policies (Callable | list[Callable] | None): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Optional assets attached to application nodes. + edge_assets (dict[str, Asset] | None): + Optional assets attached to application edges. + include_default_assets (bool): + Whether default graph assets should be included in the application. + requirement_init (InitPolicy): + Initialisation strategy applied to node and edge requirements. + flows (Literal["default"] | list[list[str]]): + Application flows requested by the caller. + default_flows (list[list[str]]): + Built-in flows exposed by the builder. + service_names (list[str]): + Service classes to resolve for the builder. + node_requirements (NodeRequirements): + Resource and QoS requirements keyed by service name. + edge_requirements (list[EdgeRequirements]): + Communication requirements keyed by source-target service pairs. + seed (int | None): + Seed forwarded to the application random generator. + package_name (str): Package that owns the service implementations. + store_step (bool): + Whether instantiated services should store their step outputs in + the internal step queue. Ignored when ``communication_interface`` + is ``None``. + + Returns: + Application: The configured application instance. + + Raises: + ValueError: If ``communication_interface`` is not supported. + """ + app = build_application( + application_id=application_id, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=edge_assets, + include_default_assets=include_default_assets, + requirement_init=requirement_init, + flows=resolve_flows(flows, default_flows), + seed=seed, + ) + add_fn, id_fn = resolve_builder_functions( + app=app, + communication_interface=communication_interface, + package_name=package_name, + service_names=service_names, + store_step=store_step, + ) + populate_application_topology( + app=app, + add_fn=add_fn, + id_fn=id_fn, + node_requirements=node_requirements, + edge_requirements=edge_requirements, + ) + return app + + +__all__ = [ + "build_application", + "build_application_from_specs", + "populate_application_topology", + "resolve_builder_functions", + "resolve_flows", +] diff --git a/eclypse/builders/application/anomaly_detection/__init__.py b/eclypse/builders/application/anomaly_detection/__init__.py new file mode 100644 index 0000000..0bb1b8a --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/__init__.py @@ -0,0 +1,10 @@ +"""The anomaly detection application models an edge monitoring pipeline. + +Telemetry is sampled, features are extracted, an anomaly score is computed, +and alerts are raised. It is inspired by the anomaly detection workload from +MLPerf Tiny. + +Source: + `MLPerf Tiny Inference Benchmark + `_ +""" diff --git a/eclypse/builders/application/anomaly_detection/application.py b/eclypse/builders/application/anomaly_detection/application.py new file mode 100644 index 0000000..e55d9a2 --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/application.py @@ -0,0 +1,161 @@ +"""Factory for an anomaly detection application.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Literal, +) + +from eclypse.builders.application._helpers import ( + build_application_from_specs, +) + +if TYPE_CHECKING: + from eclypse.graph import Application + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + CommunicationInterface, + InitPolicy, + UpdatePolicies, + ) + + +def get_anomaly_detection( + application_id: str = "AnomalyDetection", + communication_interface: CommunicationInterface | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + edge_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + requirement_init: InitPolicy = "min", + flows: Literal["default"] | list[list[str]] = "default", + store_step: bool = False, + seed: int | None = None, +) -> Application: + """Get the anomaly detection application. + + Args: + application_id (str): Identifier assigned to the generated application. + communication_interface (CommunicationInterface | None): + Communication backend used to instantiate executable services. When + ``None``, the builder returns a graph-only application. + update_policies (Callable | list[Callable] | None): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Optional assets attached to application nodes. + edge_assets (dict[str, Asset] | None): + Optional assets attached to application edges. + include_default_assets (bool): + Whether default graph assets should be included in the application. + requirement_init (InitPolicy): + Initialisation strategy applied to node and edge requirements. + flows (Literal["default"] | list[list[str]]): + User-defined application flows. Use ``"default"`` to install the + benchmark's built-in telemetry-processing flow. + store_step (bool): + Whether instantiated services should store their step outputs in + the internal step queue. Ignored when + ``communication_interface`` is ``None``. + seed (int | None): + Seed forwarded to the application random generator. + + Returns: + Application: The configured anomaly detection application. + + Raises: + ValueError: If ``communication_interface`` is not supported. + """ + default_flows = [ + [ + "SensorService", + "FeatureService", + "InferenceService", + "AlertService", + "SensorService", + ] + ] + service_names = [ + "AlertService", + "FeatureService", + "InferenceService", + "SensorService", + ] + node_requirements = { + "SensorService": { + "cpu": 0.5, + "gpu": 0, + "ram": 0.25, + "storage": 0.1, + "availability": 0.98, + "processing_time": 2, + }, + "FeatureService": { + "cpu": 1, + "gpu": 0, + "ram": 0.5, + "storage": 0.25, + "availability": 0.97, + "processing_time": 4, + }, + "InferenceService": { + "cpu": 2, + "gpu": 0.5, + "ram": 1.0, + "storage": 0.5, + "availability": 0.95, + "processing_time": 6, + }, + "AlertService": { + "cpu": 0.5, + "gpu": 0, + "ram": 0.25, + "storage": 0.1, + "availability": 0.99, + "processing_time": 2, + }, + } + edge_requirements = [ + ( + "SensorService", + "FeatureService", + {"symmetric": True, "latency": 5, "bandwidth": 5}, + ), + ( + "FeatureService", + "InferenceService", + {"symmetric": True, "latency": 8, "bandwidth": 5}, + ), + ( + "SensorService", + "InferenceService", + {"symmetric": True, "latency": 7, "bandwidth": 4}, + ), + ( + "InferenceService", + "AlertService", + {"symmetric": True, "latency": 5, "bandwidth": 3}, + ), + ( + "AlertService", + "SensorService", + {"symmetric": True, "latency": 4, "bandwidth": 2}, + ), + ] + return build_application_from_specs( + application_id=application_id, + communication_interface=communication_interface, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=edge_assets, + include_default_assets=include_default_assets, + requirement_init=requirement_init, + flows=flows, + store_step=store_step, + default_flows=default_flows, + service_names=service_names, + node_requirements=node_requirements, + edge_requirements=edge_requirements, + seed=seed, + package_name=__package__, + ) diff --git a/eclypse/builders/application/anomaly_detection/mpi_services/__init__.py b/eclypse/builders/application/anomaly_detection/mpi_services/__init__.py new file mode 100644 index 0000000..354eaa8 --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/mpi_services/__init__.py @@ -0,0 +1,13 @@ +"""MPI implementation for anomaly detection services.""" + +from .alert import AlertService +from .feature import FeatureService +from .inference import InferenceService +from .sensor import SensorService + +__all__ = [ + "AlertService", + "FeatureService", + "InferenceService", + "SensorService", +] diff --git a/eclypse/builders/application/anomaly_detection/mpi_services/alert.py b/eclypse/builders/application/anomaly_detection/mpi_services/alert.py new file mode 100644 index 0000000..86081f2 --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/mpi_services/alert.py @@ -0,0 +1,26 @@ +"""MPI workflow for anomaly alerting.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + +ANOMALY_THRESHOLD = 2.5 + + +class AlertService(Service): + """Turn anomaly scores into responses.""" + + async def step(self): + """Handle the next inference result emitted by the pipeline.""" + await self.inference_request() + + @mpi.exchange(receive=True, send=True) + def inference_request(self, _sender_id, body): + """Map the anomaly score to a status and respond to the sensor.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "SensorService", { + "response_type": "anomaly_response", + "window_id": body["window_id"], + "score": body["score"], + "status": "alert" if body["score"] >= ANOMALY_THRESHOLD else "normal", + } diff --git a/eclypse/builders/application/anomaly_detection/mpi_services/feature.py b/eclypse/builders/application/anomaly_detection/mpi_services/feature.py new file mode 100644 index 0000000..7bf92a8 --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/mpi_services/feature.py @@ -0,0 +1,25 @@ +"""MPI workflow for feature extraction.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class FeatureService(Service): + """Extract simple features from telemetry.""" + + async def step(self): + """Handle the next telemetry window produced by the sensor.""" + await self.sensor_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def sensor_request(self, _sender_id, body): + """Compute compact statistics for the received telemetry samples.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + max_sample = max(body["samples"]) + mean_sample = sum(body["samples"]) / len(body["samples"]) + return "InferenceService", { + "request_type": "score_window", + "window_id": body["window_id"], + "features": {"max": max_sample, "mean": mean_sample}, + } diff --git a/eclypse/builders/application/anomaly_detection/mpi_services/inference.py b/eclypse/builders/application/anomaly_detection/mpi_services/inference.py new file mode 100644 index 0000000..7833a33 --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/mpi_services/inference.py @@ -0,0 +1,24 @@ +"""MPI workflow for anomaly inference.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class InferenceService(Service): + """Compute a simple anomaly score.""" + + async def step(self): + """Handle the next feature payload emitted by the extractor.""" + await self.feature_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def feature_request(self, _sender_id, body): + """Estimate an anomaly score from the extracted features.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + score = body["features"]["max"] / max(body["features"]["mean"], 0.1) + return "AlertService", { + "request_type": "emit_alert", + "window_id": body["window_id"], + "score": round(score, 2), + } diff --git a/eclypse/builders/application/anomaly_detection/mpi_services/sensor.py b/eclypse/builders/application/anomaly_detection/mpi_services/sensor.py new file mode 100644 index 0000000..648ae3c --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/mpi_services/sensor.py @@ -0,0 +1,31 @@ +"""MPI workflow for telemetry capture.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class SensorService(Service): + """Generate telemetry windows and start the anomaly pipeline.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the sensor with a rolling telemetry window counter.""" + super().__init__(service_id, store_step=store_step) + self.window_id = 0 + + async def step(self): + """Create the next telemetry window and wait for the alert result.""" + self.window_id += 1 + await self.capture_window() + response = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=response)) + return response + + @mpi.exchange(send=True) + def capture_window(self): + """Send a synthetic telemetry window to the feature extractor.""" + return "FeatureService", { + "request_type": "extract_features", + "window_id": self.window_id, + "samples": [0.8, 1.2, 4.5], + } diff --git a/eclypse/builders/application/anomaly_detection/rest_services/__init__.py b/eclypse/builders/application/anomaly_detection/rest_services/__init__.py new file mode 100644 index 0000000..e29c2c3 --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/rest_services/__init__.py @@ -0,0 +1,13 @@ +"""REST implementation for anomaly detection services.""" + +from .alert import AlertService +from .feature import FeatureService +from .inference import InferenceService +from .sensor import SensorService + +__all__ = [ + "AlertService", + "FeatureService", + "InferenceService", + "SensorService", +] diff --git a/eclypse/builders/application/anomaly_detection/rest_services/alert.py b/eclypse/builders/application/anomaly_detection/rest_services/alert.py new file mode 100644 index 0000000..18bc421 --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/rest_services/alert.py @@ -0,0 +1,23 @@ +"""REST endpoints for anomaly alerting.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + +ANOMALY_THRESHOLD = 2.5 + + +class AlertService(RESTService): + """Turn anomaly scores into responses.""" + + @rest.endpoint("/alert", "POST") + def alert(self, window_id: int, score: float, **_): + """Translate the anomaly score into a compact alert response.""" + self.logger.info( + "Received request | " + format_log_kv(window_id=window_id, score=score) + ) + return 200, { + "window_id": window_id, + "score": score, + "status": "alert" if score >= ANOMALY_THRESHOLD else "normal", + } diff --git a/eclypse/builders/application/anomaly_detection/rest_services/feature.py b/eclypse/builders/application/anomaly_detection/rest_services/feature.py new file mode 100644 index 0000000..f64edcb --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/rest_services/feature.py @@ -0,0 +1,22 @@ +"""REST endpoints for feature extraction.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class FeatureService(RESTService): + """Extract simple features from telemetry.""" + + @rest.endpoint("/features", "POST") + def features(self, window_id: int, samples: list[float], **_): + """Compute compact statistics for a telemetry window.""" + self.logger.info( + "Received request | " + format_log_kv(window_id=window_id, samples=samples) + ) + max_sample = max(samples) + mean_sample = sum(samples) / len(samples) + return 200, { + "window_id": window_id, + "features": {"max": max_sample, "mean": mean_sample}, + } diff --git a/eclypse/builders/application/anomaly_detection/rest_services/inference.py b/eclypse/builders/application/anomaly_detection/rest_services/inference.py new file mode 100644 index 0000000..738785f --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/rest_services/inference.py @@ -0,0 +1,19 @@ +"""REST endpoints for anomaly inference.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class InferenceService(RESTService): + """Compute a simple anomaly score.""" + + @rest.endpoint("/score", "POST") + def score(self, window_id: int, features: dict, **_): + """Estimate an anomaly score from extracted telemetry features.""" + self.logger.info( + "Received request | " + + format_log_kv(window_id=window_id, features=features) + ) + score = features["max"] / max(features["mean"], 0.1) + return 200, {"window_id": window_id, "score": round(score, 2)} diff --git a/eclypse/builders/application/anomaly_detection/rest_services/sensor.py b/eclypse/builders/application/anomaly_detection/rest_services/sensor.py new file mode 100644 index 0000000..86ad142 --- /dev/null +++ b/eclypse/builders/application/anomaly_detection/rest_services/sensor.py @@ -0,0 +1,49 @@ +"""REST workflow for telemetry capture.""" + +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class SensorService(Service): + """Generate telemetry windows and start the anomaly pipeline.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the sensor with a rolling telemetry window counter.""" + super().__init__( + service_id, + communication_interface="rest", + store_step=store_step, + ) + self.window_id = 0 + + async def step(self): + """Drive one telemetry window through the REST inference pipeline.""" + self.window_id += 1 + feature_r = await self.rest.post( + "FeatureService/features", + window_id=self.window_id, + samples=[0.8, 1.2, 4.5], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="FeatureService", body=feature_r.body) + ) + inference_r = await self.rest.post( + "InferenceService/score", + window_id=self.window_id, + features=feature_r.body["features"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="InferenceService", body=inference_r.body) + ) + alert_r = await self.rest.post( + "AlertService/alert", + window_id=self.window_id, + score=inference_r.body["score"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="AlertService", body=alert_r.body) + ) + return alert_r diff --git a/eclypse/builders/application/crud_api/__init__.py b/eclypse/builders/application/crud_api/__init__.py new file mode 100644 index 0000000..6c9b41a --- /dev/null +++ b/eclypse/builders/application/crud_api/__init__.py @@ -0,0 +1,10 @@ +"""The CRUD API application models a small data-serving application. + +It includes request validation, entity management, and audit recording. The +structure is inspired by the CRUD API workload described in the SeBS +serverless benchmark suite. + +Source: + `SeBS benchmark applications + `_ +""" diff --git a/eclypse/builders/application/crud_api/application.py b/eclypse/builders/application/crud_api/application.py new file mode 100644 index 0000000..c0952d7 --- /dev/null +++ b/eclypse/builders/application/crud_api/application.py @@ -0,0 +1,151 @@ +"""Factory for a CRUD API application.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Literal, +) + +from eclypse.builders.application._helpers import ( + build_application_from_specs, +) + +if TYPE_CHECKING: + from eclypse.graph import Application + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + CommunicationInterface, + InitPolicy, + UpdatePolicies, + ) + + +def get_crud_api( + application_id: str = "CRUDAPI", + communication_interface: CommunicationInterface | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + edge_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + requirement_init: InitPolicy = "min", + flows: Literal["default"] | list[list[str]] = "default", + store_step: bool = False, + seed: int | None = None, +) -> Application: + """Get the CRUD API application. + + Args: + application_id (str): Identifier assigned to the generated application. + communication_interface (CommunicationInterface | None): + Communication backend used to instantiate executable services. When + ``None``, the builder returns a graph-only application. + update_policies (Callable | list[Callable] | None): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Optional assets attached to application nodes. + edge_assets (dict[str, Asset] | None): + Optional assets attached to application edges. + include_default_assets (bool): + Whether default graph assets should be included in the application. + requirement_init (InitPolicy): + Initialisation strategy applied to node and edge requirements. + flows (Literal["default"] | list[list[str]]): + User-defined application flows. Use ``"default"`` to install the + benchmark's built-in CRUD request paths. + store_step (bool): + Whether instantiated services should store their step outputs in + the internal step queue. Ignored when + ``communication_interface`` is ``None``. + seed (int | None): + Seed forwarded to the application random generator. + + Returns: + Application: The configured CRUD API application. + + Raises: + ValueError: If ``communication_interface`` is not supported. + """ + default_flows = [ + ["GatewayService", "AuthService", "GatewayService"], + ["GatewayService", "ItemService", "AuditService", "GatewayService"], + ] + service_names = [ + "AuditService", + "AuthService", + "GatewayService", + "ItemService", + ] + node_requirements = { + "GatewayService": { + "cpu": 1, + "gpu": 0, + "ram": 0.75, + "storage": 0.25, + "availability": 0.97, + "processing_time": 6, + }, + "AuthService": { + "cpu": 1, + "gpu": 0, + "ram": 0.5, + "storage": 0.25, + "availability": 0.98, + "processing_time": 5, + }, + "ItemService": { + "cpu": 2, + "gpu": 0, + "ram": 1.5, + "storage": 1.0, + "availability": 0.95, + "processing_time": 10, + }, + "AuditService": { + "cpu": 1, + "gpu": 0, + "ram": 0.5, + "storage": 0.5, + "availability": 0.96, + "processing_time": 4, + }, + } + edge_requirements = [ + ( + "GatewayService", + "AuthService", + {"symmetric": True, "latency": 12, "bandwidth": 8}, + ), + ( + "GatewayService", + "ItemService", + {"symmetric": True, "latency": 15, "bandwidth": 10}, + ), + ( + "ItemService", + "AuditService", + {"symmetric": True, "latency": 8, "bandwidth": 5}, + ), + ( + "AuditService", + "GatewayService", + {"symmetric": True, "latency": 6, "bandwidth": 4}, + ), + ] + return build_application_from_specs( + application_id=application_id, + communication_interface=communication_interface, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=edge_assets, + include_default_assets=include_default_assets, + requirement_init=requirement_init, + flows=flows, + store_step=store_step, + default_flows=default_flows, + service_names=service_names, + node_requirements=node_requirements, + edge_requirements=edge_requirements, + seed=seed, + package_name=__package__, + ) diff --git a/eclypse/builders/application/crud_api/mpi_services/__init__.py b/eclypse/builders/application/crud_api/mpi_services/__init__.py new file mode 100644 index 0000000..4f879ae --- /dev/null +++ b/eclypse/builders/application/crud_api/mpi_services/__init__.py @@ -0,0 +1,13 @@ +"""MPI implementation for CRUD API services.""" + +from .audit import AuditService +from .auth import AuthService +from .gateway import GatewayService +from .item import ItemService + +__all__ = [ + "AuditService", + "AuthService", + "GatewayService", + "ItemService", +] diff --git a/eclypse/builders/application/crud_api/mpi_services/audit.py b/eclypse/builders/application/crud_api/mpi_services/audit.py new file mode 100644 index 0000000..4a864f2 --- /dev/null +++ b/eclypse/builders/application/crud_api/mpi_services/audit.py @@ -0,0 +1,23 @@ +"""MPI workflow for auditing.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class AuditService(Service): + """Record a simple audit event.""" + + async def step(self): + """Handle the next item event emitted by the CRUD pipeline.""" + await self.item_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def item_request(self, sender_id, body): + """Record an audit message and respond to the calling service.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return sender_id, { + "response_type": "audit_response", + "status": "recorded", + "message": f"{body['action']}:{body['item_id']}", + } diff --git a/eclypse/builders/application/crud_api/mpi_services/auth.py b/eclypse/builders/application/crud_api/mpi_services/auth.py new file mode 100644 index 0000000..ba1748e --- /dev/null +++ b/eclypse/builders/application/crud_api/mpi_services/auth.py @@ -0,0 +1,23 @@ +"""MPI workflow for authentication.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class AuthService(Service): + """Validate an API key.""" + + async def step(self): + """Handle the next authentication request from the gateway.""" + await self.gateway_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def gateway_request(self, sender_id, body): + """Authorise the request and return a synthetic token.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return sender_id, { + "response_type": "auth_response", + "token": f"token:{body['api_key']}", + "status": "authorized", + } diff --git a/eclypse/builders/application/crud_api/mpi_services/gateway.py b/eclypse/builders/application/crud_api/mpi_services/gateway.py new file mode 100644 index 0000000..994bad2 --- /dev/null +++ b/eclypse/builders/application/crud_api/mpi_services/gateway.py @@ -0,0 +1,36 @@ +"""MPI workflow for the CRUD gateway.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class GatewayService(Service): + """Drive a create-and-list workflow.""" + + async def step(self): + """Authenticate the client and create a demo item.""" + await self.auth_request() + auth_response = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=auth_response)) + await self.item_request(auth_response["token"]) + item_response = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=item_response)) + return item_response + + @mpi.exchange(send=True) + def auth_request(self): + """Send a synthetic authentication request to the auth service.""" + return "AuthService", { + "request_type": "authenticate", + "api_key": "demo-key", + } + + @mpi.exchange(send=True) + def item_request(self, token: str): + """Submit a create-item request with the authorised token.""" + return "ItemService", { + "request_type": "create_item", + "token": token, + "item": {"id": "item-1", "name": "demo", "status": "active"}, + } diff --git a/eclypse/builders/application/crud_api/mpi_services/item.py b/eclypse/builders/application/crud_api/mpi_services/item.py new file mode 100644 index 0000000..e747ab4 --- /dev/null +++ b/eclypse/builders/application/crud_api/mpi_services/item.py @@ -0,0 +1,41 @@ +"""MPI workflow for item management.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class ItemService(Service): + """Create an item and emit an audit event.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the item store used by the CRUD workflow.""" + super().__init__(service_id, store_step=store_step) + self.items: dict[str, dict[str, str]] = {} + + async def step(self): + """Create the item, then wait for the audit confirmation.""" + await self.gateway_request() + return await self.audit_request() + + @mpi.exchange(receive=True, send=True) + def gateway_request(self, _sender_id, body): + """Store the item and forward an audit event.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + item = body["item"] + self.items[item["id"]] = item + return "AuditService", { + "request_type": "record_event", + "item_id": item["id"], + "action": "create", + } + + @mpi.exchange(receive=True, send=True) + def audit_request(self, _sender_id, body): + """Return the updated item list after the audit succeeds.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "GatewayService", { + "response_type": "crud_response", + "status": body["status"], + "items": list(self.items.values()), + } diff --git a/eclypse/builders/application/crud_api/rest_services/__init__.py b/eclypse/builders/application/crud_api/rest_services/__init__.py new file mode 100644 index 0000000..8886444 --- /dev/null +++ b/eclypse/builders/application/crud_api/rest_services/__init__.py @@ -0,0 +1,13 @@ +"""REST implementation for CRUD API services.""" + +from .audit import AuditService +from .auth import AuthService +from .gateway import GatewayService +from .item import ItemService + +__all__ = [ + "AuditService", + "AuthService", + "GatewayService", + "ItemService", +] diff --git a/eclypse/builders/application/crud_api/rest_services/audit.py b/eclypse/builders/application/crud_api/rest_services/audit.py new file mode 100644 index 0000000..66c732c --- /dev/null +++ b/eclypse/builders/application/crud_api/rest_services/audit.py @@ -0,0 +1,21 @@ +"""REST endpoints for auditing.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class AuditService(RESTService): + """Record a simple audit event.""" + + @rest.endpoint("/events", "POST") + def record_event(self, token: str, item_id: str, action: str, **_): + """Record an audit event for the authenticated item operation.""" + self.logger.info( + "Received request | " + + format_log_kv(token=token, item_id=item_id, action=action) + ) + return 200, { + "status": "recorded", + "message": f"{token}:{action}:{item_id}", + } diff --git a/eclypse/builders/application/crud_api/rest_services/auth.py b/eclypse/builders/application/crud_api/rest_services/auth.py new file mode 100644 index 0000000..ceff4a6 --- /dev/null +++ b/eclypse/builders/application/crud_api/rest_services/auth.py @@ -0,0 +1,15 @@ +"""REST endpoints for authentication.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class AuthService(RESTService): + """Validate an API key.""" + + @rest.endpoint("/auth", "POST") + def auth(self, api_key: str, **_): + """Authorise the request and return a synthetic token.""" + self.logger.info("Received request | " + format_log_kv(api_key=api_key)) + return 200, {"token": f"token:{api_key}", "status": "authorized"} diff --git a/eclypse/builders/application/crud_api/rest_services/gateway.py b/eclypse/builders/application/crud_api/rest_services/gateway.py new file mode 100644 index 0000000..bbc62fe --- /dev/null +++ b/eclypse/builders/application/crud_api/rest_services/gateway.py @@ -0,0 +1,34 @@ +"""REST workflow for the CRUD gateway.""" + +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class GatewayService(Service): + """Drive a create-and-list workflow.""" + + async def step(self): + """Authenticate the client and create a demo item via REST.""" + auth_r = await self.rest.post("AuthService/auth", api_key="demo-key") + self.logger.info( + "Received response | " + + format_log_kv(source="AuthService", body=auth_r.body) + ) + item_r = await self.rest.post( + "ItemService/items", + token=auth_r.body["token"], + item={"id": "item-1", "name": "demo", "status": "active"}, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="ItemService", body=item_r.body) + ) + return item_r + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the gateway with REST communication enabled.""" + super().__init__( + service_id, + communication_interface="rest", + store_step=store_step, + ) diff --git a/eclypse/builders/application/crud_api/rest_services/item.py b/eclypse/builders/application/crud_api/rest_services/item.py new file mode 100644 index 0000000..d208d93 --- /dev/null +++ b/eclypse/builders/application/crud_api/rest_services/item.py @@ -0,0 +1,35 @@ +"""REST endpoints for item management.""" + +from eclypse.remote.communication import rest +from eclypse.remote.communication.rest import HTTPStatusCode +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class ItemService(RESTService): + """Create an item and emit an audit event.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the item store used by the CRUD workflow.""" + super().__init__(service_id, store_step=store_step) + self.items: dict[str, dict[str, str]] = {} + + @rest.endpoint("/items", "POST") + async def create_item(self, token: str, item: dict, **_): + """Store the item, emit an audit event, and return all items.""" + self.logger.info("Received request | " + format_log_kv(token=token, item=item)) + self.items[item["id"]] = item + audit_r = await self.rest.post( + "AuditService/events", + token=token, + item_id=item["id"], + action="create", + ) + self.logger.info( + "Received response | " + + format_log_kv(source="AuditService", body=audit_r.body) + ) + return HTTPStatusCode.CREATED, { + "status": audit_r.body["status"], + "items": list(self.items.values()), + } diff --git a/eclypse/builders/application/deathstarbench/__init__.py b/eclypse/builders/application/deathstarbench/__init__.py new file mode 100644 index 0000000..3061e1e --- /dev/null +++ b/eclypse/builders/application/deathstarbench/__init__.py @@ -0,0 +1,21 @@ +"""DeathStarBench application builders (e.g. hotel reservation, social network). + +This package groups the ECLYPSE builders inspired by the released +DeathStarBench applications. It currently includes hotel reservation, +social-network posting, and movie-review workflows modelled after the +original benchmark suite. + +Source: + `DeathStarBench repository + `_ +""" + +from .hotel_reservation.application import get_hotel_reservation +from .media_service.application import get_media_service +from .social_network.application import get_social_network + +__all__ = [ + "get_hotel_reservation", + "get_media_service", + "get_social_network", +] diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/__init__.py b/eclypse/builders/application/deathstarbench/hotel_reservation/__init__.py new file mode 100644 index 0000000..82520d1 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/__init__.py @@ -0,0 +1,10 @@ +"""The hotel reservation application models a hotel reservation workflow. + +It includes hotel search, profile retrieval, reservation coordination, and +payment confirmation. It is inspired by the hotel reservation application +from the DeathStarBench benchmark suite. + +Source: + `DeathStarBench repository + `_ +""" diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/application.py b/eclypse/builders/application/deathstarbench/hotel_reservation/application.py new file mode 100644 index 0000000..9c4c200 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/application.py @@ -0,0 +1,167 @@ +"""Factory for a hotel reservation microservice application.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Literal, +) + +from eclypse.builders.application._helpers import ( + build_application_from_specs, +) + +if TYPE_CHECKING: + from eclypse.graph import Application + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + CommunicationInterface, + InitPolicy, + UpdatePolicies, + ) + + +def get_hotel_reservation( + application_id: str = "HotelReservation", + communication_interface: CommunicationInterface | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + edge_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + requirement_init: InitPolicy = "min", + flows: Literal["default"] | list[list[str]] = "default", + store_step: bool = False, + seed: int | None = None, +) -> Application: + """Get the hotel reservation application. + + Args: + application_id (str): Identifier assigned to the generated application. + communication_interface (CommunicationInterface | None): + Communication backend used to instantiate executable services. When + ``None``, the builder returns a graph-only application. + update_policies (Callable | list[Callable] | None): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Optional assets attached to application nodes. + edge_assets (dict[str, Asset] | None): + Optional assets attached to application edges. + include_default_assets (bool): + Whether default graph assets should be included in the application. + requirement_init (InitPolicy): + Initialisation strategy applied to node and edge requirements. + flows (Literal["default"] | list[list[str]]): + User-defined application flows. Use ``"default"`` to install the + benchmark's built-in booking flows. + store_step (bool): + Whether instantiated services should store their step outputs in + the internal step queue. Ignored when + ``communication_interface`` is ``None``. + seed (int | None): + Seed forwarded to the application random generator. + + Returns: + Application: The configured hotel reservation application. + + Raises: + ValueError: If ``communication_interface`` is not supported. + """ + default_flows = [ + ["FrontendService", "SearchService", "FrontendService"], + ["FrontendService", "ProfileService", "FrontendService"], + [ + "FrontendService", + "ReservationService", + "PaymentService", + "ReservationService", + "FrontendService", + ], + ] + service_names = [ + "FrontendService", + "PaymentService", + "ProfileService", + "ReservationService", + "SearchService", + ] + node_requirements = { + "FrontendService": { + "cpu": 2, + "gpu": 0, + "ram": 1.25, + "storage": 0.25, + "availability": 0.95, + "processing_time": 14, + }, + "SearchService": { + "cpu": 2, + "gpu": 0, + "ram": 1.75, + "storage": 0.75, + "availability": 0.94, + "processing_time": 13, + }, + "ProfileService": { + "cpu": 2, + "gpu": 0, + "ram": 1.5, + "storage": 1.0, + "availability": 0.94, + "processing_time": 12, + }, + "ReservationService": { + "cpu": 3, + "gpu": 0, + "ram": 2.5, + "storage": 1.5, + "availability": 0.92, + "processing_time": 18, + }, + "PaymentService": { + "cpu": 1, + "gpu": 0, + "ram": 1.0, + "storage": 0.25, + "availability": 0.95, + "processing_time": 10, + }, + } + edge_requirements = [ + ( + "FrontendService", + "SearchService", + {"symmetric": True, "latency": 18, "bandwidth": 10}, + ), + ( + "FrontendService", + "ProfileService", + {"symmetric": True, "latency": 16, "bandwidth": 10}, + ), + ( + "FrontendService", + "ReservationService", + {"symmetric": True, "latency": 20, "bandwidth": 12}, + ), + ( + "ReservationService", + "PaymentService", + {"symmetric": True, "latency": 14, "bandwidth": 8}, + ), + ] + return build_application_from_specs( + application_id=application_id, + communication_interface=communication_interface, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=edge_assets, + include_default_assets=include_default_assets, + requirement_init=requirement_init, + flows=flows, + store_step=store_step, + default_flows=default_flows, + service_names=service_names, + node_requirements=node_requirements, + edge_requirements=edge_requirements, + seed=seed, + package_name=__package__, + ) diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/__init__.py b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/__init__.py new file mode 100644 index 0000000..4983adb --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/__init__.py @@ -0,0 +1,15 @@ +"""MPI implementation for hotel reservation services.""" + +from .frontend import FrontendService +from .payment import PaymentService +from .profile import ProfileService +from .reservation import ReservationService +from .search import SearchService + +__all__ = [ + "FrontendService", + "PaymentService", + "ProfileService", + "ReservationService", + "SearchService", +] diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/frontend.py b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/frontend.py new file mode 100644 index 0000000..ecc0c5e --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/frontend.py @@ -0,0 +1,53 @@ +"""MPI workflow for the hotel reservation frontend.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class FrontendService(Service): + """Drive a complete hotel reservation flow.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the frontend with a default test user.""" + super().__init__(service_id, store_step=store_step) + self.user_id = 101 + + async def step(self): + """Search for hotels, fetch the profile, and submit a reservation.""" + await self.search_request() + hotels = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=hotels)) + await self.profile_request() + profile = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=profile)) + await self.reservation_request(hotels["hotels"], profile["user"]) + reservation = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=reservation)) + return reservation + + @mpi.exchange(send=True) + def search_request(self): + """Send a hotel search request for the demo travel plan.""" + return "SearchService", { + "request_type": "search_hotels", + "city": "Pisa", + "nights": 2, + } + + @mpi.exchange(send=True) + def profile_request(self): + """Request the traveller profile for the active user.""" + return "ProfileService", { + "request_type": "get_profile", + "user_id": self.user_id, + } + + @mpi.exchange(send=True) + def reservation_request(self, hotels: list[dict], user: dict): + """Reserve the first available hotel for the requested user.""" + return "ReservationService", { + "request_type": "create_reservation", + "hotel": hotels[0], + "user": user, + } diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/payment.py b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/payment.py new file mode 100644 index 0000000..64947a5 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/payment.py @@ -0,0 +1,26 @@ +"""MPI workflow for hotel payment.""" + +import random as rnd + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class PaymentService(Service): + """Charge a payment method for a reservation.""" + + async def step(self): + """Handle the next payment request emitted by the reservation flow.""" + await self.reservation_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def reservation_request(self, sender_id, body): + """Charge the reservation and return a synthetic transaction id.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return sender_id, { + "response_type": "payment_response", + "reservation_id": body["reservation_id"], + "transaction_id": f"txn-{rnd.randint(1000, 9999)}", + "status": "confirmed", + } diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/profile.py b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/profile.py new file mode 100644 index 0000000..76782e9 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/profile.py @@ -0,0 +1,26 @@ +"""MPI workflow for user profile retrieval.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class ProfileService(Service): + """Return a booking profile for the current user.""" + + async def step(self): + """Handle the next profile request from the frontend.""" + await self.frontend_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def frontend_request(self, sender_id, body): + """Return a compact traveller profile for the requested user.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return sender_id, { + "response_type": "profile_response", + "user": { + "user_id": body["user_id"], + "name": "Ada Lovelace", + "loyalty_level": "gold", + }, + } diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/reservation.py b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/reservation.py new file mode 100644 index 0000000..44b8924 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/reservation.py @@ -0,0 +1,46 @@ +"""MPI workflow for reservation orchestration.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class ReservationService(Service): + """Reserve a hotel room and coordinate payment.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the reservation orchestrator state.""" + super().__init__(service_id, store_step=store_step) + self.pending_reservation: dict[str, object] = {} + + async def step(self): + """Create a reservation and wait for the payment response.""" + await self.frontend_request() + return await self.payment_request() + + @mpi.exchange(receive=True, send=True) + def frontend_request(self, _sender_id, body): + """Store the reservation context and trigger payment.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + self.pending_reservation = { + "hotel": body["hotel"], + "user": body["user"], + "reservation_id": "rsv-2001", + } + return "PaymentService", { + "request_type": "charge_card", + "reservation_id": "rsv-2001", + "amount": body["hotel"]["price"], + } + + @mpi.exchange(receive=True, send=True) + def payment_request(self, _sender_id, body): + """Return the completed reservation once payment succeeds.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "FrontendService", { + "response_type": "reservation_response", + "reservation_id": self.pending_reservation["reservation_id"], + "hotel_name": self.pending_reservation["hotel"]["name"], + "transaction_id": body["transaction_id"], + "status": body["status"], + } diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/search.py b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/search.py new file mode 100644 index 0000000..52f1d45 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/mpi_services/search.py @@ -0,0 +1,26 @@ +"""MPI workflow for hotel search.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class SearchService(Service): + """Return a compact set of available hotels.""" + + async def step(self): + """Handle the next hotel search request from the frontend.""" + await self.frontend_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def frontend_request(self, sender_id, body): + """Return a curated list of hotels for the requested city.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return sender_id, { + "response_type": "search_results", + "city": body["city"], + "hotels": [ + {"id": "h1", "name": "Arno View", "price": 129.0}, + {"id": "h2", "name": "Tower Stay", "price": 149.0}, + ], + } diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/__init__.py b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/__init__.py new file mode 100644 index 0000000..3c75e5d --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/__init__.py @@ -0,0 +1,15 @@ +"""REST implementation for hotel reservation services.""" + +from .frontend import FrontendService +from .payment import PaymentService +from .profile import ProfileService +from .reservation import ReservationService +from .search import SearchService + +__all__ = [ + "FrontendService", + "PaymentService", + "ProfileService", + "ReservationService", + "SearchService", +] diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/frontend.py b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/frontend.py new file mode 100644 index 0000000..e133d0e --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/frontend.py @@ -0,0 +1,40 @@ +"""REST workflow for the hotel reservation frontend.""" + +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class FrontendService(Service): + """Drive a complete hotel reservation flow.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the frontend with a default test user.""" + super().__init__( + service_id, + communication_interface="rest", + store_step=store_step, + ) + self.user_id = 101 + + async def step(self): + """Search for hotels, fetch the profile, and submit a reservation.""" + hotels_r = await self.rest.get("SearchService/search", city="Pisa", nights=2) + self.logger.info( + "Received response | " + + format_log_kv(source="SearchService", body=hotels_r.body) + ) + profile_r = await self.rest.get("ProfileService/profile", user_id=self.user_id) + self.logger.info( + "Received response | " + + format_log_kv(source="ProfileService", body=profile_r.body) + ) + reservation_r = await self.rest.post( + "ReservationService/reserve", + hotel=hotels_r.body["hotels"][0], + user=profile_r.body["user"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="ReservationService", body=reservation_r.body) + ) + return reservation_r diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/payment.py b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/payment.py new file mode 100644 index 0000000..d6d6abd --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/payment.py @@ -0,0 +1,25 @@ +"""REST endpoints for hotel payment.""" + +import random as rnd + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class PaymentService(RESTService): + """Charge a payment method for a reservation.""" + + @rest.endpoint("/pay", "POST") + def pay(self, reservation_id: str, amount: float, **_): + """Charge the reservation and return a synthetic transaction id.""" + self.logger.info( + "Received request | " + + format_log_kv(reservation_id=reservation_id, amount=amount) + ) + return 200, { + "reservation_id": reservation_id, + "amount": amount, + "transaction_id": f"txn-{rnd.randint(1000, 9999)}", + "status": "confirmed", + } diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/profile.py b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/profile.py new file mode 100644 index 0000000..a51bba1 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/profile.py @@ -0,0 +1,21 @@ +"""REST endpoints for user profile retrieval.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class ProfileService(RESTService): + """Return a booking profile for the current user.""" + + @rest.endpoint("/profile", "GET") + def profile(self, user_id: int, **_): + """Return a compact traveller profile for the requested user.""" + self.logger.info("Received request | " + format_log_kv(user_id=user_id)) + return 200, { + "user": { + "user_id": user_id, + "name": "Ada Lovelace", + "loyalty_level": "gold", + } + } diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/reservation.py b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/reservation.py new file mode 100644 index 0000000..6f8e6fd --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/reservation.py @@ -0,0 +1,36 @@ +"""REST endpoints for reservation orchestration.""" + +from eclypse.remote.communication import rest +from eclypse.remote.communication.rest import HTTPStatusCode +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class ReservationService(RESTService): + """Reserve a hotel room and coordinate payment.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the reservation service with a stable booking id.""" + super().__init__(service_id, store_step=store_step) + self.reservation_id = "rsv-2001" + + @rest.endpoint("/reserve", "POST") + async def reserve(self, hotel: dict, user: dict, **_): + """Create a reservation and charge the selected hotel stay.""" + self.logger.info("Received request | " + format_log_kv(hotel=hotel, user=user)) + payment_r = await self.rest.post( + "PaymentService/pay", + reservation_id=self.reservation_id, + amount=hotel["price"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="PaymentService", body=payment_r.body) + ) + return HTTPStatusCode.CREATED, { + "reservation_id": self.reservation_id, + "hotel_name": hotel["name"], + "guest_name": user["name"], + "transaction_id": payment_r.body["transaction_id"], + "status": payment_r.body["status"], + } diff --git a/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/search.py b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/search.py new file mode 100644 index 0000000..225a928 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/hotel_reservation/rest_services/search.py @@ -0,0 +1,24 @@ +"""REST endpoints for hotel search.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class SearchService(RESTService): + """Return a compact set of available hotels.""" + + @rest.endpoint("/search", "GET") + def search(self, city: str, nights: int, **_): + """Return a curated list of hotels for the requested city.""" + self.logger.info( + "Received request | " + format_log_kv(city=city, nights=nights) + ) + return 200, { + "city": city, + "nights": nights, + "hotels": [ + {"id": "h1", "name": "Arno View", "price": 129.0}, + {"id": "h2", "name": "Tower Stay", "price": 149.0}, + ], + } diff --git a/eclypse/builders/application/deathstarbench/media_service/__init__.py b/eclypse/builders/application/deathstarbench/media_service/__init__.py new file mode 100644 index 0000000..0f14633 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/__init__.py @@ -0,0 +1,10 @@ +"""The media service application models a movie-review composition workflow. + +It includes review composition, movie lookup, rating and text enrichment, +review storage, review indexing, and movie-information reads. It is inspired +by the media microservices application from the DeathStarBench benchmark suite. + +Source: + `DeathStarBench repository + `_ +""" diff --git a/eclypse/builders/application/deathstarbench/media_service/application.py b/eclypse/builders/application/deathstarbench/media_service/application.py new file mode 100644 index 0000000..a34acca --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/application.py @@ -0,0 +1,275 @@ +"""Factory for a media service microservice application.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Literal, +) + +from eclypse.builders.application._helpers import build_application_from_specs + +if TYPE_CHECKING: + from eclypse.graph import Application + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + CommunicationInterface, + InitPolicy, + UpdatePolicies, + ) + + +def get_media_service( + application_id: str = "MediaService", + communication_interface: CommunicationInterface | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + edge_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + requirement_init: InitPolicy = "min", + flows: Literal["default"] | list[list[str]] = "default", + store_step: bool = False, + seed: int | None = None, +) -> Application: + """Get the media service application. + + Args: + application_id (str): Identifier assigned to the generated application. + communication_interface (CommunicationInterface | None): + Communication backend used to instantiate executable services. When + ``None``, the builder returns a graph-only application. + update_policies (Callable | list[Callable] | None): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Optional assets attached to application nodes. + edge_assets (dict[str, Asset] | None): + Optional assets attached to application edges. + include_default_assets (bool): + Whether default graph assets should be included in the application. + requirement_init (InitPolicy): + Initialisation strategy applied to node and edge requirements. + flows (Literal["default"] | list[list[str]]): + User-defined application flows. Use ``"default"`` to install the + benchmark's built-in review-composition and movie-information + flows. + store_step (bool): + Whether instantiated services should store their step outputs in + the internal step queue. Ignored when + ``communication_interface`` is ``None``. + seed (int | None): + Seed forwarded to the application random generator. + + Returns: + Application: The configured media service application. + + Raises: + ValueError: If ``communication_interface`` is not supported. + """ + default_flows = [ + [ + "ComposeReviewService", + "UniqueIdService", + "MovieIdService", + "TextService", + "RatingService", + "UserService", + "ReviewStorageService", + "UserReviewService", + "MovieReviewService", + "ComposeReviewService", + ], + ["MovieInfoService", "CastInfoService", "MovieInfoService"], + ["MovieInfoService", "PlotService", "MovieInfoService"], + ["MovieInfoService", "MovieReviewService", "MovieInfoService"], + ] + service_names = [ + "ComposeReviewService", + "UniqueIdService", + "MovieIdService", + "TextService", + "RatingService", + "UserService", + "ReviewStorageService", + "UserReviewService", + "MovieReviewService", + "MovieInfoService", + "CastInfoService", + "PlotService", + ] + node_requirements = { + "ComposeReviewService": { + "cpu": 2, + "gpu": 0, + "ram": 1.75, + "storage": 0.5, + "availability": 0.94, + "processing_time": 16, + }, + "UniqueIdService": { + "cpu": 1, + "gpu": 0, + "ram": 0.25, + "storage": 0.25, + "availability": 0.98, + "processing_time": 4, + }, + "MovieIdService": { + "cpu": 1, + "gpu": 0, + "ram": 1.0, + "storage": 0.75, + "availability": 0.94, + "processing_time": 8, + }, + "TextService": { + "cpu": 1, + "gpu": 0, + "ram": 0.75, + "storage": 0.25, + "availability": 0.96, + "processing_time": 6, + }, + "RatingService": { + "cpu": 1, + "gpu": 0, + "ram": 0.25, + "storage": 0.25, + "availability": 0.98, + "processing_time": 4, + }, + "UserService": { + "cpu": 1, + "gpu": 0, + "ram": 1.25, + "storage": 1.0, + "availability": 0.94, + "processing_time": 9, + }, + "ReviewStorageService": { + "cpu": 3, + "gpu": 0, + "ram": 2.5, + "storage": 3.0, + "availability": 0.91, + "processing_time": 13, + }, + "UserReviewService": { + "cpu": 2, + "gpu": 0, + "ram": 1.5, + "storage": 1.5, + "availability": 0.93, + "processing_time": 10, + }, + "MovieReviewService": { + "cpu": 2, + "gpu": 0, + "ram": 1.75, + "storage": 1.75, + "availability": 0.92, + "processing_time": 11, + }, + "MovieInfoService": { + "cpu": 2, + "gpu": 0, + "ram": 1.75, + "storage": 1.5, + "availability": 0.93, + "processing_time": 13, + }, + "CastInfoService": { + "cpu": 1, + "gpu": 0, + "ram": 1.0, + "storage": 1.0, + "availability": 0.94, + "processing_time": 8, + }, + "PlotService": { + "cpu": 1, + "gpu": 0, + "ram": 1.0, + "storage": 1.0, + "availability": 0.94, + "processing_time": 8, + }, + } + edge_requirements = [ + ( + "ComposeReviewService", + "UniqueIdService", + {"symmetric": True, "latency": 12, "bandwidth": 8}, + ), + ( + "UniqueIdService", + "MovieIdService", + {"symmetric": True, "latency": 10, "bandwidth": 8}, + ), + ( + "MovieIdService", + "TextService", + {"symmetric": True, "latency": 10, "bandwidth": 8}, + ), + ( + "TextService", + "RatingService", + {"symmetric": True, "latency": 10, "bandwidth": 6}, + ), + ( + "RatingService", + "UserService", + {"symmetric": True, "latency": 10, "bandwidth": 8}, + ), + ( + "UserService", + "ReviewStorageService", + {"symmetric": True, "latency": 14, "bandwidth": 14}, + ), + ( + "ReviewStorageService", + "UserReviewService", + {"symmetric": True, "latency": 14, "bandwidth": 14}, + ), + ( + "UserReviewService", + "MovieReviewService", + {"symmetric": True, "latency": 14, "bandwidth": 12}, + ), + ( + "MovieReviewService", + "ComposeReviewService", + {"symmetric": True, "latency": 12, "bandwidth": 10}, + ), + ( + "MovieInfoService", + "CastInfoService", + {"symmetric": True, "latency": 12, "bandwidth": 8}, + ), + ( + "MovieInfoService", + "PlotService", + {"symmetric": True, "latency": 12, "bandwidth": 8}, + ), + ( + "MovieInfoService", + "MovieReviewService", + {"symmetric": True, "latency": 14, "bandwidth": 12}, + ), + ] + return build_application_from_specs( + application_id=application_id, + communication_interface=communication_interface, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=edge_assets, + include_default_assets=include_default_assets, + requirement_init=requirement_init, + flows=flows, + store_step=store_step, + default_flows=default_flows, + service_names=service_names, + node_requirements=node_requirements, + edge_requirements=edge_requirements, + seed=seed, + package_name=__package__, + ) diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/__init__.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/__init__.py new file mode 100644 index 0000000..957edf6 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/__init__.py @@ -0,0 +1,29 @@ +"""MPI implementation for media service services.""" + +from .cast_info import CastInfoService +from .compose_review import ComposeReviewService +from .movie_id import MovieIdService +from .movie_info import MovieInfoService +from .movie_review import MovieReviewService +from .plot import PlotService +from .rating import RatingService +from .review_storage import ReviewStorageService +from .text import TextService +from .unique_id import UniqueIdService +from .user import UserService +from .user_review import UserReviewService + +__all__ = [ + "CastInfoService", + "ComposeReviewService", + "MovieIdService", + "MovieInfoService", + "MovieReviewService", + "PlotService", + "RatingService", + "ReviewStorageService", + "TextService", + "UniqueIdService", + "UserReviewService", + "UserService", +] diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/cast_info.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/cast_info.py new file mode 100644 index 0000000..917fd23 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/cast_info.py @@ -0,0 +1,26 @@ +"""MPI workflow for movie cast metadata.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class CastInfoService(Service): + """Return cast metadata for a movie.""" + + async def step(self): + """Handle the next cast-info request.""" + await self.handle_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def handle_request(self, sender_id, body): + """Return a small cast list for the requested movie.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + casts = { + "m1": ["Keanu Reeves", "Carrie-Anne Moss"], + "m2": ["Amy Adams", "Jeremy Renner"], + } + return sender_id, { + "response_type": "cast_response", + "cast": casts.get(body["movie_id"], []), + } diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/compose_review.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/compose_review.py new file mode 100644 index 0000000..50f201b --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/compose_review.py @@ -0,0 +1,38 @@ +"""MPI workflow for movie-review composition.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class ComposeReviewService(Service): + """Drive one media-service compose-review workflow.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the compose-review workflow state.""" + super().__init__(service_id, store_step=store_step) + self.req_id = 0 + self.user_id = 101 + self.username = "ada" + + async def step(self): + """Start the compose-review workflow and await the final response.""" + self.req_id += 1 + await self.submit_review() + response = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=response)) + return response + + @mpi.exchange(send=True) + def submit_review(self): + """Send a compose-review request into the media-service pipeline.""" + return "UniqueIdService", { + "request_type": "compose_review", + "reply_to": self.id, + "req_id": self.req_id, + "user_id": self.user_id, + "username": self.username, + "movie_title": "The Matrix", + "rating": 5, + "text": "A sharp and timeless science-fiction classic.", + } diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/movie_id.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/movie_id.py new file mode 100644 index 0000000..ef277d0 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/movie_id.py @@ -0,0 +1,36 @@ +"""MPI workflow for movie identifier resolution.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class MovieIdService(Service): + """Resolve movie identifiers for review requests.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the movie lookup fixture data.""" + super().__init__(service_id, store_step=store_step) + self.movies = { + "The Matrix": {"movie_id": "m1", "title": "The Matrix"}, + "Arrival": {"movie_id": "m2", "title": "Arrival"}, + } + + async def step(self): + """Handle the next movie-id request.""" + await self.handle_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def handle_request(self, sender_id, body): + """Resolve the movie id for compose or lookup requests.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + movie = self.movies[body["movie_title"]] + if body["request_type"] == "lookup_movie": + return sender_id, { + "response_type": "lookup_movie_response", + **movie, + } + return "TextService", { + **body, + "movie_id": movie["movie_id"], + } diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/movie_info.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/movie_info.py new file mode 100644 index 0000000..1c089c3 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/movie_info.py @@ -0,0 +1,59 @@ +"""MPI workflow for aggregated movie information.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class MovieInfoService(Service): + """Aggregate movie metadata and reviews.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the pending movie info request state.""" + super().__init__(service_id, store_step=store_step) + self.pending_request: dict[str, object] = {} + + async def step(self): + """Start a movie-info request and aggregate all downstream replies.""" + await self.request_cast() # pylint: disable=no-value-for-parameter + cast = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=cast)) + await self.request_plot() + plot = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=plot)) + await self.request_reviews() + reviews = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=reviews)) + return { + "movie_id": self.pending_request["movie_id"], + "movie_title": self.pending_request["movie_title"], + "cast": cast["cast"], + "plot": plot["plot"], + "reviews": reviews["reviews"], + } + + @mpi.exchange(receive=True, send=True) + def request_cast(self, _sender_id, body): + """Store the movie-info request and ask for cast metadata.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + self.pending_request = body + return "CastInfoService", { + "request_type": "get_cast", + "movie_id": body["movie_id"], + } + + @mpi.exchange(send=True) + def request_plot(self): + """Request plot metadata for the pending movie.""" + return "PlotService", { + "request_type": "get_plot", + "movie_id": self.pending_request["movie_id"], + } + + @mpi.exchange(send=True) + def request_reviews(self): + """Request reviews for the pending movie.""" + return "MovieReviewService", { + "request_type": "read_movie_reviews", + "movie_id": self.pending_request["movie_id"], + } diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/movie_review.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/movie_review.py new file mode 100644 index 0000000..0fd8bc9 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/movie_review.py @@ -0,0 +1,38 @@ +"""MPI workflow for per-movie review indexing.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class MovieReviewService(Service): + """Index reviews by movie and serve review lookups.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the movie review index.""" + super().__init__(service_id, store_step=store_step) + self.by_movie: dict[str, list[dict[str, object]]] = {} + + async def step(self): + """Handle the next movie-review request.""" + await self.handle_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def handle_request(self, sender_id, body): + """Index reviews by movie or return stored movie reviews.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + if body["request_type"] == "read_movie_reviews": + return sender_id, { + "response_type": "read_movie_reviews_response", + "reviews": self.by_movie.get(body["movie_id"], []), + } + + self.by_movie.setdefault(body["review"]["movie_id"], []).append(body["review"]) + return body["reply_to"], { + "response_type": "compose_review_response", + "review_id": body["review"]["review_id"], + "movie_id": body["review"]["movie_id"], + "movie_title": body["review"]["movie_title"], + "status": "stored", + "review_count": len(self.by_movie[body["review"]["movie_id"]]), + } diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/plot.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/plot.py new file mode 100644 index 0000000..fee496a --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/plot.py @@ -0,0 +1,26 @@ +"""MPI workflow for movie plot metadata.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class PlotService(Service): + """Return plot metadata for a movie.""" + + async def step(self): + """Handle the next plot-info request.""" + await self.handle_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def handle_request(self, sender_id, body): + """Return a short plot summary for the requested movie.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + plots = { + "m1": "A hacker discovers the world is a simulation.", + "m2": "A linguist learns to communicate with alien visitors.", + } + return sender_id, { + "response_type": "plot_response", + "plot": plots.get(body["movie_id"], ""), + } diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/rating.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/rating.py new file mode 100644 index 0000000..af9de39 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/rating.py @@ -0,0 +1,22 @@ +"""MPI workflow for rating validation.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class RatingService(Service): + """Validate review ratings before review storage.""" + + async def step(self): + """Handle the next rating-validation request.""" + await self.compose_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def compose_request(self, _sender_id, body): + """Validate the rating and forward the request to user lookup.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "UserService", { + **body, + "rating": max(1, min(5, body["rating"])), + } diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/review_storage.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/review_storage.py new file mode 100644 index 0000000..b0fb6ed --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/review_storage.py @@ -0,0 +1,47 @@ +"""MPI workflow for review persistence.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class ReviewStorageService(Service): + """Persist reviews and provide review lookups.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the in-memory review store.""" + super().__init__(service_id, store_step=store_step) + self.reviews: dict[int, dict[str, object]] = {} + + async def step(self): + """Handle the next review-storage request.""" + await self.handle_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def handle_request(self, sender_id, body): + """Store reviews or return a batch of stored reviews.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + if body["request_type"] == "read_reviews": + return sender_id, { + "response_type": "read_reviews_response", + "reviews": [ + self.reviews[review_id] + for review_id in body["review_ids"] + if review_id in self.reviews + ], + } + + review = { + "review_id": body["review_id"], + "movie_id": body["movie_id"], + "movie_title": body["movie_title"], + "rating": body["rating"], + "text": body["text"], + "user": body["user"], + } + self.reviews[body["review_id"]] = review + return "UserReviewService", { + **body, + "request_type": "write_user_review", + "review": review, + } diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/text.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/text.py new file mode 100644 index 0000000..2c36dc6 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/text.py @@ -0,0 +1,22 @@ +"""MPI workflow for review text processing.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class TextService(Service): + """Normalise review text before storing the review.""" + + async def step(self): + """Handle the next text-processing request.""" + await self.compose_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def compose_request(self, _sender_id, body): + """Normalise review text and forward the request to ratings.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "RatingService", { + **body, + "text": body["text"].strip(), + } diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/unique_id.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/unique_id.py new file mode 100644 index 0000000..613acfa --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/unique_id.py @@ -0,0 +1,28 @@ +"""MPI workflow for review identifier generation.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class UniqueIdService(Service): + """Assign review identifiers for compose-review requests.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the review identifier counter.""" + super().__init__(service_id, store_step=store_step) + self.next_review_id = 7000 + + async def step(self): + """Handle the next compose-review request from the workflow.""" + await self.compose_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def compose_request(self, _sender_id, body): + """Assign a review id and forward the request to movie lookup.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + self.next_review_id += 1 + return "MovieIdService", { + **body, + "review_id": self.next_review_id, + } diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/user.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/user.py new file mode 100644 index 0000000..65ed7b5 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/user.py @@ -0,0 +1,25 @@ +"""MPI workflow for media-service user data.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class UserService(Service): + """Resolve user identities for review requests.""" + + async def step(self): + """Handle the next user-resolution request.""" + await self.compose_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def compose_request(self, _sender_id, body): + """Attach user metadata and forward the request to storage.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "ReviewStorageService", { + **body, + "user": { + "user_id": body["user_id"], + "username": body["username"], + }, + } diff --git a/eclypse/builders/application/deathstarbench/media_service/mpi_services/user_review.py b/eclypse/builders/application/deathstarbench/media_service/mpi_services/user_review.py new file mode 100644 index 0000000..6678c12 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/mpi_services/user_review.py @@ -0,0 +1,30 @@ +"""MPI workflow for per-user review indexing.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class UserReviewService(Service): + """Index reviews by author.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the user review index.""" + super().__init__(service_id, store_step=store_step) + self.by_user: dict[int, list[int]] = {} + + async def step(self): + """Handle the next user-review request.""" + await self.handle_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def handle_request(self, _sender_id, body): + """Index the review by user and forward it to the movie review index.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + self.by_user.setdefault(body["review"]["user"]["user_id"], []).append( + body["review"]["review_id"], + ) + return "MovieReviewService", { + **body, + "request_type": "write_movie_review", + } diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/__init__.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/__init__.py new file mode 100644 index 0000000..d6c13db --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/__init__.py @@ -0,0 +1,29 @@ +"""REST implementation for media service services.""" + +from .cast_info import CastInfoService +from .compose_review import ComposeReviewService +from .movie_id import MovieIdService +from .movie_info import MovieInfoService +from .movie_review import MovieReviewService +from .plot import PlotService +from .rating import RatingService +from .review_storage import ReviewStorageService +from .text import TextService +from .unique_id import UniqueIdService +from .user import UserService +from .user_review import UserReviewService + +__all__ = [ + "CastInfoService", + "ComposeReviewService", + "MovieIdService", + "MovieInfoService", + "MovieReviewService", + "PlotService", + "RatingService", + "ReviewStorageService", + "TextService", + "UniqueIdService", + "UserReviewService", + "UserService", +] diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/cast_info.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/cast_info.py new file mode 100644 index 0000000..a0a3bf6 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/cast_info.py @@ -0,0 +1,19 @@ +"""REST endpoints for movie cast metadata.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class CastInfoService(RESTService): + """Return cast metadata for a movie.""" + + @rest.endpoint("/cast", "GET") + def cast(self, movie_id: str, **_): + """Return a small cast list for the requested movie.""" + self.logger.info("Received request | " + format_log_kv(movie_id=movie_id)) + casts = { + "m1": ["Keanu Reeves", "Carrie-Anne Moss"], + "m2": ["Amy Adams", "Jeremy Renner"], + } + return 200, {"cast": casts.get(movie_id, [])} diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/compose_review.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/compose_review.py new file mode 100644 index 0000000..212e067 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/compose_review.py @@ -0,0 +1,38 @@ +"""REST workflow for movie-review composition.""" + +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class ComposeReviewService(Service): + """Drive one media-service compose-review workflow.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the compose-review workflow state.""" + super().__init__( + service_id, + communication_interface="rest", + store_step=store_step, + ) + self.req_id = 0 + self.user_id = 101 + self.username = "ada" + + async def step(self): + """Compose a review and trigger the downstream review pipeline.""" + self.req_id += 1 + response = await self.rest.post( + "UniqueIdService/compose", + req_id=self.req_id, + reply_to=self.id, + user_id=self.user_id, + username=self.username, + movie_title="The Matrix", + rating=5, + text="A sharp and timeless science-fiction classic.", + ) + self.logger.info( + "Received response | " + + format_log_kv(source="UniqueIdService", body=response.body) + ) + return response diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/movie_id.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/movie_id.py new file mode 100644 index 0000000..d922f08 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/movie_id.py @@ -0,0 +1,40 @@ +"""REST endpoints for movie identifier resolution.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class MovieIdService(RESTService): + """Resolve movie identifiers for review requests.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the movie lookup fixture data.""" + super().__init__(service_id, store_step=store_step) + self.movies = { + "The Matrix": {"movie_id": "m1", "title": "The Matrix"}, + "Arrival": {"movie_id": "m2", "title": "Arrival"}, + } + + @rest.endpoint("/compose", "POST") + async def compose(self, movie_title: str, **payload): + """Resolve the movie id and forward the request to text parsing.""" + self.logger.info("Received request | " + format_log_kv(movie_title=movie_title)) + movie = self.movies[movie_title] + response = await self.rest.post( + "TextService/compose", + **payload, + movie_title=movie_title, + movie_id=movie["movie_id"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="TextService", body=response.body) + ) + return 200, response.body + + @rest.endpoint("/lookup", "GET") + def lookup(self, movie_title: str, **_): + """Return the movie descriptor for the requested title.""" + self.logger.info("Received request | " + format_log_kv(movie_title=movie_title)) + return 200, self.movies[movie_title] diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/movie_info.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/movie_info.py new file mode 100644 index 0000000..138c1e0 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/movie_info.py @@ -0,0 +1,38 @@ +"""REST endpoints for aggregated movie information.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class MovieInfoService(RESTService): + """Aggregate movie metadata and reviews.""" + + @rest.endpoint("/details", "GET") + async def details(self, movie_id: str, movie_title: str, **_): + """Return a combined view of cast, plot, and stored reviews.""" + self.logger.info( + "Received request | " + + format_log_kv(movie_id=movie_id, movie_title=movie_title) + ) + cast = await self.rest.get("CastInfoService/cast", movie_id=movie_id) + self.logger.info( + "Received response | " + + format_log_kv(source="CastInfoService", body=cast.body) + ) + plot = await self.rest.get("PlotService/plot", movie_id=movie_id) + self.logger.info( + "Received response | " + format_log_kv(source="PlotService", body=plot.body) + ) + reviews = await self.rest.get("MovieReviewService/read", movie_id=movie_id) + self.logger.info( + "Received response | " + + format_log_kv(source="MovieReviewService", body=reviews.body) + ) + return 200, { + "movie_id": movie_id, + "movie_title": movie_title, + "cast": cast.body["cast"], + "plot": plot.body["plot"], + "reviews": reviews.body["reviews"], + } diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/movie_review.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/movie_review.py new file mode 100644 index 0000000..72688f2 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/movie_review.py @@ -0,0 +1,38 @@ +"""REST endpoints for per-movie review indexing.""" + +from eclypse.remote.communication import rest +from eclypse.remote.communication.rest import HTTPStatusCode +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class MovieReviewService(RESTService): + """Index reviews by movie and serve review lookups.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the movie review index.""" + super().__init__(service_id, store_step=store_step) + self.by_movie: dict[str, list[dict[str, object]]] = {} + + @rest.endpoint("/write", "POST") + def write(self, review: dict, reply_to: str, **_): + """Index the review by movie and return the compose-review result.""" + self.logger.info( + "Received request | " + + format_log_kv(review_id=review["review_id"], movie_id=review["movie_id"]) + ) + self.by_movie.setdefault(review["movie_id"], []).append(review) + return HTTPStatusCode.CREATED, { + "reply_to": reply_to, + "review_id": review["review_id"], + "movie_id": review["movie_id"], + "movie_title": review["movie_title"], + "status": "stored", + "review_count": len(self.by_movie[review["movie_id"]]), + } + + @rest.endpoint("/read", "GET") + def read(self, movie_id: str, **_): + """Return the indexed reviews for a movie.""" + self.logger.info("Received request | " + format_log_kv(movie_id=movie_id)) + return 200, {"reviews": self.by_movie.get(movie_id, [])} diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/plot.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/plot.py new file mode 100644 index 0000000..3162cc2 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/plot.py @@ -0,0 +1,19 @@ +"""REST endpoints for movie plot metadata.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class PlotService(RESTService): + """Return plot metadata for a movie.""" + + @rest.endpoint("/plot", "GET") + def plot(self, movie_id: str, **_): + """Return a short plot summary for the requested movie.""" + self.logger.info("Received request | " + format_log_kv(movie_id=movie_id)) + plots = { + "m1": "A hacker discovers the world is a simulation.", + "m2": "A linguist learns to communicate with alien visitors.", + } + return 200, {"plot": plots.get(movie_id, "")} diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/rating.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/rating.py new file mode 100644 index 0000000..3277db4 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/rating.py @@ -0,0 +1,24 @@ +"""REST endpoints for rating validation.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class RatingService(RESTService): + """Validate review ratings before review storage.""" + + @rest.endpoint("/compose", "POST") + async def compose(self, rating: int, **payload): + """Validate the rating and forward the request to user lookup.""" + self.logger.info("Received request | " + format_log_kv(rating=rating)) + response = await self.rest.post( + "UserService/compose", + **payload, + rating=max(1, min(5, rating)), + ) + self.logger.info( + "Received response | " + + format_log_kv(source="UserService", body=response.body) + ) + return 200, response.body diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/review_storage.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/review_storage.py new file mode 100644 index 0000000..393c20d --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/review_storage.py @@ -0,0 +1,65 @@ +"""REST endpoints for review persistence.""" + +from eclypse.remote.communication import rest +from eclypse.remote.communication.rest import HTTPStatusCode +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class ReviewStorageService(RESTService): + """Persist reviews and provide review lookups.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the in-memory review store.""" + super().__init__(service_id, store_step=store_step) + self.reviews: dict[int, dict[str, object]] = {} + + @rest.endpoint("/store", "POST") + async def store( + self, + review_id: int, + movie_id: str, + movie_title: str, + rating: int, + text: str, + user: dict, + reply_to: str, + **payload, + ): + """Store the review and forward it to the user review index.""" + self.logger.info( + "Received request | " + + format_log_kv(review_id=review_id, movie_id=movie_id, user=user) + ) + review = { + "review_id": review_id, + "movie_id": movie_id, + "movie_title": movie_title, + "rating": rating, + "text": text, + "user": user, + } + self.reviews[review_id] = review + response = await self.rest.post( + "UserReviewService/write", + **payload, + review=review, + reply_to=reply_to, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="UserReviewService", body=response.body) + ) + return HTTPStatusCode.CREATED, response.body + + @rest.endpoint("/read_many", "GET") + def read_many(self, review_ids: list[int], **_): + """Read a batch of reviews from the in-memory store.""" + self.logger.info("Received request | " + format_log_kv(review_ids=review_ids)) + return 200, { + "reviews": [ + self.reviews[review_id] + for review_id in review_ids + if review_id in self.reviews + ] + } diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/text.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/text.py new file mode 100644 index 0000000..eb581f9 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/text.py @@ -0,0 +1,24 @@ +"""REST endpoints for review text processing.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class TextService(RESTService): + """Normalise review text before storing the review.""" + + @rest.endpoint("/compose", "POST") + async def compose(self, text: str, **payload): + """Normalise review text and forward the request to rating handling.""" + self.logger.info("Received request | " + format_log_kv(text=text)) + response = await self.rest.post( + "RatingService/compose", + **payload, + text=text.strip(), + ) + self.logger.info( + "Received response | " + + format_log_kv(source="RatingService", body=response.body) + ) + return 200, response.body diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/unique_id.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/unique_id.py new file mode 100644 index 0000000..8e0d78e --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/unique_id.py @@ -0,0 +1,30 @@ +"""REST endpoints for review identifier generation.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class UniqueIdService(RESTService): + """Assign review identifiers for compose-review requests.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the review identifier counter.""" + super().__init__(service_id, store_step=store_step) + self.next_review_id = 7000 + + @rest.endpoint("/compose", "POST") + async def compose(self, **payload): + """Assign a review id and forward the request to movie lookup.""" + self.logger.info("Received request | " + format_log_kv(payload=payload)) + self.next_review_id += 1 + response = await self.rest.post( + "MovieIdService/compose", + **payload, + review_id=self.next_review_id, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="MovieIdService", body=response.body) + ) + return 200, response.body diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/user.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/user.py new file mode 100644 index 0000000..9821047 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/user.py @@ -0,0 +1,35 @@ +"""REST endpoints for media-service user data.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class UserService(RESTService): + """Resolve user identities for review requests.""" + + @rest.endpoint("/compose", "POST") + async def compose(self, user_id: int, username: str, **payload): + """Attach user metadata and store the review.""" + self.logger.info( + "Received request | " + format_log_kv(user_id=user_id, username=username) + ) + user = {"user_id": user_id, "username": username} + response = await self.rest.post( + "ReviewStorageService/store", + **payload, + user=user, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="ReviewStorageService", body=response.body) + ) + return 200, response.body + + @rest.endpoint("/user", "GET") + def user(self, user_id: int, username: str, **_): + """Return a compact user descriptor for the review author.""" + self.logger.info( + "Received request | " + format_log_kv(user_id=user_id, username=username) + ) + return 200, {"user": {"user_id": user_id, "username": username}} diff --git a/eclypse/builders/application/deathstarbench/media_service/rest_services/user_review.py b/eclypse/builders/application/deathstarbench/media_service/rest_services/user_review.py new file mode 100644 index 0000000..7fd60d6 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/media_service/rest_services/user_review.py @@ -0,0 +1,35 @@ +"""REST endpoints for per-user review indexing.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class UserReviewService(RESTService): + """Index reviews by author.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the user review index.""" + super().__init__(service_id, store_step=store_step) + self.by_user: dict[int, list[int]] = {} + + @rest.endpoint("/write", "POST") + async def write(self, review: dict, reply_to: str, **_): + """Index the review by user and forward it to the movie review index.""" + self.logger.info( + "Received request | " + + format_log_kv(review_id=review["review_id"], user=review["user"]) + ) + self.by_user.setdefault(review["user"]["user_id"], []).append( + review["review_id"] + ) + response = await self.rest.post( + "MovieReviewService/write", + review=review, + reply_to=reply_to, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="MovieReviewService", body=response.body) + ) + return 200, response.body diff --git a/eclypse/builders/application/deathstarbench/social_network/__init__.py b/eclypse/builders/application/deathstarbench/social_network/__init__.py new file mode 100644 index 0000000..03e4c59 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/__init__.py @@ -0,0 +1,10 @@ +"""The social network application models a broadcast-style posting workflow. + +It includes post composition, text and media enrichment, user resolution, +timeline fan-out, and social-graph queries. It is inspired by the social +network application from the DeathStarBench benchmark suite. + +Source: + `DeathStarBench repository + `_ +""" diff --git a/eclypse/builders/application/deathstarbench/social_network/application.py b/eclypse/builders/application/deathstarbench/social_network/application.py new file mode 100644 index 0000000..1288308 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/application.py @@ -0,0 +1,272 @@ +"""Factory for a social network microservice application.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Literal, +) + +from eclypse.builders.application._helpers import build_application_from_specs + +if TYPE_CHECKING: + from eclypse.graph import Application + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + CommunicationInterface, + InitPolicy, + UpdatePolicies, + ) + + +def get_social_network( + application_id: str = "SocialNetwork", + communication_interface: CommunicationInterface | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + edge_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + requirement_init: InitPolicy = "min", + flows: Literal["default"] | list[list[str]] = "default", + store_step: bool = False, + seed: int | None = None, +) -> Application: + """Get the social network application. + + Args: + application_id (str): Identifier assigned to the generated application. + communication_interface (CommunicationInterface | None): + Communication backend used to instantiate executable services. When + ``None``, the builder returns a graph-only application. + update_policies (Callable | list[Callable] | None): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Optional assets attached to application nodes. + edge_assets (dict[str, Asset] | None): + Optional assets attached to application edges. + include_default_assets (bool): + Whether default graph assets should be included in the application. + requirement_init (InitPolicy): + Initialisation strategy applied to node and edge requirements. + flows (Literal["default"] | list[list[str]]): + User-defined application flows. Use ``"default"`` to install the + benchmark's built-in posting and timeline flows. + store_step (bool): + Whether instantiated services should store their step outputs in + the internal step queue. Ignored when + ``communication_interface`` is ``None``. + seed (int | None): + Seed forwarded to the application random generator. + + Returns: + Application: The configured social network application. + + Raises: + ValueError: If ``communication_interface`` is not supported. + """ + default_flows = [ + [ + "ComposePostService", + "UniqueIdService", + "TextService", + "UserMentionService", + "UrlShortenService", + "MediaService", + "UserService", + "PostStorageService", + "UserTimelineService", + "HomeTimelineService", + "SocialGraphService", + "HomeTimelineService", + "ComposePostService", + ], + ["UserTimelineService", "PostStorageService", "UserTimelineService"], + ["HomeTimelineService", "PostStorageService", "HomeTimelineService"], + ] + service_names = [ + "ComposePostService", + "UniqueIdService", + "TextService", + "UserMentionService", + "UrlShortenService", + "MediaService", + "UserService", + "PostStorageService", + "UserTimelineService", + "HomeTimelineService", + "SocialGraphService", + ] + node_requirements = { + "ComposePostService": { + "cpu": 2, + "gpu": 0, + "ram": 1.75, + "storage": 0.5, + "availability": 0.94, + "processing_time": 16, + }, + "UniqueIdService": { + "cpu": 1, + "gpu": 0, + "ram": 0.25, + "storage": 0.25, + "availability": 0.98, + "processing_time": 4, + }, + "TextService": { + "cpu": 1, + "gpu": 0, + "ram": 0.75, + "storage": 0.25, + "availability": 0.96, + "processing_time": 6, + }, + "UserMentionService": { + "cpu": 1, + "gpu": 0, + "ram": 0.5, + "storage": 0.25, + "availability": 0.96, + "processing_time": 5, + }, + "UrlShortenService": { + "cpu": 1, + "gpu": 0, + "ram": 1.0, + "storage": 0.75, + "availability": 0.95, + "processing_time": 8, + }, + "MediaService": { + "cpu": 2, + "gpu": 0, + "ram": 1.75, + "storage": 1.5, + "availability": 0.93, + "processing_time": 11, + }, + "UserService": { + "cpu": 1, + "gpu": 0, + "ram": 1.25, + "storage": 1.0, + "availability": 0.94, + "processing_time": 9, + }, + "PostStorageService": { + "cpu": 3, + "gpu": 0, + "ram": 2.5, + "storage": 3.0, + "availability": 0.91, + "processing_time": 13, + }, + "UserTimelineService": { + "cpu": 2, + "gpu": 0, + "ram": 2.0, + "storage": 1.75, + "availability": 0.92, + "processing_time": 11, + }, + "HomeTimelineService": { + "cpu": 2, + "gpu": 0, + "ram": 2.0, + "storage": 1.75, + "availability": 0.92, + "processing_time": 12, + }, + "SocialGraphService": { + "cpu": 2, + "gpu": 0, + "ram": 1.5, + "storage": 1.5, + "availability": 0.93, + "processing_time": 10, + }, + } + edge_requirements = [ + ( + "ComposePostService", + "UniqueIdService", + {"symmetric": True, "latency": 12, "bandwidth": 8}, + ), + ( + "UniqueIdService", + "TextService", + {"symmetric": True, "latency": 10, "bandwidth": 8}, + ), + ( + "TextService", + "UserMentionService", + {"symmetric": True, "latency": 10, "bandwidth": 6}, + ), + ( + "UserMentionService", + "UrlShortenService", + {"symmetric": True, "latency": 12, "bandwidth": 6}, + ), + ( + "UrlShortenService", + "MediaService", + {"symmetric": True, "latency": 14, "bandwidth": 12}, + ), + ( + "MediaService", + "UserService", + {"symmetric": True, "latency": 12, "bandwidth": 10}, + ), + ( + "UserService", + "PostStorageService", + {"symmetric": True, "latency": 14, "bandwidth": 14}, + ), + ( + "PostStorageService", + "UserTimelineService", + {"symmetric": True, "latency": 16, "bandwidth": 14}, + ), + ( + "UserTimelineService", + "HomeTimelineService", + {"symmetric": True, "latency": 14, "bandwidth": 12}, + ), + ( + "HomeTimelineService", + "SocialGraphService", + {"symmetric": True, "latency": 12, "bandwidth": 10}, + ), + ( + "HomeTimelineService", + "ComposePostService", + {"symmetric": True, "latency": 12, "bandwidth": 8}, + ), + ( + "UserTimelineService", + "PostStorageService", + {"symmetric": True, "latency": 14, "bandwidth": 14}, + ), + ( + "HomeTimelineService", + "PostStorageService", + {"symmetric": True, "latency": 14, "bandwidth": 14}, + ), + ] + return build_application_from_specs( + application_id=application_id, + communication_interface=communication_interface, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=edge_assets, + include_default_assets=include_default_assets, + requirement_init=requirement_init, + flows=flows, + store_step=store_step, + default_flows=default_flows, + service_names=service_names, + node_requirements=node_requirements, + edge_requirements=edge_requirements, + seed=seed, + package_name=__package__, + ) diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/__init__.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/__init__.py new file mode 100644 index 0000000..9a59d0d --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/__init__.py @@ -0,0 +1,27 @@ +"""MPI implementation for social network services.""" + +from .compose_post import ComposePostService +from .home_timeline import HomeTimelineService +from .media import MediaService +from .post_storage import PostStorageService +from .social_graph import SocialGraphService +from .text import TextService +from .unique_id import UniqueIdService +from .url_shorten import UrlShortenService +from .user import UserService +from .user_mention import UserMentionService +from .user_timeline import UserTimelineService + +__all__ = [ + "ComposePostService", + "HomeTimelineService", + "MediaService", + "PostStorageService", + "SocialGraphService", + "TextService", + "UniqueIdService", + "UrlShortenService", + "UserMentionService", + "UserService", + "UserTimelineService", +] diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/compose_post.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/compose_post.py new file mode 100644 index 0000000..8c72712 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/compose_post.py @@ -0,0 +1,39 @@ +"""MPI workflow for social-network post composition.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class ComposePostService(Service): + """Drive one social-network compose-post workflow.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the compose-post workflow state.""" + super().__init__(service_id, store_step=store_step) + self.req_id = 0 + self.user_id = 101 + self.username = "alice" + + async def step(self): + """Start the compose-post workflow and await the final response.""" + self.req_id += 1 + await self.submit_post() + response = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=response)) + return response + + @mpi.exchange(send=True) + def submit_post(self): + """Send a compose-post request into the social-network pipeline.""" + return "UniqueIdService", { + "request_type": "compose_post", + "reply_to": self.id, + "req_id": self.req_id, + "user_id": self.user_id, + "username": self.username, + "text": "Hello @bob check https://example.com", + "media_ids": [11], + "media_types": ["image"], + "post_type": "POST", + } diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/home_timeline.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/home_timeline.py new file mode 100644 index 0000000..91f037c --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/home_timeline.py @@ -0,0 +1,54 @@ +"""MPI workflow for home-timeline fan-out.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class HomeTimelineService(Service): + """Fan out posts to follower home timelines.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the home timeline store.""" + super().__init__(service_id, store_step=store_step) + self.timelines: dict[int, list[int]] = {} + self.pending_request: dict[str, object] = {} + + async def step(self): + """Handle the next home-timeline request.""" + await self.handle_request() + return await self.followers_response() + + @mpi.exchange(receive=True, send=True) + def handle_request(self, sender_id, body): + """Request followers before fan-out completes.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + self.pending_request = { + **body, + "sender_id": sender_id, + } + return "SocialGraphService", { + "request_type": "get_followers", + "user_id": body["creator"]["user_id"], + } + + @mpi.exchange(receive=True, send=True) + def followers_response(self, _sender_id, body): + """Complete the home-timeline fan-out once followers are known.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + home_receivers = [ + self.pending_request["creator"]["user_id"], + *body["followers"], + ] + for user_id in home_receivers: + self.timelines.setdefault(user_id, []).append( + self.pending_request["post_id"], + ) + return self.pending_request["reply_to"], { + "response_type": "compose_post_response", + "post_id": self.pending_request["post_id"], + "follower_count": len(body["followers"]), + "delivered_to": home_receivers, + "status": "posted", + "text": self.pending_request["post"]["text"], + } diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/media.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/media.py new file mode 100644 index 0000000..6039ed0 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/media.py @@ -0,0 +1,29 @@ +"""MPI workflow for media attachment composition.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class MediaService(Service): + """Attach media metadata to a social-network post.""" + + async def step(self): + """Handle the next media-composition request.""" + await self.compose_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def compose_request(self, _sender_id, body): + """Build media descriptors and forward the workflow to user lookup.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "UserService", { + **body, + "media": [ + {"media_id": media_id, "media_type": media_type} + for media_id, media_type in zip( + body["media_ids"], + body["media_types"], + strict=False, + ) + ], + } diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/post_storage.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/post_storage.py new file mode 100644 index 0000000..813bdd7 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/post_storage.py @@ -0,0 +1,47 @@ +"""MPI workflow for post persistence.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class PostStorageService(Service): + """Persist posts and serve timeline reads.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the in-memory post store.""" + super().__init__(service_id, store_step=store_step) + self.posts: dict[int, dict[str, object]] = {} + + async def step(self): + """Handle the next post-storage request.""" + await self.handle_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def handle_request(self, sender_id, body): + """Store posts or return a batch of posts to the requester.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + if body["request_type"] == "read_posts": + return sender_id, { + "response_type": "read_posts_response", + "posts": [ + self.posts[post_id] + for post_id in body["post_ids"] + if post_id in self.posts + ], + } + + post = { + "post_id": body["post_id"], + "creator": body["creator"], + "text": body["text"], + "user_mentions": body["user_mentions"], + "media": body["media"], + "urls": body["shortened_urls"], + } + self.posts[body["post_id"]] = post + return "UserTimelineService", { + **body, + "request_type": "write_user_timeline", + "post": post, + } diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/social_graph.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/social_graph.py new file mode 100644 index 0000000..d638e29 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/social_graph.py @@ -0,0 +1,31 @@ +"""MPI workflow for social-graph queries.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class SocialGraphService(Service): + """Return follower relationships for fan-out operations.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the social-graph fixture data.""" + super().__init__(service_id, store_step=store_step) + self.followers_map = { + 101: [202, 303], + 202: [101], + 303: [101, 202], + } + + async def step(self): + """Handle the next social-graph request.""" + await self.handle_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def handle_request(self, sender_id, body): + """Return followers for the requested user.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return sender_id, { + "response_type": "followers_response", + "followers": self.followers_map.get(body["user_id"], []), + } diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/text.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/text.py new file mode 100644 index 0000000..2d77df6 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/text.py @@ -0,0 +1,28 @@ +"""MPI workflow for text parsing and enrichment.""" + +import re + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + +_MENTION_RE = re.compile(r"@([a-zA-Z0-9_]+)") +_URL_RE = re.compile(r"https?://[^\\s]+") + + +class TextService(Service): + """Extract mentions and URLs from post text.""" + + async def step(self): + """Handle the next text-enrichment request.""" + await self.compose_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def compose_request(self, _sender_id, body): + """Parse post text and forward the workflow to mention resolution.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "UserMentionService", { + **body, + "mentions": _MENTION_RE.findall(body["text"]), + "urls": _URL_RE.findall(body["text"]), + } diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/unique_id.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/unique_id.py new file mode 100644 index 0000000..e5ce89e --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/unique_id.py @@ -0,0 +1,28 @@ +"""MPI workflow for social-network identifier generation.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class UniqueIdService(Service): + """Assign post identifiers for compose-post requests.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the identifier counter.""" + super().__init__(service_id, store_step=store_step) + self.next_post_id = 5000 + + async def step(self): + """Handle the next compose-post request from the workflow.""" + await self.compose_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def compose_request(self, _sender_id, body): + """Assign a post id and forward the workflow to the text service.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + self.next_post_id += 1 + return "TextService", { + **body, + "post_id": self.next_post_id, + } diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/url_shorten.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/url_shorten.py new file mode 100644 index 0000000..9c85e12 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/url_shorten.py @@ -0,0 +1,25 @@ +"""MPI workflow for URL shortening.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class UrlShortenService(Service): + """Shorten URLs contained in a social-network post.""" + + async def step(self): + """Handle the next URL-shortening request.""" + await self.compose_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def compose_request(self, _sender_id, body): + """Shorten URLs and forward the workflow to media handling.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "MediaService", { + **body, + "shortened_urls": [ + {"expanded_url": url, "shortened_url": f"https://t.ec/{idx}"} + for idx, url in enumerate(body["urls"], start=1) + ], + } diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/user.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/user.py new file mode 100644 index 0000000..707d4e0 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/user.py @@ -0,0 +1,25 @@ +"""MPI workflow for social-network user data.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class UserService(Service): + """Resolve user identities for compose-post requests.""" + + async def step(self): + """Handle the next user-resolution request.""" + await self.compose_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def compose_request(self, _sender_id, body): + """Attach creator metadata and forward the workflow to storage.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "PostStorageService", { + **body, + "creator": { + "user_id": body["user_id"], + "username": body["username"], + }, + } diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/user_mention.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/user_mention.py new file mode 100644 index 0000000..800206b --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/user_mention.py @@ -0,0 +1,25 @@ +"""MPI workflow for user-mention resolution.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class UserMentionService(Service): + """Resolve textual mentions into user identifiers.""" + + async def step(self): + """Handle the next mention-resolution request.""" + await self.compose_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def compose_request(self, _sender_id, body): + """Resolve mentions and forward the workflow to URL shortening.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "UrlShortenService", { + **body, + "user_mentions": [ + {"user_id": 200 + idx, "username": mention} + for idx, mention in enumerate(body["mentions"], start=1) + ], + } diff --git a/eclypse/builders/application/deathstarbench/social_network/mpi_services/user_timeline.py b/eclypse/builders/application/deathstarbench/social_network/mpi_services/user_timeline.py new file mode 100644 index 0000000..7051eb8 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/mpi_services/user_timeline.py @@ -0,0 +1,37 @@ +"""MPI workflow for user-timeline management.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class UserTimelineService(Service): + """Store and read per-user timelines.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the user timeline store.""" + super().__init__(service_id, store_step=store_step) + self.timelines: dict[int, list[int]] = {} + + async def step(self): + """Handle the next user-timeline request.""" + await self.handle_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def handle_request(self, sender_id, body): + """Write or read a user timeline.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + if body["request_type"] == "read_user_timeline": + return "PostStorageService", { + "request_type": "read_posts", + "reply_to": sender_id, + "post_ids": self.timelines.get(body["user_id"], []), + } + + self.timelines.setdefault(body["creator"]["user_id"], []).append( + body["post_id"] + ) + return "HomeTimelineService", { + **body, + "request_type": "write_home_timeline", + } diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/__init__.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/__init__.py new file mode 100644 index 0000000..aefab55 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/__init__.py @@ -0,0 +1,27 @@ +"""REST implementation for social network services.""" + +from .compose_post import ComposePostService +from .home_timeline import HomeTimelineService +from .media import MediaService +from .post_storage import PostStorageService +from .social_graph import SocialGraphService +from .text import TextService +from .unique_id import UniqueIdService +from .url_shorten import UrlShortenService +from .user import UserService +from .user_mention import UserMentionService +from .user_timeline import UserTimelineService + +__all__ = [ + "ComposePostService", + "HomeTimelineService", + "MediaService", + "PostStorageService", + "SocialGraphService", + "TextService", + "UniqueIdService", + "UrlShortenService", + "UserMentionService", + "UserService", + "UserTimelineService", +] diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/compose_post.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/compose_post.py new file mode 100644 index 0000000..f4f66f8 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/compose_post.py @@ -0,0 +1,39 @@ +"""REST workflow for social network post composition.""" + +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class ComposePostService(Service): + """Drive one social-network compose-post workflow.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the compose-post workflow state.""" + super().__init__( + service_id, + communication_interface="rest", + store_step=store_step, + ) + self.req_id = 0 + self.user_id = 101 + self.username = "alice" + + async def step(self): + """Compose a post and trigger the downstream fan-out pipeline.""" + self.req_id += 1 + response = await self.rest.post( + "UniqueIdService/compose", + req_id=self.req_id, + reply_to=self.id, + username=self.username, + user_id=self.user_id, + text="Hello @bob check https://example.com", + media_ids=[11], + media_types=["image"], + post_type="POST", + ) + self.logger.info( + "Received response | " + + format_log_kv(source="UniqueIdService", body=response.body) + ) + return response diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/home_timeline.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/home_timeline.py new file mode 100644 index 0000000..3f09595 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/home_timeline.py @@ -0,0 +1,56 @@ +"""REST endpoints for home-timeline fan-out.""" + +from eclypse.remote.communication import rest +from eclypse.remote.communication.rest import HTTPStatusCode +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class HomeTimelineService(RESTService): + """Fan out posts to follower home timelines.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the home timeline store.""" + super().__init__(service_id, store_step=store_step) + self.timelines: dict[int, list[int]] = {} + + @rest.endpoint("/write", "POST") + async def write(self, creator: dict, post_id: int, reply_to: str, **payload): + """Fan out the stored post to followers of the creator.""" + self.logger.info( + "Received request | " + format_log_kv(creator=creator, post_id=post_id) + ) + followers = await self.rest.get( + "SocialGraphService/followers", + user_id=creator["user_id"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="SocialGraphService", body=followers.body) + ) + home_receivers = [creator["user_id"], *followers.body["followers"]] + for user_id in home_receivers: + self.timelines.setdefault(user_id, []).append(post_id) + return HTTPStatusCode.CREATED, { + "reply_to": reply_to, + "post_id": post_id, + "follower_count": len(followers.body["followers"]), + "delivered_to": home_receivers, + "status": "posted", + "text": payload["post"]["text"], + } + + @rest.endpoint("/read", "GET") + async def read(self, user_id: int, **_): + """Read the home timeline for a user.""" + self.logger.info("Received request | " + format_log_kv(user_id=user_id)) + post_ids = self.timelines.get(user_id, []) + response = await self.rest.get( + "PostStorageService/read_many", + post_ids=post_ids, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="PostStorageService", body=response.body) + ) + return 200, {"user_id": user_id, "posts": response.body["posts"]} diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/media.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/media.py new file mode 100644 index 0000000..fa132ed --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/media.py @@ -0,0 +1,38 @@ +"""REST endpoints for media attachment composition.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class MediaService(RESTService): + """Attach media metadata to a social-network post.""" + + @rest.endpoint("/compose", "POST") + async def compose( + self, + media_ids: list[int], + media_types: list[str], + **payload, + ): + """Build media descriptors and forward the compose request.""" + self.logger.info( + "Received request | " + + format_log_kv(media_ids=media_ids, media_types=media_types) + ) + media = [ + {"media_id": media_id, "media_type": media_type} + for media_id, media_type in zip(media_ids, media_types, strict=False) + ] + response = await self.rest.post( + "UserService/compose", + **payload, + media_ids=media_ids, + media_types=media_types, + media=media, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="UserService", body=response.body) + ) + return 200, response.body diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/post_storage.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/post_storage.py new file mode 100644 index 0000000..f44c45d --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/post_storage.py @@ -0,0 +1,64 @@ +"""REST endpoints for post persistence.""" + +from eclypse.remote.communication import rest +from eclypse.remote.communication.rest import HTTPStatusCode +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class PostStorageService(RESTService): + """Persist posts and serve timeline reads.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the in-memory post store.""" + super().__init__(service_id, store_step=store_step) + self.posts: dict[int, dict[str, object]] = {} + + @rest.endpoint("/store", "POST") + async def store( + self, + post_id: int, + creator: dict, + text: str, + user_mentions: list[dict], + media: list[dict], + shortened_urls: list[dict], + reply_to: str, + **payload, + ): + """Store the composed post and forward it to the user timeline.""" + self.logger.info( + "Received request | " + format_log_kv(post_id=post_id, creator=creator) + ) + post = { + "post_id": post_id, + "creator": creator, + "text": text, + "user_mentions": user_mentions, + "media": media, + "urls": shortened_urls, + } + self.posts[post_id] = post + response = await self.rest.post( + "UserTimelineService/write", + **payload, + post_id=post_id, + creator=creator, + post=post, + reply_to=reply_to, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="UserTimelineService", body=response.body) + ) + return HTTPStatusCode.CREATED, response.body + + @rest.endpoint("/read_many", "GET") + def read_many(self, post_ids: list[int], **_): + """Read a batch of posts from the in-memory store.""" + self.logger.info("Received request | " + format_log_kv(post_ids=post_ids)) + return 200, { + "posts": [ + self.posts[post_id] for post_id in post_ids if post_id in self.posts + ] + } diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/social_graph.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/social_graph.py new file mode 100644 index 0000000..8688e17 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/social_graph.py @@ -0,0 +1,36 @@ +"""REST endpoints for social-graph queries.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class SocialGraphService(RESTService): + """Return follower relationships for fan-out operations.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the social-graph fixture data.""" + super().__init__(service_id, store_step=store_step) + self.followers_map = { + 101: [202, 303], + 202: [101], + 303: [101, 202], + } + + @rest.endpoint("/followers", "GET") + def followers(self, user_id: int, **_): + """Return the followers of the requested user.""" + self.logger.info("Received request | " + format_log_kv(user_id=user_id)) + return 200, {"followers": self.followers_map.get(user_id, [])} + + @rest.endpoint("/follow", "POST") + def follow(self, user_id: int, follower_id: int, **_): + """Add a follower relationship to the in-memory social graph.""" + self.logger.info( + "Received request | " + + format_log_kv(user_id=user_id, follower_id=follower_id) + ) + self.followers_map.setdefault(user_id, []) + if follower_id not in self.followers_map[user_id]: + self.followers_map[user_id].append(follower_id) + return 200, {"followers": self.followers_map[user_id]} diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/text.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/text.py new file mode 100644 index 0000000..a65e7c2 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/text.py @@ -0,0 +1,33 @@ +"""REST endpoints for text parsing and enrichment.""" + +import re + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + +_MENTION_RE = re.compile(r"@([a-zA-Z0-9_]+)") +_URL_RE = re.compile(r"https?://[^\\s]+") + + +class TextService(RESTService): + """Extract mentions and URLs from post text.""" + + @rest.endpoint("/compose", "POST") + async def compose(self, text: str, **payload): + """Parse the post text and forward enriched state downstream.""" + self.logger.info("Received request | " + format_log_kv(text=text)) + mentions = _MENTION_RE.findall(text) + urls = _URL_RE.findall(text) + response = await self.rest.post( + "UserMentionService/compose", + **payload, + text=text, + mentions=mentions, + urls=urls, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="UserMentionService", body=response.body) + ) + return 200, response.body diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/unique_id.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/unique_id.py new file mode 100644 index 0000000..1c5be37 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/unique_id.py @@ -0,0 +1,30 @@ +"""REST endpoints for social network identifier generation.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class UniqueIdService(RESTService): + """Assign post identifiers for compose-post requests.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the identifier counter.""" + super().__init__(service_id, store_step=store_step) + self.next_post_id = 5000 + + @rest.endpoint("/compose", "POST") + async def compose(self, **payload): + """Assign a post id and forward the compose request to the text stage.""" + self.logger.info("Received request | " + format_log_kv(payload=payload)) + self.next_post_id += 1 + response = await self.rest.post( + "TextService/compose", + **payload, + post_id=self.next_post_id, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="TextService", body=response.body) + ) + return 200, response.body diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/url_shorten.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/url_shorten.py new file mode 100644 index 0000000..aeb3945 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/url_shorten.py @@ -0,0 +1,29 @@ +"""REST endpoints for URL shortening.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class UrlShortenService(RESTService): + """Shorten URLs contained in a social-network post.""" + + @rest.endpoint("/compose", "POST") + async def compose(self, urls: list[str], **payload): + """Shorten URLs and forward the compose request to media handling.""" + self.logger.info("Received request | " + format_log_kv(urls=urls)) + shortened_urls = [ + {"expanded_url": url, "shortened_url": f"https://t.ec/{idx}"} + for idx, url in enumerate(urls, start=1) + ] + response = await self.rest.post( + "MediaService/compose", + **payload, + urls=urls, + shortened_urls=shortened_urls, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="MediaService", body=response.body) + ) + return 200, response.body diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/user.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/user.py new file mode 100644 index 0000000..23c76d0 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/user.py @@ -0,0 +1,37 @@ +"""REST endpoints for social-network user data.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class UserService(RESTService): + """Resolve user identities for compose-post requests.""" + + @rest.endpoint("/compose", "POST") + async def compose(self, user_id: int, username: str, **payload): + """Attach creator metadata and store the composed post.""" + self.logger.info( + "Received request | " + format_log_kv(user_id=user_id, username=username) + ) + creator = {"user_id": user_id, "username": username} + response = await self.rest.post( + "PostStorageService/store", + **payload, + user_id=user_id, + username=username, + creator=creator, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="PostStorageService", body=response.body) + ) + return 200, response.body + + @rest.endpoint("/creator", "GET") + def creator(self, user_id: int, username: str, **_): + """Return a compact creator description for the requested user.""" + self.logger.info( + "Received request | " + format_log_kv(user_id=user_id, username=username) + ) + return 200, {"creator": {"user_id": user_id, "username": username}} diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/user_mention.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/user_mention.py new file mode 100644 index 0000000..91f2e94 --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/user_mention.py @@ -0,0 +1,29 @@ +"""REST endpoints for user-mention resolution.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class UserMentionService(RESTService): + """Resolve textual mentions into user identifiers.""" + + @rest.endpoint("/compose", "POST") + async def compose(self, mentions: list[str], **payload): + """Resolve mentioned usernames and forward the compose request.""" + self.logger.info("Received request | " + format_log_kv(mentions=mentions)) + user_mentions = [ + {"user_id": 200 + idx, "username": mention} + for idx, mention in enumerate(mentions, start=1) + ] + response = await self.rest.post( + "UrlShortenService/compose", + **payload, + mentions=mentions, + user_mentions=user_mentions, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="UrlShortenService", body=response.body) + ) + return 200, response.body diff --git a/eclypse/builders/application/deathstarbench/social_network/rest_services/user_timeline.py b/eclypse/builders/application/deathstarbench/social_network/rest_services/user_timeline.py new file mode 100644 index 0000000..5e1e21e --- /dev/null +++ b/eclypse/builders/application/deathstarbench/social_network/rest_services/user_timeline.py @@ -0,0 +1,49 @@ +"""REST endpoints for user-timeline management.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class UserTimelineService(RESTService): + """Store and read per-user timelines.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the user timeline store.""" + super().__init__(service_id, store_step=store_step) + self.timelines: dict[int, list[int]] = {} + + @rest.endpoint("/write", "POST") + async def write(self, creator: dict, post_id: int, post: dict, reply_to: str, **_): + """Append a post to the creator timeline and fan out to home timelines.""" + self.logger.info( + "Received request | " + format_log_kv(creator=creator, post_id=post_id) + ) + self.timelines.setdefault(creator["user_id"], []).append(post_id) + response = await self.rest.post( + "HomeTimelineService/write", + creator=creator, + post_id=post_id, + post=post, + reply_to=reply_to, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="HomeTimelineService", body=response.body) + ) + return 200, response.body + + @rest.endpoint("/read", "GET") + async def read(self, user_id: int, **_): + """Read the stored posts for a user timeline.""" + self.logger.info("Received request | " + format_log_kv(user_id=user_id)) + post_ids = self.timelines.get(user_id, []) + response = await self.rest.get( + "PostStorageService/read_many", + post_ids=post_ids, + ) + self.logger.info( + "Received response | " + + format_log_kv(source="PostStorageService", body=response.body) + ) + return 200, {"user_id": user_id, "posts": response.body["posts"]} diff --git a/eclypse/builders/application/keyword_spotting/__init__.py b/eclypse/builders/application/keyword_spotting/__init__.py new file mode 100644 index 0000000..8a121e8 --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/__init__.py @@ -0,0 +1,10 @@ +"""The keyword spotting application models an edge inference pipeline. + +Short audio windows are captured, preprocessed, classified, and turned into a +command decision. It is inspired by the keyword spotting workload from MLPerf +Tiny. + +Source: + `MLPerf Tiny Inference Benchmark + `_ +""" diff --git a/eclypse/builders/application/keyword_spotting/application.py b/eclypse/builders/application/keyword_spotting/application.py new file mode 100644 index 0000000..f698448 --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/application.py @@ -0,0 +1,161 @@ +"""Factory for a keyword spotting application.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Literal, +) + +from eclypse.builders.application._helpers import ( + build_application_from_specs, +) + +if TYPE_CHECKING: + from eclypse.graph import Application + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + CommunicationInterface, + InitPolicy, + UpdatePolicies, + ) + + +def get_keyword_spotting( + application_id: str = "KeywordSpotting", + communication_interface: CommunicationInterface | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + edge_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + requirement_init: InitPolicy = "min", + flows: Literal["default"] | list[list[str]] = "default", + store_step: bool = False, + seed: int | None = None, +) -> Application: + """Get the keyword spotting application. + + Args: + application_id (str): Identifier assigned to the generated application. + communication_interface (CommunicationInterface | None): + Communication backend used to instantiate executable services. When + ``None``, the builder returns a graph-only application. + update_policies (Callable | list[Callable] | None): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Optional assets attached to application nodes. + edge_assets (dict[str, Asset] | None): + Optional assets attached to application edges. + include_default_assets (bool): + Whether default graph assets should be included in the application. + requirement_init (InitPolicy): + Initialisation strategy applied to node and edge requirements. + flows (Literal["default"] | list[list[str]]): + User-defined application flows. Use ``"default"`` to install the + benchmark's built-in audio-processing flow. + store_step (bool): + Whether instantiated services should store their step outputs in + the internal step queue. Ignored when + ``communication_interface`` is ``None``. + seed (int | None): + Seed forwarded to the application random generator. + + Returns: + Application: The configured keyword spotting application. + + Raises: + ValueError: If ``communication_interface`` is not supported. + """ + default_flows = [ + [ + "SensorService", + "PreprocessService", + "InferenceService", + "ActionService", + "SensorService", + ] + ] + service_names = [ + "ActionService", + "InferenceService", + "PreprocessService", + "SensorService", + ] + node_requirements = { + "SensorService": { + "cpu": 0.5, + "gpu": 0, + "ram": 0.25, + "storage": 0.1, + "availability": 0.98, + "processing_time": 2, + }, + "PreprocessService": { + "cpu": 1, + "gpu": 0, + "ram": 0.5, + "storage": 0.25, + "availability": 0.97, + "processing_time": 4, + }, + "InferenceService": { + "cpu": 2, + "gpu": 0.5, + "ram": 1.0, + "storage": 0.5, + "availability": 0.95, + "processing_time": 6, + }, + "ActionService": { + "cpu": 0.5, + "gpu": 0, + "ram": 0.25, + "storage": 0.1, + "availability": 0.99, + "processing_time": 2, + }, + } + edge_requirements = [ + ( + "SensorService", + "PreprocessService", + {"symmetric": True, "latency": 5, "bandwidth": 5}, + ), + ( + "PreprocessService", + "InferenceService", + {"symmetric": True, "latency": 8, "bandwidth": 5}, + ), + ( + "SensorService", + "InferenceService", + {"symmetric": True, "latency": 7, "bandwidth": 4}, + ), + ( + "InferenceService", + "ActionService", + {"symmetric": True, "latency": 5, "bandwidth": 3}, + ), + ( + "ActionService", + "SensorService", + {"symmetric": True, "latency": 4, "bandwidth": 2}, + ), + ] + return build_application_from_specs( + application_id=application_id, + communication_interface=communication_interface, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=edge_assets, + include_default_assets=include_default_assets, + requirement_init=requirement_init, + flows=flows, + store_step=store_step, + default_flows=default_flows, + service_names=service_names, + node_requirements=node_requirements, + edge_requirements=edge_requirements, + seed=seed, + package_name=__package__, + ) diff --git a/eclypse/builders/application/keyword_spotting/mpi_services/__init__.py b/eclypse/builders/application/keyword_spotting/mpi_services/__init__.py new file mode 100644 index 0000000..a6d778e --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/mpi_services/__init__.py @@ -0,0 +1,13 @@ +"""MPI implementation for keyword spotting services.""" + +from .action import ActionService +from .inference import InferenceService +from .preprocess import PreprocessService +from .sensor import SensorService + +__all__ = [ + "ActionService", + "InferenceService", + "PreprocessService", + "SensorService", +] diff --git a/eclypse/builders/application/keyword_spotting/mpi_services/action.py b/eclypse/builders/application/keyword_spotting/mpi_services/action.py new file mode 100644 index 0000000..3a37d34 --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/mpi_services/action.py @@ -0,0 +1,23 @@ +"""MPI workflow for acting on a detected keyword.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class ActionService(Service): + """Convert a keyword into a final command response.""" + + async def step(self): + """Handle the next inference result produced by the keyword model.""" + await self.inference_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def inference_request(self, _sender_id, body): + """Map the detected keyword to a command for the sensor.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "SensorService", { + "response_type": "keyword_response", + "window_id": body["window_id"], + "command": "wake" if body["keyword"] == "eclypse" else "idle", + } diff --git a/eclypse/builders/application/keyword_spotting/mpi_services/inference.py b/eclypse/builders/application/keyword_spotting/mpi_services/inference.py new file mode 100644 index 0000000..eef8393 --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/mpi_services/inference.py @@ -0,0 +1,28 @@ +"""MPI workflow for keyword inference.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + +KEYWORD_THRESHOLD = 5 + + +class InferenceService(Service): + """Infer a keyword from preprocessed features.""" + + async def step(self): + """Handle the next preprocessed audio feature vector.""" + await self.preprocess_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def preprocess_request(self, _sender_id, body): + """Infer the most likely keyword from the extracted features.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + keyword = ( + "eclypse" if sum(body["features"]) > KEYWORD_THRESHOLD else "background" + ) + return "ActionService", { + "request_type": "dispatch_action", + "window_id": body["window_id"], + "keyword": keyword, + } diff --git a/eclypse/builders/application/keyword_spotting/mpi_services/preprocess.py b/eclypse/builders/application/keyword_spotting/mpi_services/preprocess.py new file mode 100644 index 0000000..e01d244 --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/mpi_services/preprocess.py @@ -0,0 +1,24 @@ +"""MPI workflow for audio preprocessing.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class PreprocessService(Service): + """Turn raw samples into simple features.""" + + async def step(self): + """Handle the next raw audio window emitted by the sensor.""" + await self.sensor_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def sensor_request(self, _sender_id, body): + """Convert raw audio samples into a simple feature vector.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + features = [sample * 10 for sample in body["samples"]] + return "InferenceService", { + "request_type": "run_inference", + "window_id": body["window_id"], + "features": features, + } diff --git a/eclypse/builders/application/keyword_spotting/mpi_services/sensor.py b/eclypse/builders/application/keyword_spotting/mpi_services/sensor.py new file mode 100644 index 0000000..4e8e5f3 --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/mpi_services/sensor.py @@ -0,0 +1,31 @@ +"""MPI workflow for audio window capture.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class SensorService(Service): + """Generate audio windows and start the spotting pipeline.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the sensor with a rolling audio window counter.""" + super().__init__(service_id, store_step=store_step) + self.window_id = 0 + + async def step(self): + """Create the next audio window and wait for the command response.""" + self.window_id += 1 + await self.capture_window() + response = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=response)) + return response + + @mpi.exchange(send=True) + def capture_window(self): + """Send a synthetic audio window to the preprocessing service.""" + return "PreprocessService", { + "request_type": "preprocess_audio", + "window_id": self.window_id, + "samples": [0.1, 0.3, 0.2], + } diff --git a/eclypse/builders/application/keyword_spotting/rest_services/__init__.py b/eclypse/builders/application/keyword_spotting/rest_services/__init__.py new file mode 100644 index 0000000..fe52e9f --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/rest_services/__init__.py @@ -0,0 +1,13 @@ +"""REST implementation for keyword spotting services.""" + +from .action import ActionService +from .inference import InferenceService +from .preprocess import PreprocessService +from .sensor import SensorService + +__all__ = [ + "ActionService", + "InferenceService", + "PreprocessService", + "SensorService", +] diff --git a/eclypse/builders/application/keyword_spotting/rest_services/action.py b/eclypse/builders/application/keyword_spotting/rest_services/action.py new file mode 100644 index 0000000..4fe7dab --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/rest_services/action.py @@ -0,0 +1,20 @@ +"""REST endpoints for acting on a detected keyword.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class ActionService(RESTService): + """Convert a keyword into a final command response.""" + + @rest.endpoint("/action", "POST") + def action(self, window_id: int, keyword: str, **_): + """Return the command associated with the detected keyword.""" + self.logger.info( + "Received request | " + format_log_kv(window_id=window_id, keyword=keyword) + ) + return 200, { + "window_id": window_id, + "command": "wake" if keyword == "eclypse" else "idle", + } diff --git a/eclypse/builders/application/keyword_spotting/rest_services/inference.py b/eclypse/builders/application/keyword_spotting/rest_services/inference.py new file mode 100644 index 0000000..0383cf7 --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/rest_services/inference.py @@ -0,0 +1,21 @@ +"""REST endpoints for keyword inference.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + +KEYWORD_THRESHOLD = 5 + + +class InferenceService(RESTService): + """Infer a keyword from preprocessed features.""" + + @rest.endpoint("/infer", "POST") + def infer(self, window_id: int, features: list[float], **_): + """Infer the most likely keyword from the extracted features.""" + self.logger.info( + "Received request | " + + format_log_kv(window_id=window_id, features=features) + ) + keyword = "eclypse" if sum(features) > KEYWORD_THRESHOLD else "background" + return 200, {"window_id": window_id, "keyword": keyword} diff --git a/eclypse/builders/application/keyword_spotting/rest_services/preprocess.py b/eclypse/builders/application/keyword_spotting/rest_services/preprocess.py new file mode 100644 index 0000000..7d57705 --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/rest_services/preprocess.py @@ -0,0 +1,20 @@ +"""REST endpoints for audio preprocessing.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class PreprocessService(RESTService): + """Turn raw samples into simple features.""" + + @rest.endpoint("/preprocess", "POST") + def preprocess(self, window_id: int, samples: list[float], **_): + """Convert raw audio samples into a simple feature vector.""" + self.logger.info( + "Received request | " + format_log_kv(window_id=window_id, samples=samples) + ) + return 200, { + "window_id": window_id, + "features": [sample * 10 for sample in samples], + } diff --git a/eclypse/builders/application/keyword_spotting/rest_services/sensor.py b/eclypse/builders/application/keyword_spotting/rest_services/sensor.py new file mode 100644 index 0000000..43b01bd --- /dev/null +++ b/eclypse/builders/application/keyword_spotting/rest_services/sensor.py @@ -0,0 +1,49 @@ +"""REST workflow for audio window capture.""" + +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class SensorService(Service): + """Generate audio windows and start the spotting pipeline.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the sensor with a rolling audio window counter.""" + super().__init__( + service_id, + communication_interface="rest", + store_step=store_step, + ) + self.window_id = 0 + + async def step(self): + """Drive one audio window through the REST spotting pipeline.""" + self.window_id += 1 + preprocess_r = await self.rest.post( + "PreprocessService/preprocess", + window_id=self.window_id, + samples=[0.1, 0.3, 0.2], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="PreprocessService", body=preprocess_r.body) + ) + inference_r = await self.rest.post( + "InferenceService/infer", + window_id=self.window_id, + features=preprocess_r.body["features"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="InferenceService", body=inference_r.body) + ) + action_r = await self.rest.post( + "ActionService/action", + window_id=self.window_id, + keyword=inference_r.body["keyword"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="ActionService", body=action_r.body) + ) + return action_r diff --git a/eclypse/builders/application/sock_shop/__init__.py b/eclypse/builders/application/sock_shop/__init__.py index 7b01519..b9124ba 100644 --- a/eclypse/builders/application/sock_shop/__init__.py +++ b/eclypse/builders/application/sock_shop/__init__.py @@ -1,9 +1,12 @@ -"""Module for the Sock Shop application builder. +"""The Sock Shop application models an e-commerce microservice benchmark. -It provides a getter function to create a Sock Shop application in 3 -different configurations, by choosing the `communication interface` in the getter: +It represents a storefront workflow with user authentication, catalogue +browsing, cart management, checkout orchestration, payment processing, and +shipping coordination. The builder captures typical user-facing interactions +such as browsing products, updating the cart, placing orders, and tracking +their fulfilment across multiple services. -- "None": The application graph without any remote Service configuration. -- "rest": The application graph with REST Services. -- "mpi": The application graph with MPI Services. +Source: + `Sock Shop - A Microservices Demo Application + `_ """ diff --git a/eclypse/builders/application/sock_shop/application.py b/eclypse/builders/application/sock_shop/application.py index 1ca405e..1ab7256 100644 --- a/eclypse/builders/application/sock_shop/application.py +++ b/eclypse/builders/application/sock_shop/application.py @@ -1,40 +1,26 @@ -# pylint: disable=import-outside-toplevel -"""Factory for the SockShop microservice application. - -Defines the SockShop e-commerce demo as an Application object, modelling -typical user interactions such as browsing, cart updates, checkout, and -order tracking. Each microservice is assigned realistic compute and -performance requirements. - -Service interactions and structure are based on: -Sock Shop — A Microservices Demo Application, -https://github.com/ocp-power-demos/sock-shop-demo -""" +"""Factory for the Sock Shop application.""" from __future__ import annotations from typing import ( TYPE_CHECKING, Literal, - get_args, ) -from eclypse.builders._helpers import prune_assets -from eclypse.graph import Application -from eclypse.utils.types import CommunicationInterface +from eclypse.builders.application._helpers import ( + build_application_from_specs, +) if TYPE_CHECKING: + from eclypse.graph import Application from eclypse.graph.assets import Asset from eclypse.utils.types import ( + CommunicationInterface, InitPolicy, UpdatePolicies, ) -_SUPPORTED_COMMUNICATION_INTERFACES = get_args(CommunicationInterface) -"""Supported remote communication interfaces for the Sock Shop builders.""" - - def get_sock_shop( application_id: str = "SockShop", communication_interface: CommunicationInterface | None = None, @@ -44,6 +30,7 @@ def get_sock_shop( include_default_assets: bool = False, requirement_init: InitPolicy = "min", flows: Literal["default"] | list[list[str]] = "default", + store_step: bool = False, seed: int | None = None, ) -> Application: """Get the Sock Shop application. @@ -60,199 +47,147 @@ def get_sock_shop( Whether to include the default assets. Default is False. requirement_init (InitPolicy): The initialization of the requirements. flows (Literal["default"] | list[list[str]]): The flows of the application. + store_step (bool): + Whether instantiated services should store their step outputs in + the internal step queue. Ignored when + ``communication_interface`` is ``None``. seed (int | None): The seed for the random number generator. Returns: Application: The Sock Shop application. """ - if flows == "default": - _flows = [ - ["FrontendService", "UserService", "FrontendService"], # Login - ["FrontendService", "CatalogService", "FrontendService"], # Browsing - [ - "FrontendService", - "CatalogService", - "CartService", - "FrontendService", - ], # Adding to cart - [ - "FrontendService", - "PaymentService", - "OrderService", - "ShippingService", - "FrontendService", - ], # Checkout - [ - "FrontendService", - "OrderService", - "ShippingService", - "FrontendService", - ], # Shipping monitoring - ] - else: - _flows = flows - - app = Application( + default_flows = [ + ["FrontendService", "UserService", "FrontendService"], + ["FrontendService", "CatalogService", "FrontendService"], + ["FrontendService", "CartService", "FrontendService"], + [ + "FrontendService", + "OrderService", + "PaymentService", + "OrderService", + "FrontendService", + ], + [ + "FrontendService", + "OrderService", + "ShippingService", + "OrderService", + "FrontendService", + ], + ] + service_names = [ + "CatalogService", + "UserService", + "CartService", + "OrderService", + "PaymentService", + "ShippingService", + "FrontendService", + ] + node_requirements = { + "UserService": { + "cpu": 1, + "gpu": 0, + "ram": 0.75, + "storage": 0.3, + "availability": 0.92, + "processing_time": 9, + }, + "FrontendService": { + "cpu": 2, + "gpu": 0, + "ram": 1.5, + "storage": 0.3, + "availability": 0.94, + "processing_time": 20, + }, + "CatalogService": { + "cpu": 1, + "gpu": 0, + "ram": 1.0, + "storage": 0.75, + "availability": 0.92, + "processing_time": 11, + }, + "OrderService": { + "cpu": 2, + "gpu": 0, + "ram": 3.0, + "storage": 0.75, + "availability": 0.92, + "processing_time": 20, + }, + "CartService": { + "cpu": 2, + "gpu": 0, + "ram": 1.5, + "storage": 0.3, + "availability": 0.92, + "processing_time": 14, + }, + "PaymentService": { + "cpu": 1, + "gpu": 0, + "ram": 0.75, + "storage": 0.3, + "availability": 0.95, + "processing_time": 11, + }, + "ShippingService": { + "cpu": 2, + "gpu": 0, + "ram": 1.5, + "storage": 0.3, + "availability": 0.92, + "processing_time": 15, + }, + } + edge_requirements = [ + ( + "FrontendService", + "CatalogService", + {"symmetric": True, "latency": 40, "bandwidth": 2}, + ), + ( + "FrontendService", + "UserService", + {"symmetric": True, "latency": 40, "bandwidth": 2}, + ), + ( + "FrontendService", + "CartService", + {"symmetric": True, "latency": 40, "bandwidth": 2}, + ), + ( + "FrontendService", + "OrderService", + {"symmetric": True, "latency": 50, "bandwidth": 10}, + ), + ( + "OrderService", + "PaymentService", + {"symmetric": True, "latency": 50, "bandwidth": 10}, + ), + ( + "OrderService", + "ShippingService", + {"symmetric": True, "latency": 70, "bandwidth": 10}, + ), + ] + return build_application_from_specs( application_id=application_id, + communication_interface=communication_interface, update_policies=update_policies, node_assets=node_assets, edge_assets=edge_assets, include_default_assets=include_default_assets, requirement_init=requirement_init, - flows=_flows, + flows=flows, + store_step=store_step, + default_flows=default_flows, + service_names=service_names, + node_requirements=node_requirements, + edge_requirements=edge_requirements, seed=seed, + package_name=__package__, ) - - if communication_interface is None: - add_fn = app.add_node - - def id_fn(service): - return service - - elif communication_interface in _SUPPORTED_COMMUNICATION_INTERFACES: - add_fn = app.add_service # type: ignore[assignment] - if communication_interface == "mpi": - from . import mpi_services as services - else: - from . import rest_services as services # type: ignore[no-redef] - - classes = { - "CatalogService": services.CatalogService, - "UserService": services.UserService, - "CartService": services.CartService, - "OrderService": services.OrderService, - "PaymentService": services.PaymentService, - "ShippingService": services.ShippingService, - "FrontendService": services.FrontendService, - } - - def id_fn(service): - return classes[service](service) - - else: - raise ValueError(f"Unknown communication interface: {communication_interface}") - - add_fn( - id_fn("UserService"), - **prune_assets( - app.node_assets, - cpu=1, - gpu=0, - ram=0.75, - storage=0.3, - availability=0.91, - processing_time=10, - ), - ) - add_fn( - id_fn("FrontendService"), - **prune_assets( - app.node_assets, - cpu=1, - gpu=0, - ram=0.75, - storage=0.3, - availability=0.94, - processing_time=30, - ), - ) - add_fn( - id_fn("CatalogService"), - **prune_assets( - app.node_assets, - cpu=1, - gpu=0, - ram=1.5, - storage=0.75, - availability=0.91, - processing_time=12.5, - ), - ) - add_fn( - id_fn("OrderService"), - **prune_assets( - app.node_assets, - cpu=2, - gpu=0, - ram=3.0, - storage=0.75, - availability=0.92, - processing_time=20, - ), - ) - add_fn( - id_fn("CartService"), - **prune_assets( - app.node_assets, - cpu=1, - gpu=0, - ram=0.75, - storage=0.3, - availability=0.91, - processing_time=10, - ), - ) - add_fn( - id_fn("PaymentService"), - **prune_assets( - app.node_assets, - cpu=1, - gpu=0, - ram=0.75, - storage=0.3, - availability=0.95, - processing_time=12.5, - ), - ) - add_fn( - id_fn("ShippingService"), - **prune_assets( - app.node_assets, - cpu=1, - gpu=0, - ram=0.75, - storage=0.3, - availability=0.915, - processing_time=17.5, - ), - ) - - app.add_edge( - "FrontendService", - "CatalogService", - symmetric=True, - **prune_assets(app.edge_assets, latency=40, bandwidth=2), - ) - app.add_edge( - "FrontendService", - "UserService", - symmetric=True, - **prune_assets(app.edge_assets, latency=40, bandwidth=2), - ) - app.add_edge( - "FrontendService", - "CartService", - symmetric=True, - **prune_assets(app.edge_assets, latency=40, bandwidth=2), - ) - app.add_edge( - "FrontendService", - "OrderService", - symmetric=True, - **prune_assets(app.edge_assets, latency=50, bandwidth=10), - ) - - app.add_edge( - "OrderService", - "PaymentService", - symmetric=True, - **prune_assets(app.edge_assets, latency=50, bandwidth=10), - ) - app.add_edge( - "OrderService", - "ShippingService", - symmetric=True, - **prune_assets(app.edge_assets, latency=70, bandwidth=10), - ) - - return app diff --git a/eclypse/builders/application/sock_shop/mpi_services/cart.py b/eclypse/builders/application/sock_shop/mpi_services/cart.py index 5ba3148..0c9ab3e 100644 --- a/eclypse/builders/application/sock_shop/mpi_services/cart.py +++ b/eclypse/builders/application/sock_shop/mpi_services/cart.py @@ -7,6 +7,7 @@ from eclypse.remote.communication import mpi from eclypse.remote.service import Service +from eclypse.utils import format_log_kv class CartService(Service): @@ -31,15 +32,15 @@ def frontend_request(self, sender_id, body): str: The ID of the recipient. dict: The response body. """ - self.logger.info(f"{self.id} - {body}") + self.logger.info("Received request | " + format_log_kv(request=body)) # Send response to FrontendService if body.get("request_type") == "cart_data": frontend_response = { "response_type": "cart_response", "items": [ - {"product_id": "1", "quantity": 2}, - {"product_id": "2", "quantity": 1}, + {"id": "1", "quantity": 2}, + {"id": "2", "quantity": 1}, ], } else: diff --git a/eclypse/builders/application/sock_shop/mpi_services/catalog.py b/eclypse/builders/application/sock_shop/mpi_services/catalog.py index 0381d14..5e79f17 100644 --- a/eclypse/builders/application/sock_shop/mpi_services/catalog.py +++ b/eclypse/builders/application/sock_shop/mpi_services/catalog.py @@ -9,6 +9,7 @@ from eclypse.remote.communication import mpi from eclypse.remote.service import Service +from eclypse.utils import format_log_kv class CatalogService(Service): @@ -34,7 +35,7 @@ def frontend_request(self, sender_id, body): str: The ID of the recipient. dict: The response body. """ - self.logger.info(f"{self.id} - {body}") + self.logger.info("Received request | " + format_log_kv(request=body)) # Send response to FrontendService if body.get("request_type") == "catalog_data": diff --git a/eclypse/builders/application/sock_shop/mpi_services/frontend.py b/eclypse/builders/application/sock_shop/mpi_services/frontend.py index e8417fd..38989f5 100644 --- a/eclypse/builders/application/sock_shop/mpi_services/frontend.py +++ b/eclypse/builders/application/sock_shop/mpi_services/frontend.py @@ -13,18 +13,21 @@ from eclypse.remote.communication import mpi from eclypse.remote.service import Service +from eclypse.utils import format_log_kv class FrontendService(Service): """MPI workflow of the Frontend service.""" - def __init__(self, name): + def __init__(self, name, store_step: bool = False): """Initialize the FrontendService with a user ID. Args: name (str): The name of the service. + store_step (bool, optional): Whether to store the results of + each step. Defaults to False. """ - super().__init__(name) + super().__init__(name, store_step=store_step) self.user_id = 12345 async def step(self): @@ -39,7 +42,10 @@ async def step(self): # Receive response from CatalogService catalog_response = await self.mpi.recv() - self.logger.info(f"{self.id} - {catalog_response}") + self.logger.info( + "Received response | " + + format_log_kv(source="CatalogService", body=catalog_response) + ) # Send request to UserService user_request = {"request_type": "user_data", "user_id": self.user_id} @@ -47,7 +53,10 @@ async def step(self): # Receive response from UserService user_response = await self.mpi.recv() - self.logger.info(f"{self.id} - {user_response}") + self.logger.info( + "Received response | " + + format_log_kv(source="UserService", body=user_response) + ) # Send request to CartService cart_request = {"request_type": "cart_data", "user_id": self.user_id} @@ -55,21 +64,42 @@ async def step(self): # Receive response from CartService cart_response = await self.mpi.recv() - self.logger.info(f"{self.id} - {cart_response}") + self.logger.info( + "Received response | " + + format_log_kv(source="CartService", body=cart_response) + ) + products = catalog_response.get("products", []) cart_items = cart_response.get("items", []) + order_items = [ + { + "id": item["id"], + "amount": next( + ( + product["price"] * item["quantity"] + for product in products + if product["id"] == item["id"] + ), + 0.0, + ), + } + for item in cart_items + ] # Send request to OrderService order_request = { "request_type": "order_request", "user_id": self.user_id, - "items": cart_items, + "items": order_items, } self.mpi.send("OrderService", order_request) # Receive response from OrderService order_response = await self.mpi.recv() - self.logger.info(f"{self.id} - {order_response}") + self.logger.info( + "Received response | " + + format_log_kv(source="OrderService", body=order_response) + ) @mpi.exchange(send=True) def catalog_request(self): diff --git a/eclypse/builders/application/sock_shop/mpi_services/order.py b/eclypse/builders/application/sock_shop/mpi_services/order.py index 4640b5a..e7270d7 100644 --- a/eclypse/builders/application/sock_shop/mpi_services/order.py +++ b/eclypse/builders/application/sock_shop/mpi_services/order.py @@ -11,26 +11,27 @@ - Tracks the status of placed orders (e.g., pending, confirmed, shipped). """ -import random as rnd - from eclypse.remote.communication import mpi from eclypse.remote.service import Service +from eclypse.utils import format_log_kv class OrderService(Service): """MPI workflow of the Order service.""" - def __init__(self, name): + def __init__(self, name, store_step: bool = False): """Initialize the OrderService with an order ID. Args: name (str): The name of the service. + store_step (bool, optional): Whether to store the results of + each step. Defaults to False. """ - super().__init__(name) + super().__init__(name, store_step=store_step) self.order_id = 54321 - self.transaction_id = None - self.shipping_details = {} - self.items = [] + self.transaction_id: int | None = None + self.shipping_details: dict[str, str] = {} + self.items: list[dict[str, int]] = [] async def step(self): """Example workflow of the `OrderService` class. @@ -53,10 +54,10 @@ def frontend_request(self, _, body): str: The ID of the recipient. dict: The response body. """ - self.logger.info(f"{self.id} - {body}") + self.logger.info("Received request | " + format_log_kv(request=body)) self.items = body.get("items", []) - total_amount = sum(rnd.randint(20, 100) for _ in self.items) + total_amount = sum(item.get("amount", 0.0) for item in self.items) # Send request to PaymentService payment_request = { @@ -78,9 +79,12 @@ def payment_request(self, _, body): str: The ID of the recipient. dict: The response body. """ - self.logger.info(f"{self.id} - {body}") + self.logger.info("Received request | " + format_log_kv(request=body)) self.transaction_id = body.get("transaction_id") + self.logger.info( + "Received response | " + format_log_kv(source="PaymentService", body=body) + ) # Send request to ShippingService shipping_request = { "request_type": "shipping_request", @@ -100,9 +104,12 @@ def shipping_request(self, _, body): str: The ID of the recipient. dict: The response body. """ - self.logger.info(f"{self.id} - {body}") + self.logger.info("Received request | " + format_log_kv(request=body)) self.shipping_details = body.get("details") + self.logger.info( + "Received response | " + format_log_kv(source="ShippingService", body=body) + ) # Send response to FrontendService if self.transaction_id is not None: diff --git a/eclypse/builders/application/sock_shop/mpi_services/payment.py b/eclypse/builders/application/sock_shop/mpi_services/payment.py index bb144ca..dd87b1f 100644 --- a/eclypse/builders/application/sock_shop/mpi_services/payment.py +++ b/eclypse/builders/application/sock_shop/mpi_services/payment.py @@ -12,6 +12,7 @@ from eclypse.remote.communication import mpi from eclypse.remote.service import Service +from eclypse.utils import format_log_kv class PaymentService(Service): @@ -36,13 +37,14 @@ def order_request(self, sender_id, body): str: The ID of the recipient. dict: The response body. """ - self.logger.info(f"{self.id} - {body}") + self.logger.info("Received request | " + format_log_kv(request=body)) # Send response to OrderService if body.get("request_type") == "payment_request": payment_response = { "response_type": "payment_response", "order_id": body.get("order_id"), + "transaction_id": rnd.randint(1000, 9999), "status": "success" if rnd.choice([True, False]) else "failure", } else: diff --git a/eclypse/builders/application/sock_shop/mpi_services/shipping.py b/eclypse/builders/application/sock_shop/mpi_services/shipping.py index e891439..636b2ef 100644 --- a/eclypse/builders/application/sock_shop/mpi_services/shipping.py +++ b/eclypse/builders/application/sock_shop/mpi_services/shipping.py @@ -10,6 +10,7 @@ from eclypse.remote.communication import mpi from eclypse.remote.service import Service +from eclypse.utils import format_log_kv class ShippingService(Service): @@ -34,7 +35,7 @@ def order_request(self, sender_id, body): str: The ID of the recipient. dict: The response body. """ - self.logger.info(f"{self.id} - {body}") + self.logger.info("Received request | " + format_log_kv(request=body)) # Send response to OrderService if body.get("request_type") == "shipping_request": diff --git a/eclypse/builders/application/sock_shop/mpi_services/user.py b/eclypse/builders/application/sock_shop/mpi_services/user.py index 6353852..dab78f4 100644 --- a/eclypse/builders/application/sock_shop/mpi_services/user.py +++ b/eclypse/builders/application/sock_shop/mpi_services/user.py @@ -10,6 +10,7 @@ from eclypse.remote.communication import mpi from eclypse.remote.service import Service +from eclypse.utils import format_log_kv class UserService(Service): @@ -34,7 +35,7 @@ def frontend_request(self, sender_id, body): str: The ID of the recipient. dict: The response body. """ - self.logger.info(f"{self.id} - {body}") + self.logger.info("Received request | " + format_log_kv(request=body)) # Send response to FrontendService if body.get("request_type") == "user_data": diff --git a/eclypse/builders/application/sock_shop/rest_services/cart.py b/eclypse/builders/application/sock_shop/rest_services/cart.py index f7b7870..7033a4c 100644 --- a/eclypse/builders/application/sock_shop/rest_services/cart.py +++ b/eclypse/builders/application/sock_shop/rest_services/cart.py @@ -7,6 +7,7 @@ from eclypse.remote.communication import rest from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv class CartService(RESTService): @@ -33,6 +34,7 @@ def get_cart(self, **_): }, ) """ + self.logger.info("Received request | " + format_log_kv()) return 200, { "items": [ {"id": "1", "quantity": 2}, diff --git a/eclypse/builders/application/sock_shop/rest_services/catalog.py b/eclypse/builders/application/sock_shop/rest_services/catalog.py index 7cfd093..d989ee8 100644 --- a/eclypse/builders/application/sock_shop/rest_services/catalog.py +++ b/eclypse/builders/application/sock_shop/rest_services/catalog.py @@ -9,6 +9,7 @@ from eclypse.remote.communication import rest from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv class CatalogService(RESTService): @@ -35,6 +36,7 @@ def get_catalog(self, **_): }, ) """ + self.logger.info("Received request | " + format_log_kv()) return 200, { "products": [ {"id": "1", "name": "Product 1", "price": 19.99}, diff --git a/eclypse/builders/application/sock_shop/rest_services/frontend.py b/eclypse/builders/application/sock_shop/rest_services/frontend.py index c3fa4c8..7bf8c0e 100644 --- a/eclypse/builders/application/sock_shop/rest_services/frontend.py +++ b/eclypse/builders/application/sock_shop/rest_services/frontend.py @@ -12,14 +12,19 @@ """ from eclypse.remote.service import Service +from eclypse.utils import format_log_kv class FrontendService(Service): """Example workflow of the Frontend service.""" - def __init__(self, name): + def __init__(self, name, store_step: bool = False): """Initialise the Frontend service with the REST interface.""" - super().__init__(name, communication_interface="rest") + super().__init__( + name, + communication_interface="rest", + store_step=store_step, + ) self.user_id = 12345 async def step(self): @@ -32,10 +37,21 @@ async def step(self): user_r = await self.rest.get("UserService/user", user_id=self.user_id) cart_r = await self.rest.get("CartService/cart") + self.logger.info( + "Received response | " + + format_log_kv(source="CatalogService", body=catalog_r.body) + ) + self.logger.info( + "Received response | " + + format_log_kv(source="UserService", body=user_r.body) + ) + self.logger.info( + "Received response | " + + format_log_kv(source="CartService", body=cart_r.body) + ) + products = catalog_r.body.get("products", []) items = cart_r.body.get("items", []) - user_data = user_r.body - self.logger.info(f"{self.id} - {user_data}") order_items = [ { @@ -53,4 +69,7 @@ async def step(self): ] order_r = await self.rest.post("OrderService/order", items=order_items) - self.logger.info(f"{order_r.body}") + self.logger.info( + "Received response | " + + format_log_kv(source="OrderService", body=order_r.body) + ) diff --git a/eclypse/builders/application/sock_shop/rest_services/order.py b/eclypse/builders/application/sock_shop/rest_services/order.py index 752e261..6c9c89e 100644 --- a/eclypse/builders/application/sock_shop/rest_services/order.py +++ b/eclypse/builders/application/sock_shop/rest_services/order.py @@ -14,18 +14,21 @@ from eclypse.remote.communication import rest from eclypse.remote.communication.rest import HTTPStatusCode from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv class OrderService(RESTService): """REST endpoints for the Order service.""" - def __init__(self, name): + def __init__(self, name, store_step: bool = False): """Initialize the OrderService with an order ID. Args: name (str): The name of the service. + store_step (bool, optional): Whether to store the results of + each step. Defaults to False. """ - super().__init__(name) + super().__init__(name, store_step=store_step) self.order_id = 54321 @rest.endpoint("/order", "POST") @@ -56,6 +59,8 @@ async def create_order(self, items, **_): }, ) """ + self.logger.info("Received request | " + format_log_kv(items=items)) + amount = sum(item["amount"] for item in items) payment_r = self.rest.post( "PaymentService/pay", @@ -70,7 +75,14 @@ async def create_order(self, items, **_): shipping_details = shipping_r.body.get("shipping_details") transaction_id = payment_r.body.get("transaction_id") - self.logger.info(f"{transaction_id}") + self.logger.info( + "Received response | " + + format_log_kv(source="PaymentService", transaction_id=transaction_id) + ) + self.logger.info( + "Received response | " + + format_log_kv(source="ShippingService", body=shipping_r.body) + ) return HTTPStatusCode.CREATED, { "order_id": self.order_id, diff --git a/eclypse/builders/application/sock_shop/rest_services/payment.py b/eclypse/builders/application/sock_shop/rest_services/payment.py index d29bc7d..89ad526 100644 --- a/eclypse/builders/application/sock_shop/rest_services/payment.py +++ b/eclypse/builders/application/sock_shop/rest_services/payment.py @@ -13,19 +13,22 @@ from eclypse.remote.communication import rest from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv from eclypse.utils.constants import RND_SEED class PaymentService(RESTService): """REST service for payment processing.""" - def __init__(self, service_id: str): + def __init__(self, service_id: str, store_step: bool = False): """Initialize the PaymentService with a random number generator. Args: service_id (str): The ID of the service. + store_step (bool, optional): Whether to store the results of + each step. Defaults to False. """ - super().__init__(service_id) + super().__init__(service_id, store_step=store_step) self.rnd = rnd.Random(os.getenv(RND_SEED)) @rest.endpoint("/pay", "POST") @@ -52,6 +55,9 @@ def execute_payment(self, order_id: int, amount: float, **_): }, ) """ + self.logger.info( + "Received request | " + format_log_kv(order_id=order_id, amount=amount) + ) return 200, { "order_id": order_id, "amount": amount + self.rnd.randint(1, 10), diff --git a/eclypse/builders/application/sock_shop/rest_services/shipping.py b/eclypse/builders/application/sock_shop/rest_services/shipping.py index db8f20d..486e92e 100644 --- a/eclypse/builders/application/sock_shop/rest_services/shipping.py +++ b/eclypse/builders/application/sock_shop/rest_services/shipping.py @@ -10,6 +10,7 @@ from eclypse.remote.communication import rest from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv class ShippingService(RESTService): @@ -42,6 +43,7 @@ def get_shipping_detils(self, order_id, **_): }, ) """ + self.logger.info("Received request | " + format_log_kv(order_id=order_id)) return 200, { "order_id": order_id, "status": "success", diff --git a/eclypse/builders/application/sock_shop/rest_services/user.py b/eclypse/builders/application/sock_shop/rest_services/user.py index 958dd5b..30287e5 100644 --- a/eclypse/builders/application/sock_shop/rest_services/user.py +++ b/eclypse/builders/application/sock_shop/rest_services/user.py @@ -10,6 +10,7 @@ from eclypse.remote.communication import rest from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv class UserService(RESTService): @@ -40,6 +41,7 @@ def get_catalog(self, user_id: int, **_): }, ) """ + self.logger.info("Received request | " + format_log_kv(user_id=user_id)) return 200, { "user_id": user_id, "name": "John Doe", diff --git a/eclypse/builders/application/thumbnailer/__init__.py b/eclypse/builders/application/thumbnailer/__init__.py new file mode 100644 index 0000000..daf29df --- /dev/null +++ b/eclypse/builders/application/thumbnailer/__init__.py @@ -0,0 +1,10 @@ +"""The thumbnailer application models a media-processing pipeline. + +An uploaded image is transformed into a thumbnail, stored, and acknowledged. +It is inspired by the thumbnailer workload from the SeBS serverless benchmark +suite. + +Source: + `SeBS benchmark applications + `_ +""" diff --git a/eclypse/builders/application/thumbnailer/application.py b/eclypse/builders/application/thumbnailer/application.py new file mode 100644 index 0000000..8cc268e --- /dev/null +++ b/eclypse/builders/application/thumbnailer/application.py @@ -0,0 +1,161 @@ +"""Factory for a thumbnailer application.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Literal, +) + +from eclypse.builders.application._helpers import ( + build_application_from_specs, +) + +if TYPE_CHECKING: + from eclypse.graph import Application + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + CommunicationInterface, + InitPolicy, + UpdatePolicies, + ) + + +def get_thumbnailer( + application_id: str = "Thumbnailer", + communication_interface: CommunicationInterface | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + edge_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + requirement_init: InitPolicy = "min", + flows: Literal["default"] | list[list[str]] = "default", + store_step: bool = False, + seed: int | None = None, +) -> Application: + """Get the thumbnailer application. + + Args: + application_id (str): Identifier assigned to the generated application. + communication_interface (CommunicationInterface | None): + Communication backend used to instantiate executable services. When + ``None``, the builder returns a graph-only application. + update_policies (Callable | list[Callable] | None): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Optional assets attached to application nodes. + edge_assets (dict[str, Asset] | None): + Optional assets attached to application edges. + include_default_assets (bool): + Whether default graph assets should be included in the application. + requirement_init (InitPolicy): + Initialisation strategy applied to node and edge requirements. + flows (Literal["default"] | list[list[str]]): + User-defined application flows. Use ``"default"`` to install the + benchmark's built-in media-processing flow. + store_step (bool): + Whether instantiated services should store their step outputs in + the internal step queue. Ignored when + ``communication_interface`` is ``None``. + seed (int | None): + Seed forwarded to the application random generator. + + Returns: + Application: The configured thumbnailer application. + + Raises: + ValueError: If ``communication_interface`` is not supported. + """ + default_flows = [ + [ + "UploadService", + "TransformService", + "StorageService", + "NotificationService", + "UploadService", + ] + ] + service_names = [ + "NotificationService", + "StorageService", + "TransformService", + "UploadService", + ] + node_requirements = { + "UploadService": { + "cpu": 0.5, + "gpu": 0, + "ram": 0.25, + "storage": 0.25, + "availability": 0.98, + "processing_time": 3, + }, + "TransformService": { + "cpu": 2, + "gpu": 0.5, + "ram": 1.0, + "storage": 0.5, + "availability": 0.96, + "processing_time": 8, + }, + "StorageService": { + "cpu": 1, + "gpu": 0, + "ram": 0.5, + "storage": 1.5, + "availability": 0.97, + "processing_time": 5, + }, + "NotificationService": { + "cpu": 0.5, + "gpu": 0, + "ram": 0.25, + "storage": 0.1, + "availability": 0.99, + "processing_time": 2, + }, + } + edge_requirements = [ + ( + "UploadService", + "TransformService", + {"symmetric": True, "latency": 8, "bandwidth": 8}, + ), + ( + "TransformService", + "StorageService", + {"symmetric": True, "latency": 10, "bandwidth": 10}, + ), + ( + "UploadService", + "StorageService", + {"symmetric": True, "latency": 9, "bandwidth": 8}, + ), + ( + "StorageService", + "NotificationService", + {"symmetric": True, "latency": 5, "bandwidth": 3}, + ), + ( + "NotificationService", + "UploadService", + {"symmetric": True, "latency": 4, "bandwidth": 2}, + ), + ] + return build_application_from_specs( + application_id=application_id, + communication_interface=communication_interface, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=edge_assets, + include_default_assets=include_default_assets, + requirement_init=requirement_init, + flows=flows, + store_step=store_step, + default_flows=default_flows, + service_names=service_names, + node_requirements=node_requirements, + edge_requirements=edge_requirements, + seed=seed, + package_name=__package__, + ) diff --git a/eclypse/builders/application/thumbnailer/mpi_services/__init__.py b/eclypse/builders/application/thumbnailer/mpi_services/__init__.py new file mode 100644 index 0000000..04e88c9 --- /dev/null +++ b/eclypse/builders/application/thumbnailer/mpi_services/__init__.py @@ -0,0 +1,13 @@ +"""MPI implementation for thumbnailer services.""" + +from .notification import NotificationService +from .storage import StorageService +from .transform import TransformService +from .upload import UploadService + +__all__ = [ + "NotificationService", + "StorageService", + "TransformService", + "UploadService", +] diff --git a/eclypse/builders/application/thumbnailer/mpi_services/notification.py b/eclypse/builders/application/thumbnailer/mpi_services/notification.py new file mode 100644 index 0000000..1f4e0c4 --- /dev/null +++ b/eclypse/builders/application/thumbnailer/mpi_services/notification.py @@ -0,0 +1,24 @@ +"""MPI workflow for upload notification.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class NotificationService(Service): + """Return the final thumbnail location.""" + + async def step(self): + """Handle the next storage confirmation emitted by the pipeline.""" + await self.storage_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def storage_request(self, _sender_id, body): + """Return the final storage location to the upload service.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "UploadService", { + "response_type": "thumbnail_ready", + "image_id": body["image_id"], + "uri": body["uri"], + "status": "stored", + } diff --git a/eclypse/builders/application/thumbnailer/mpi_services/storage.py b/eclypse/builders/application/thumbnailer/mpi_services/storage.py new file mode 100644 index 0000000..a41e763 --- /dev/null +++ b/eclypse/builders/application/thumbnailer/mpi_services/storage.py @@ -0,0 +1,23 @@ +"""MPI workflow for thumbnail storage.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class StorageService(Service): + """Store thumbnail metadata.""" + + async def step(self): + """Handle the next thumbnail metadata payload.""" + await self.transform_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def transform_request(self, _sender_id, body): + """Persist the thumbnail metadata and publish its storage URI.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "NotificationService", { + "request_type": "notify_upload", + "image_id": body["image_id"], + "uri": f"s3://thumbs/{body['image_id']}.jpg", + } diff --git a/eclypse/builders/application/thumbnailer/mpi_services/transform.py b/eclypse/builders/application/thumbnailer/mpi_services/transform.py new file mode 100644 index 0000000..7270ff9 --- /dev/null +++ b/eclypse/builders/application/thumbnailer/mpi_services/transform.py @@ -0,0 +1,27 @@ +"""MPI workflow for thumbnail creation.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class TransformService(Service): + """Create thumbnail metadata from an image.""" + + async def step(self): + """Handle the next image uploaded to the thumbnail pipeline.""" + await self.upload_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def upload_request(self, _sender_id, body): + """Create thumbnail metadata for the uploaded image.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "StorageService", { + "request_type": "store_thumbnail", + "image_id": body["image_id"], + "thumbnail": { + "width": 320, + "height": 180, + "format": "jpeg", + }, + } diff --git a/eclypse/builders/application/thumbnailer/mpi_services/upload.py b/eclypse/builders/application/thumbnailer/mpi_services/upload.py new file mode 100644 index 0000000..95b47c9 --- /dev/null +++ b/eclypse/builders/application/thumbnailer/mpi_services/upload.py @@ -0,0 +1,31 @@ +"""MPI workflow for image upload.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class UploadService(Service): + """Start the thumbnailing pipeline.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the uploader with a rolling image counter.""" + super().__init__(service_id, store_step=store_step) + self.image_id = 0 + + async def step(self): + """Upload the next image and wait for the final storage response.""" + self.image_id += 1 + await self.upload_image() + response = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=response)) + return response + + @mpi.exchange(send=True) + def upload_image(self): + """Send a synthetic image payload to the transform service.""" + return "TransformService", { + "request_type": "create_thumbnail", + "image_id": f"img-{self.image_id}", + "resolution": [1920, 1080], + } diff --git a/eclypse/builders/application/thumbnailer/rest_services/__init__.py b/eclypse/builders/application/thumbnailer/rest_services/__init__.py new file mode 100644 index 0000000..850d584 --- /dev/null +++ b/eclypse/builders/application/thumbnailer/rest_services/__init__.py @@ -0,0 +1,13 @@ +"""REST implementation for thumbnailer services.""" + +from .notification import NotificationService +from .storage import StorageService +from .transform import TransformService +from .upload import UploadService + +__all__ = [ + "NotificationService", + "StorageService", + "TransformService", + "UploadService", +] diff --git a/eclypse/builders/application/thumbnailer/rest_services/notification.py b/eclypse/builders/application/thumbnailer/rest_services/notification.py new file mode 100644 index 0000000..2218080 --- /dev/null +++ b/eclypse/builders/application/thumbnailer/rest_services/notification.py @@ -0,0 +1,21 @@ +"""REST endpoints for upload notification.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class NotificationService(RESTService): + """Return the final thumbnail location.""" + + @rest.endpoint("/notify", "POST") + def notify(self, image_id: str, uri: str, **_): + """Return the final storage location for the generated thumbnail.""" + self.logger.info( + "Received request | " + format_log_kv(image_id=image_id, uri=uri) + ) + return 200, { + "image_id": image_id, + "uri": uri, + "status": "stored", + } diff --git a/eclypse/builders/application/thumbnailer/rest_services/storage.py b/eclypse/builders/application/thumbnailer/rest_services/storage.py new file mode 100644 index 0000000..1b7fcbc --- /dev/null +++ b/eclypse/builders/application/thumbnailer/rest_services/storage.py @@ -0,0 +1,22 @@ +"""REST endpoints for thumbnail storage.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class StorageService(RESTService): + """Store thumbnail metadata.""" + + @rest.endpoint("/store", "POST") + def store(self, image_id: str, thumbnail: dict, **_): + """Persist thumbnail metadata and report its storage URI.""" + self.logger.info( + "Received request | " + + format_log_kv(image_id=image_id, thumbnail=thumbnail) + ) + return 200, { + "image_id": image_id, + "thumbnail": thumbnail, + "uri": f"s3://thumbs/{image_id}.jpg", + } diff --git a/eclypse/builders/application/thumbnailer/rest_services/transform.py b/eclypse/builders/application/thumbnailer/rest_services/transform.py new file mode 100644 index 0000000..bc51866 --- /dev/null +++ b/eclypse/builders/application/thumbnailer/rest_services/transform.py @@ -0,0 +1,25 @@ +"""REST endpoints for thumbnail creation.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class TransformService(RESTService): + """Create thumbnail metadata from an image.""" + + @rest.endpoint("/thumbnail", "POST") + def thumbnail(self, image_id: str, resolution: list[int], **_): + """Build thumbnail metadata for the uploaded image.""" + self.logger.info( + "Received request | " + + format_log_kv(image_id=image_id, resolution=resolution) + ) + return 200, { + "image_id": image_id, + "thumbnail": { + "width": resolution[0] // 6, + "height": resolution[1] // 6, + "format": "jpeg", + }, + } diff --git a/eclypse/builders/application/thumbnailer/rest_services/upload.py b/eclypse/builders/application/thumbnailer/rest_services/upload.py new file mode 100644 index 0000000..4fe5441 --- /dev/null +++ b/eclypse/builders/application/thumbnailer/rest_services/upload.py @@ -0,0 +1,49 @@ +"""REST workflow for image upload.""" + +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class UploadService(Service): + """Start the thumbnailing pipeline.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the uploader with a rolling image counter.""" + super().__init__( + service_id, + communication_interface="rest", + store_step=store_step, + ) + self.image_id = 0 + + async def step(self): + """Drive one image through the REST thumbnailing workflow.""" + self.image_id += 1 + transform_r = await self.rest.post( + "TransformService/thumbnail", + image_id=f"img-{self.image_id}", + resolution=[1920, 1080], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="TransformService", body=transform_r.body) + ) + storage_r = await self.rest.post( + "StorageService/store", + image_id=f"img-{self.image_id}", + thumbnail=transform_r.body["thumbnail"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="StorageService", body=storage_r.body) + ) + notification_r = await self.rest.post( + "NotificationService/notify", + image_id=f"img-{self.image_id}", + uri=storage_r.body["uri"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="NotificationService", body=notification_r.body) + ) + return notification_r diff --git a/eclypse/builders/application/video_analytics_serving/__init__.py b/eclypse/builders/application/video_analytics_serving/__init__.py new file mode 100644 index 0000000..ae9a919 --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/__init__.py @@ -0,0 +1,11 @@ +"""The video analytics serving application models a containerised pipeline. + +Frames are captured, analysed, tracked, and aggregated before returning a +result to the originating camera gateway. It captures a common edge AI +deployment pattern in which low-latency processing is split across multiple +services. + +Source: + `Intel Edge Video Analytics Microservice + `_ +""" diff --git a/eclypse/builders/application/video_analytics_serving/application.py b/eclypse/builders/application/video_analytics_serving/application.py new file mode 100644 index 0000000..d37d3d8 --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/application.py @@ -0,0 +1,172 @@ +"""Factory for a video analytics serving application.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Literal, +) + +from eclypse.builders.application._helpers import ( + build_application_from_specs, +) + +if TYPE_CHECKING: + from eclypse.graph import Application + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + CommunicationInterface, + InitPolicy, + UpdatePolicies, + ) + + +def get_video_analytics_serving( + application_id: str = "VideoAnalyticsServing", + communication_interface: CommunicationInterface | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + edge_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + requirement_init: InitPolicy = "min", + flows: Literal["default"] | list[list[str]] = "default", + store_step: bool = False, + seed: int | None = None, +) -> Application: + """Get the video analytics serving application. + + Args: + application_id (str): Identifier assigned to the generated application. + communication_interface (CommunicationInterface | None): + Communication backend used to instantiate executable services. When + ``None``, the builder returns a graph-only application. + update_policies (Callable | list[Callable] | None): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Optional assets attached to application nodes. + edge_assets (dict[str, Asset] | None): + Optional assets attached to application edges. + include_default_assets (bool): + Whether default graph assets should be included in the application. + requirement_init (InitPolicy): + Initialisation strategy applied to node and edge requirements. + flows (Literal["default"] | list[list[str]]): + User-defined application flows. Use ``"default"`` to install the + benchmark's built-in request paths. + store_step (bool): + Whether instantiated services should store their step outputs in + the internal step queue. Ignored when + ``communication_interface`` is ``None``. + seed (int | None): + Seed forwarded to the application random generator. + + Returns: + Application: The configured video analytics serving application. + + Raises: + ValueError: If ``communication_interface`` is not supported. + """ + default_flows = [ + [ + "CameraGatewayService", + "DetectionService", + "TrackingService", + "AnalyticsService", + "CameraGatewayService", + ], + [ + "CameraGatewayService", + "DetectionService", + "AnalyticsService", + "CameraGatewayService", + ], + ] + service_names = [ + "AnalyticsService", + "CameraGatewayService", + "DetectionService", + "TrackingService", + ] + node_requirements = { + "CameraGatewayService": { + "cpu": 1, + "gpu": 0, + "ram": 1.0, + "storage": 0.5, + "availability": 0.95, + "processing_time": 8, + }, + "DetectionService": { + "cpu": 3, + "gpu": 1, + "ram": 4.0, + "storage": 2.0, + "availability": 0.92, + "processing_time": 18, + }, + "TrackingService": { + "cpu": 2, + "gpu": 1, + "ram": 3.0, + "storage": 1.0, + "availability": 0.93, + "processing_time": 14, + }, + "AnalyticsService": { + "cpu": 2, + "gpu": 0.5, + "ram": 2.0, + "storage": 1.0, + "availability": 0.94, + "processing_time": 10, + }, + } + edge_requirements = [ + ( + "CameraGatewayService", + "DetectionService", + {"symmetric": True, "latency": 15, "bandwidth": 25}, + ), + ( + "DetectionService", + "TrackingService", + {"symmetric": True, "latency": 20, "bandwidth": 20}, + ), + ( + "CameraGatewayService", + "TrackingService", + {"symmetric": True, "latency": 18, "bandwidth": 18}, + ), + ( + "TrackingService", + "AnalyticsService", + {"symmetric": True, "latency": 10, "bandwidth": 15}, + ), + ( + "DetectionService", + "AnalyticsService", + {"symmetric": True, "latency": 18, "bandwidth": 10}, + ), + ( + "AnalyticsService", + "CameraGatewayService", + {"symmetric": True, "latency": 12, "bandwidth": 8}, + ), + ] + return build_application_from_specs( + application_id=application_id, + communication_interface=communication_interface, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=edge_assets, + include_default_assets=include_default_assets, + requirement_init=requirement_init, + flows=flows, + store_step=store_step, + default_flows=default_flows, + service_names=service_names, + node_requirements=node_requirements, + edge_requirements=edge_requirements, + seed=seed, + package_name=__package__, + ) diff --git a/eclypse/builders/application/video_analytics_serving/mpi_services/__init__.py b/eclypse/builders/application/video_analytics_serving/mpi_services/__init__.py new file mode 100644 index 0000000..8d9b45b --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/mpi_services/__init__.py @@ -0,0 +1,13 @@ +"""MPI implementation for video analytics serving services.""" + +from .analytics import AnalyticsService +from .camera_gateway import CameraGatewayService +from .detection import DetectionService +from .tracking import TrackingService + +__all__ = [ + "AnalyticsService", + "CameraGatewayService", + "DetectionService", + "TrackingService", +] diff --git a/eclypse/builders/application/video_analytics_serving/mpi_services/analytics.py b/eclypse/builders/application/video_analytics_serving/mpi_services/analytics.py new file mode 100644 index 0000000..5f89c9c --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/mpi_services/analytics.py @@ -0,0 +1,26 @@ +"""MPI workflow for the analytics service.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class AnalyticsService(Service): + """Aggregate tracked events into a compact result.""" + + async def step(self): + """Handle the next tracking result emitted by the pipeline.""" + await self.tracking_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def tracking_request(self, _sender_id, body): + """Summarise tracked objects for the camera gateway.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + labels = [track["label"] for track in body["tracks"]] + return "CameraGatewayService", { + "response_type": "analytics_result", + "frame_id": body["frame_id"], + "stream_id": body["stream_id"], + "object_count": len(body["tracks"]), + "summary": ", ".join(labels), + } diff --git a/eclypse/builders/application/video_analytics_serving/mpi_services/camera_gateway.py b/eclypse/builders/application/video_analytics_serving/mpi_services/camera_gateway.py new file mode 100644 index 0000000..a2c591a --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/mpi_services/camera_gateway.py @@ -0,0 +1,32 @@ +"""MPI workflow for the camera gateway service.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class CameraGatewayService(Service): + """Entry-point service that starts the video analytics pipeline.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the gateway with a rolling frame counter.""" + super().__init__(service_id, store_step=store_step) + self.frame_id = 0 + + async def step(self): + """Capture the next frame and wait for the analytics summary.""" + self.frame_id += 1 + await self.start_pipeline() + response = await self.mpi.recv() + self.logger.info("Received response | " + format_log_kv(response=response)) + return response + + @mpi.exchange(send=True) + def start_pipeline(self): + """Send a synthetic frame to the detection service.""" + return "DetectionService", { + "request_type": "analyse_frame", + "frame_id": self.frame_id, + "stream_id": "camera-a", + "objects": ["person", "forklift"], + } diff --git a/eclypse/builders/application/video_analytics_serving/mpi_services/detection.py b/eclypse/builders/application/video_analytics_serving/mpi_services/detection.py new file mode 100644 index 0000000..fe800a4 --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/mpi_services/detection.py @@ -0,0 +1,24 @@ +"""MPI workflow for the detection service.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class DetectionService(Service): + """Detect objects in incoming frames.""" + + async def step(self): + """Handle the next frame produced by the camera gateway.""" + await self.gateway_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def gateway_request(self, _sender_id, body): + """Convert the incoming frame payload into detections.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + return "TrackingService", { + "request_type": "track_objects", + "frame_id": body["frame_id"], + "stream_id": body["stream_id"], + "detections": body["objects"], + } diff --git a/eclypse/builders/application/video_analytics_serving/mpi_services/tracking.py b/eclypse/builders/application/video_analytics_serving/mpi_services/tracking.py new file mode 100644 index 0000000..f0b9f29 --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/mpi_services/tracking.py @@ -0,0 +1,28 @@ +"""MPI workflow for the tracking service.""" + +from eclypse.remote.communication import mpi +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class TrackingService(Service): + """Track detected objects across frames.""" + + async def step(self): + """Handle the next detection payload produced by the detector.""" + await self.detection_request() # pylint: disable=no-value-for-parameter + + @mpi.exchange(receive=True, send=True) + def detection_request(self, _sender_id, body): + """Assign synthetic track identifiers to each detected object.""" + self.logger.info("Received request | " + format_log_kv(request=body)) + tracks = [ + {"label": label, "track_id": index + 1} + for index, label in enumerate(body["detections"]) + ] + return "AnalyticsService", { + "request_type": "aggregate_events", + "frame_id": body["frame_id"], + "stream_id": body["stream_id"], + "tracks": tracks, + } diff --git a/eclypse/builders/application/video_analytics_serving/rest_services/__init__.py b/eclypse/builders/application/video_analytics_serving/rest_services/__init__.py new file mode 100644 index 0000000..d997382 --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/rest_services/__init__.py @@ -0,0 +1,13 @@ +"""REST implementation for video analytics serving services.""" + +from .analytics import AnalyticsService +from .camera_gateway import CameraGatewayService +from .detection import DetectionService +from .tracking import TrackingService + +__all__ = [ + "AnalyticsService", + "CameraGatewayService", + "DetectionService", + "TrackingService", +] diff --git a/eclypse/builders/application/video_analytics_serving/rest_services/analytics.py b/eclypse/builders/application/video_analytics_serving/rest_services/analytics.py new file mode 100644 index 0000000..25e9307 --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/rest_services/analytics.py @@ -0,0 +1,24 @@ +"""REST endpoints for the analytics service.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class AnalyticsService(RESTService): + """Aggregate tracked events into a compact result.""" + + @rest.endpoint("/analyse", "POST") + def analyse(self, frame_id: int, stream_id: str, tracks: list[dict], **_): + """Summarise tracked objects for the requested frame.""" + self.logger.info( + "Received request | " + + format_log_kv(frame_id=frame_id, stream_id=stream_id, tracks=tracks) + ) + labels = [track["label"] for track in tracks] + return 200, { + "frame_id": frame_id, + "stream_id": stream_id, + "object_count": len(tracks), + "summary": ", ".join(labels), + } diff --git a/eclypse/builders/application/video_analytics_serving/rest_services/camera_gateway.py b/eclypse/builders/application/video_analytics_serving/rest_services/camera_gateway.py new file mode 100644 index 0000000..b6a11ad --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/rest_services/camera_gateway.py @@ -0,0 +1,52 @@ +"""REST workflow for the camera gateway service.""" + +from eclypse.remote.service import Service +from eclypse.utils import format_log_kv + + +class CameraGatewayService(Service): + """Entry-point service that starts the video analytics pipeline.""" + + def __init__(self, service_id: str, store_step: bool = False): + """Initialise the gateway with a rolling frame counter.""" + super().__init__( + service_id, + communication_interface="rest", + store_step=store_step, + ) + self.frame_id = 0 + + async def step(self): + """Drive one frame through the REST analytics pipeline.""" + self.frame_id += 1 + detection_r = await self.rest.post( + "DetectionService/detect", + frame_id=self.frame_id, + stream_id="camera-a", + objects=["person", "forklift"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="DetectionService", body=detection_r.body) + ) + tracking_r = await self.rest.post( + "TrackingService/track", + frame_id=self.frame_id, + stream_id="camera-a", + detections=detection_r.body["detections"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="TrackingService", body=tracking_r.body) + ) + analytics_r = await self.rest.post( + "AnalyticsService/analyse", + frame_id=self.frame_id, + stream_id="camera-a", + tracks=tracking_r.body["tracks"], + ) + self.logger.info( + "Received response | " + + format_log_kv(source="AnalyticsService", body=analytics_r.body) + ) + return analytics_r diff --git a/eclypse/builders/application/video_analytics_serving/rest_services/detection.py b/eclypse/builders/application/video_analytics_serving/rest_services/detection.py new file mode 100644 index 0000000..9daa569 --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/rest_services/detection.py @@ -0,0 +1,22 @@ +"""REST endpoints for the detection service.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class DetectionService(RESTService): + """Detect objects in incoming frames.""" + + @rest.endpoint("/detect", "POST") + def detect(self, frame_id: int, stream_id: str, objects: list[str], **_): + """Return the objects detected in the incoming frame.""" + self.logger.info( + "Received request | " + + format_log_kv(frame_id=frame_id, stream_id=stream_id, objects=objects) + ) + return 200, { + "frame_id": frame_id, + "stream_id": stream_id, + "detections": objects, + } diff --git a/eclypse/builders/application/video_analytics_serving/rest_services/tracking.py b/eclypse/builders/application/video_analytics_serving/rest_services/tracking.py new file mode 100644 index 0000000..1c10225 --- /dev/null +++ b/eclypse/builders/application/video_analytics_serving/rest_services/tracking.py @@ -0,0 +1,29 @@ +"""REST endpoints for the tracking service.""" + +from eclypse.remote.communication import rest +from eclypse.remote.service import RESTService +from eclypse.utils import format_log_kv + + +class TrackingService(RESTService): + """Track detected objects across frames.""" + + @rest.endpoint("/track", "POST") + def track(self, frame_id: int, stream_id: str, detections: list[str], **_): + """Assign synthetic track identifiers to each detected object.""" + self.logger.info( + "Received request | " + + format_log_kv( + frame_id=frame_id, + stream_id=stream_id, + detections=detections, + ) + ) + return 200, { + "frame_id": frame_id, + "stream_id": stream_id, + "tracks": [ + {"label": label, "track_id": index + 1} + for index, label in enumerate(detections) + ], + } diff --git a/eclypse/builders/infrastructure/__init__.py b/eclypse/builders/infrastructure/__init__.py index a2f7079..ed3eb07 100644 --- a/eclypse/builders/infrastructure/__init__.py +++ b/eclypse/builders/infrastructure/__init__.py @@ -1,19 +1,65 @@ -"""Infrastructure builders.""" +"""Infrastructure builders (e.g. get_hierarchical, get_mec_5g, get_orion_cev). +The package groups the off-the-shelf infrastructure builders provided by +ECLYPSE. It combines generic topology generators, architecture-shaped +deployment patterns, and named reference topologies derived from literature or +datasets, while re-exporting them from a single public entrypoint. +""" + +from . import ( + generators, + patterns, + references, +) from .generators import ( - b_cube, - fat_tree, - hierarchical, - random, - star, + get_b_cube, + get_fat_tree, + get_hierarchical, + get_random, + get_scale_free, + get_small_world, + get_star, +) +from .patterns import ( + get_continuum_tiered, + get_factory_cells, + get_industrial_tsn, + get_mec_5g, + get_multi_region_wan, + get_vehicular_edge, +) +from .references import get_orion_cev +from .references.topohub import ( + get_backbone, + get_caida, + get_gabriel, + get_sndlib, + get_topohub, + get_topology_zoo, ) -from .orion_cev import get_orion_cev __all__ = [ - "b_cube", - "fat_tree", + "generators", + "get_b_cube", + "get_backbone", + "get_caida", + "get_continuum_tiered", + "get_factory_cells", + "get_fat_tree", + "get_gabriel", + "get_hierarchical", + "get_industrial_tsn", + "get_mec_5g", + "get_multi_region_wan", "get_orion_cev", - "hierarchical", - "random", - "star", + "get_random", + "get_scale_free", + "get_small_world", + "get_sndlib", + "get_star", + "get_topohub", + "get_topology_zoo", + "get_vehicular_edge", + "patterns", + "references", ] diff --git a/eclypse/builders/infrastructure/_helpers.py b/eclypse/builders/infrastructure/_helpers.py new file mode 100644 index 0000000..00011c0 --- /dev/null +++ b/eclypse/builders/infrastructure/_helpers.py @@ -0,0 +1,215 @@ +"""Helper functions shared by infrastructure builders.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import networkx as nx + +from eclypse.builders._helpers import prune_assets + +if TYPE_CHECKING: + from typing import Any + + from eclypse.graph import Infrastructure + + +def add_nodes( + infrastructure: Infrastructure, + node_ids: list[str], + strict: bool = False, + **assets: Any, +) -> None: + """Add a batch of nodes with the same asset values. + + Args: + infrastructure (Infrastructure): + Infrastructure receiving the nodes. + node_ids (list[str]): + Node identifiers to add. + strict (bool): + Whether inconsistent asset values should raise. + **assets: + Concrete node asset values applied to every node. + """ + for node_id in node_ids: + infrastructure.add_node(node_id, strict=strict, **assets) + + +def connect_pairs( + infrastructure: Infrastructure, + pairs: list[tuple[str, str]], + symmetric: bool = False, + strict: bool = False, + **assets: Any, +) -> None: + """Add links described as explicit source-target pairs. + + Args: + infrastructure (Infrastructure): + Infrastructure receiving the links. + pairs (list[tuple[str, str]]): + Ordered source-target pairs to connect. + symmetric (bool): + Whether to mirror each link in the opposite direction. + strict (bool): + Whether inconsistent asset values should raise. + **assets: + Concrete edge asset values applied to every link. + """ + for source, target in pairs: + infrastructure.add_edge( + source, + target, + symmetric=symmetric, + strict=strict, + **assets, + ) + + +def connect_round_robin( + infrastructure: Infrastructure, + sources: list[str], + targets: list[str], + symmetric: bool = False, + strict: bool = False, + **assets: Any, +) -> None: + """Connect each source to a target chosen in round-robin order. + + Args: + infrastructure (Infrastructure): + Infrastructure receiving the links. + sources (list[str]): + Source node identifiers. + targets (list[str]): + Target node identifiers. + symmetric (bool): + Whether to mirror each link in the opposite direction. + strict (bool): + Whether inconsistent asset values should raise. + **assets: + Concrete edge asset values applied to every link. + + Raises: + ValueError: If ``targets`` is empty. + """ + if not targets: + raise ValueError("At least one target node is required.") + + for index, source in enumerate(sources): + infrastructure.add_edge( + source, + targets[index % len(targets)], + symmetric=symmetric, + strict=strict, + **assets, + ) + + +def connect_clique( + infrastructure: Infrastructure, + node_ids: list[str], + symmetric: bool = True, + strict: bool = False, + **assets: Any, +) -> None: + """Connect every distinct pair of nodes in the provided group. + + Args: + infrastructure (Infrastructure): + Infrastructure receiving the links. + node_ids (list[str]): + Node identifiers to connect as a clique. + symmetric (bool): + Whether to mirror each link in the opposite direction. + strict (bool): + Whether inconsistent asset values should raise. + **assets: + Concrete edge asset values applied to every link. + """ + pairs = [ + (node_ids[i], node_ids[j]) + for i in range(len(node_ids)) + for j in range(i + 1, len(node_ids)) + ] + connect_pairs( + infrastructure, + pairs, + symmetric=symmetric, + strict=strict, + **assets, + ) + + +def relabel_hierarchical_levels( + infrastructure: Infrastructure, + level_prefixes: list[str], +) -> Infrastructure: + """Rename ``hierarchical`` level nodes using semantic tier prefixes. + + Args: + infrastructure (Infrastructure): + Infrastructure returned by the hierarchical generator. + level_prefixes (list[str]): + Semantic prefix for each hierarchy level in order. + + Returns: + Infrastructure: The relabelled infrastructure. + """ + mapping: dict[str, str] = {} + for node_id in list(infrastructure.nodes): + level, index = node_id.split("_", maxsplit=1) + prefix = level_prefixes[int(level[1:])] + mapping[node_id] = f"{prefix}_{index}" + + nx.relabel_nodes(infrastructure, mapping, copy=False) + infrastructure._invalidate_cache() # pylint: disable=protected-access + return infrastructure + + +def tier_node_assets( + infrastructure: Infrastructure, + **asset_values: Any, +) -> dict[str, Any]: + """Return only the node assets supported by the infrastructure. + + Args: + infrastructure (Infrastructure): + Infrastructure whose node asset bucket is inspected. + **asset_values: + Candidate asset values. + + Returns: + dict[str, Any]: Asset values supported by the infrastructure. + """ + return prune_assets(infrastructure.node_assets, **asset_values) + + +def tier_link_assets( + infrastructure: Infrastructure, + **asset_values: Any, +) -> dict[str, Any]: + """Return only the edge assets supported by the infrastructure. + + Args: + infrastructure (Infrastructure): + Infrastructure whose edge asset bucket is inspected. + **asset_values: + Candidate asset values. + + Returns: + dict[str, Any]: Asset values supported by the infrastructure. + """ + return prune_assets(infrastructure.edge_assets, **asset_values) + + +__all__ = [ + "add_nodes", + "connect_clique", + "connect_pairs", + "connect_round_robin", + "relabel_hierarchical_levels", + "tier_link_assets", + "tier_node_assets", +] diff --git a/eclypse/builders/infrastructure/generators/__init__.py b/eclypse/builders/infrastructure/generators/__init__.py index 1922d5a..edbe9cc 100644 --- a/eclypse/builders/infrastructure/generators/__init__.py +++ b/eclypse/builders/infrastructure/generators/__init__.py @@ -1,24 +1,25 @@ -"""Module for the infrastructure builders. +"""Infrastructure generators (e.g. get_star, get_hierarchical, get_small_world). -It has the following builders: - -- b_cube: A BCube infrastructure with switches and hosts. -- fat_tree: A Fat-Tree infrastructure with switches and hosts. -- hierarchical: A hierarchical infrastructure made of nodes partitioned into groups. -- star: A star infrastructure with clients connected to a central node. -- random: A random infrastructure with nodes connected with a given probability. +The package collects topology-first infrastructure builders whose primary role +is to generate reusable graph families. These generators expose structural +models such as stars, random graphs, layered hierarchies, and data-centre or +hub-oriented networks without tying them to a specific application domain. """ -from .b_cube import b_cube -from .fat_tree import fat_tree -from .hierarchical import hierarchical -from .random import random -from .star import star +from .b_cube import get_b_cube +from .fat_tree import get_fat_tree +from .hierarchical import get_hierarchical +from .random import get_random +from .scale_free import get_scale_free +from .small_world import get_small_world +from .star import get_star __all__ = [ - "b_cube", - "fat_tree", - "hierarchical", - "random", - "star", + "get_b_cube", + "get_fat_tree", + "get_hierarchical", + "get_random", + "get_scale_free", + "get_small_world", + "get_star", ] diff --git a/eclypse/builders/infrastructure/generators/b_cube.py b/eclypse/builders/infrastructure/generators/b_cube.py index ffe3d72..e105d49 100644 --- a/eclypse/builders/infrastructure/generators/b_cube.py +++ b/eclypse/builders/infrastructure/generators/b_cube.py @@ -34,14 +34,13 @@ import networkx as nx from eclypse.graph.assets import Asset - from eclypse.placement.strategies import PlacementStrategy from eclypse.utils.types import ( InitPolicy, UpdatePolicies, ) -def b_cube( +def get_b_cube( k: int, n: int, infrastructure_id: str = "b_cube", @@ -52,7 +51,6 @@ def b_cube( strict: bool = False, resource_init: InitPolicy = "max", path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, - placement_strategy: PlacementStrategy | None = None, seed: int | None = None, ) -> Infrastructure: """Factory for generating a BCube(k, n) topology. @@ -79,7 +77,6 @@ def b_cube( Defaults to "max". path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): \ Algorithm to compute paths. Defaults to None. - placement_strategy (PlacementStrategy | None): Strategy for resource placement.\ Defaults to None. seed (int | None): Seed for random number generation. Defaults to None. @@ -94,7 +91,6 @@ def b_cube( include_default_assets=include_default_assets, resource_init=resource_init, path_algorithm=path_algorithm, - placement_strategy=placement_strategy, seed=seed, ) diff --git a/eclypse/builders/infrastructure/generators/fat_tree.py b/eclypse/builders/infrastructure/generators/fat_tree.py index c24ca4e..82b4897 100644 --- a/eclypse/builders/infrastructure/generators/fat_tree.py +++ b/eclypse/builders/infrastructure/generators/fat_tree.py @@ -27,14 +27,13 @@ import networkx as nx from eclypse.graph.assets import Asset - from eclypse.placement.strategies import PlacementStrategy from eclypse.utils.types import ( InitPolicy, UpdatePolicies, ) -def fat_tree( +def get_fat_tree( k: int, infrastructure_id: str = "fat_tree", update_policies: UpdatePolicies = None, @@ -44,7 +43,6 @@ def fat_tree( strict: bool = False, resource_init: InitPolicy = "max", path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, - placement_strategy: PlacementStrategy | None = None, seed: int | None = None, ) -> Infrastructure: """Factory for generating a Fat-Tree network topology. @@ -72,7 +70,6 @@ def fat_tree( Defaults to "max". path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): \ Algorithm to compute paths. Defaults to None. - placement_strategy (PlacementStrategy | None): Strategy for resource placement.\ Defaults to None. seed (int | None): Seed for random number generation. Defaults to None. @@ -90,7 +87,6 @@ def fat_tree( include_default_assets=include_default_assets, resource_init=resource_init, path_algorithm=path_algorithm, - placement_strategy=placement_strategy, seed=seed, ) num_pods = k diff --git a/eclypse/builders/infrastructure/generators/hierarchical.py b/eclypse/builders/infrastructure/generators/hierarchical.py index 1d37ca9..c8f6be8 100644 --- a/eclypse/builders/infrastructure/generators/hierarchical.py +++ b/eclypse/builders/infrastructure/generators/hierarchical.py @@ -31,7 +31,6 @@ from networkx import nx from eclypse.graph.assets import Asset - from eclypse.placement.strategies import PlacementStrategy from eclypse.utils.types import ( ConnectivityFn, InitPolicy, @@ -41,7 +40,7 @@ DEFAULT_NODE_PARTITIONING = [0.35, 0.3, 0.2, 0.15] -def hierarchical( +def get_hierarchical( n: int, infrastructure_id: str = "hierarchical", symmetric: bool = False, @@ -54,7 +53,6 @@ def hierarchical( include_default_assets: bool = False, strict: bool = False, resource_init: InitPolicy = "max", - placement_strategy: PlacementStrategy | None = None, path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, seed: int | None = None, ): @@ -90,9 +88,6 @@ def hierarchical( consistent with their spaces. Defaults to False. resource_init (InitPolicy): The initialization policy for the resources. Defaults to "min". - placement_strategy (PlacementStrategy | None): - The placement strategy for the infrastructure. Defaults to - None. path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): The algorithm to compute the paths between nodes. Defaults to None. @@ -140,7 +135,6 @@ def hierarchical( include_default_assets=include_default_assets, resource_init=resource_init, path_algorithm=path_algorithm, - placement_strategy=placement_strategy, seed=seed, ) diff --git a/eclypse/builders/infrastructure/generators/random.py b/eclypse/builders/infrastructure/generators/random.py index b2008f0..2617b8d 100644 --- a/eclypse/builders/infrastructure/generators/random.py +++ b/eclypse/builders/infrastructure/generators/random.py @@ -25,14 +25,13 @@ from collections.abc import Callable from eclypse.graph.assets import Asset - from eclypse.placement.strategies import PlacementStrategy from eclypse.utils.types import ( InitPolicy, UpdatePolicies, ) -def random( +def get_random( n: int, infrastructure_id: str = "random", p: float = 0.5, @@ -44,7 +43,6 @@ def random( strict: bool = False, resource_init: InitPolicy = "min", path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, - placement_strategy: PlacementStrategy | None = None, seed: int | None = None, ): """Create a random infrastructure with `n` nodes and a connection probability `p`. @@ -72,8 +70,6 @@ def random( path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): The algorithm to compute the paths between nodes. Defaults to None. - placement_strategy (PlacementStrategy | None): - The strategy to place the resources. Defaults to None. seed (int | None): The seed for the random number generator. Defaults to None. Returns: @@ -87,7 +83,6 @@ def random( include_default_assets=include_default_assets, resource_init=resource_init, path_algorithm=path_algorithm, - placement_strategy=placement_strategy, seed=seed, ) diff --git a/eclypse/builders/infrastructure/generators/scale_free.py b/eclypse/builders/infrastructure/generators/scale_free.py new file mode 100644 index 0000000..53921a3 --- /dev/null +++ b/eclypse/builders/infrastructure/generators/scale_free.py @@ -0,0 +1,101 @@ +"""Scale-free infrastructure generator. + +This module provides a Barabasi-Albert style topology generator for infrastructure +graphs dominated by a small number of highly connected hubs. New nodes attach +preferentially to already well-connected nodes, creating a network with a few +backbone-like hubs and many low-degree peripheral nodes. + +This is useful for infrastructures where QoS depends on hub capacity and +resilience: most flows traverse a limited set of critical nodes, so the topology +is well suited to studying congestion, bottlenecks, and the impact of hub +failures on latency and bandwidth. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import networkx as nx + +from eclypse.graph import Infrastructure + +if TYPE_CHECKING: + from collections.abc import Callable + + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_scale_free( + n: int, + m: int, + infrastructure_id: str = "scale_free", + symmetric: bool = False, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + strict: bool = False, + resource_init: InitPolicy = "min", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create a scale-free infrastructure using the Barabasi-Albert model. + + Args: + n (int): + Number of nodes in the generated topology. + m (int): + Number of edges attached from each new node to existing nodes. + infrastructure_id (str): + Identifier assigned to the infrastructure. + symmetric (bool): + Whether generated links should be mirrored. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + strict (bool): + Whether inconsistent asset values should raise. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the random graph model. + + Returns: + Infrastructure: The generated scale-free infrastructure. + """ + infrastructure = Infrastructure( + infrastructure_id=infrastructure_id, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + for index in range(n): + infrastructure.add_node(f"n{index}", strict=strict) + + graph = nx.barabasi_albert_graph(n=n, m=m, seed=seed) + node_ids = list(infrastructure.nodes) + for source, target in graph.edges: + infrastructure.add_edge( + node_ids[source], + node_ids[target], + symmetric=symmetric, + strict=strict, + ) + + return infrastructure diff --git a/eclypse/builders/infrastructure/generators/small_world.py b/eclypse/builders/infrastructure/generators/small_world.py new file mode 100644 index 0000000..3e55894 --- /dev/null +++ b/eclypse/builders/infrastructure/generators/small_world.py @@ -0,0 +1,105 @@ +"""Small-world infrastructure generator. + +This module provides a Watts-Strogatz style topology generator for infrastructure +graphs with strong local clustering and a small number of long-range shortcuts. +Each node starts in a ring-like neighbourhood, which preserves short-hop local +connectivity, and then a fraction of links is rewired to introduce longer +shortcuts across the graph. + +The resulting topology is useful when modelling peer infrastructures with soft +QoS expectations: nearby nodes can communicate through short local paths, while +the added shortcuts reduce the average end-to-end distance for latency-sensitive +traffic without imposing a rigid hierarchy or a single backbone. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import networkx as nx + +from eclypse.graph import Infrastructure + +if TYPE_CHECKING: + from collections.abc import Callable + + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_small_world( + n: int, + k: int, + p: float, + infrastructure_id: str = "small_world", + symmetric: bool = False, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + strict: bool = False, + resource_init: InitPolicy = "min", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create a small-world infrastructure using the Watts-Strogatz model. + + Args: + n (int): + Number of nodes in the generated topology. + k (int): + Number of nearest neighbours joined to each node before rewiring. + p (float): + Rewiring probability for each edge in the ring lattice. + infrastructure_id (str): + Identifier assigned to the infrastructure. + symmetric (bool): + Whether generated links should be mirrored. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + strict (bool): + Whether inconsistent asset values should raise. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the random graph model. + + Returns: + Infrastructure: The generated small-world infrastructure. + """ + infrastructure = Infrastructure( + infrastructure_id=infrastructure_id, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + for index in range(n): + infrastructure.add_node(f"n{index}", strict=strict) + + graph = nx.watts_strogatz_graph(n=n, k=k, p=p, seed=seed) + node_ids = list(infrastructure.nodes) + for source, target in graph.edges: + infrastructure.add_edge( + node_ids[source], + node_ids[target], + symmetric=symmetric, + strict=strict, + ) + + return infrastructure diff --git a/eclypse/builders/infrastructure/generators/star.py b/eclypse/builders/infrastructure/generators/star.py index b98e542..85abaf7 100644 --- a/eclypse/builders/infrastructure/generators/star.py +++ b/eclypse/builders/infrastructure/generators/star.py @@ -25,14 +25,13 @@ import networkx as nx from eclypse.graph.assets import Asset - from eclypse.placement.strategies import PlacementStrategy from eclypse.utils.types import ( InitPolicy, UpdatePolicies, ) -def star( +def get_star( n_clients: int, infrastructure_id: str = "star", symmetric: bool = False, @@ -45,7 +44,6 @@ def star( strict: bool = False, resource_init: InitPolicy = "min", path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, - placement_strategy: PlacementStrategy | None = None, seed: int | None = None, ): """Create a star infrastructure with `n_clients` clients around a central node. @@ -75,9 +73,6 @@ def star( path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): The algorithm to compute the paths between nodes. Defaults to None. - placement_strategy (PlacementStrategy | None): - The placement strategy for the infrastructure. Defaults to - None. seed (int | None): The seed for the random number generator. Defaults to None. Returns: @@ -91,11 +86,10 @@ def star( include_default_assets=include_default_assets, resource_init=resource_init, path_algorithm=path_algorithm, - placement_strategy=placement_strategy, seed=seed, ) - _outer_assets_values = {} if outer_assets_values is None else outer_assets_values - _center_assets_values = {} if center_assets_values is None else center_assets_values + _outer_assets_values = outer_assets_values or {} + _center_assets_values = center_assets_values or {} for i in range(n_clients): infrastructure.add_node(f"outer_{i}", strict=strict, **_outer_assets_values) infrastructure.add_node("center", strict=strict, **_center_assets_values) diff --git a/eclypse/builders/infrastructure/patterns/__init__.py b/eclypse/builders/infrastructure/patterns/__init__.py new file mode 100644 index 0000000..a670d17 --- /dev/null +++ b/eclypse/builders/infrastructure/patterns/__init__.py @@ -0,0 +1,23 @@ +"""Infrastructure patterns (e.g. get_continuum_tiered, get_mec_5g). + +The package groups parameterised infrastructure blueprints whose structure is +tied to an architectural deployment model rather than to a pure graph family. +These builders encode recognisable system layouts such as cloud-edge continua, +MEC deployments, industrial cells, and vehicular edge backbones. +""" + +from .continuum_tiered import get_continuum_tiered +from .factory_cells import get_factory_cells +from .industrial_tsn import get_industrial_tsn +from .mec_5g import get_mec_5g +from .multi_region_wan import get_multi_region_wan +from .vehicular_edge import get_vehicular_edge + +__all__ = [ + "get_continuum_tiered", + "get_factory_cells", + "get_industrial_tsn", + "get_mec_5g", + "get_multi_region_wan", + "get_vehicular_edge", +] diff --git a/eclypse/builders/infrastructure/patterns/continuum_tiered.py b/eclypse/builders/infrastructure/patterns/continuum_tiered.py new file mode 100644 index 0000000..913977f --- /dev/null +++ b/eclypse/builders/infrastructure/patterns/continuum_tiered.py @@ -0,0 +1,189 @@ +"""Continuum-tiered infrastructure pattern. + +The continuum-tiered pattern models a device-edge-fog-cloud deployment by adapting the +generic hierarchical generator with tier-aware naming and resource defaults. It +proposes up to four semantic layers: device, edge, fog, and cloud. Nodes are +grouped by tier and connected primarily across adjacent layers, while the +intra-tier connectivity grows progressively richer from device to cloud. + +The pattern is designed for continuum QoS studies where latency and capacity are +not uniform across the stack: devices are close to the data source but resource +poor, edge and fog tiers progressively improve compute and availability, and the +cloud tier offers the highest capacity with the loosest proximity guarantees. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.builders.infrastructure._helpers import ( + relabel_hierarchical_levels, + tier_node_assets, +) +from eclypse.builders.infrastructure.generators.hierarchical import get_hierarchical + +if TYPE_CHECKING: + from collections.abc import Callable + + import networkx as nx + + from eclypse.graph import Infrastructure + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_continuum_tiered( + device_count: int, + edge_count: int, + fog_count: int = 0, + cloud_count: int = 1, + infrastructure_id: str = "continuum_tiered", + symmetric: bool = True, + connectivity: list[float] | None = None, + cross_level_connectivity: list[float] | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + strict: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create an IoT-edge-cloud continuum from the hierarchical generator. + + Args: + device_count (int): + Number of device-tier nodes. + edge_count (int): + Number of edge-tier nodes. + fog_count (int): + Number of fog-tier nodes. + cloud_count (int): + Number of cloud-tier nodes. + infrastructure_id (str): + Identifier assigned to the infrastructure. + symmetric (bool): + Whether generated links should be mirrored. + connectivity (list[float] | None): + Cross-tier connectivity probabilities passed to ``get_hierarchical``. + cross_level_connectivity (list[float] | None): + Intra-tier connectivity probabilities passed to ``get_hierarchical``. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + strict (bool): + Whether inconsistent asset values should raise. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the underlying hierarchical generator. + + Returns: + Infrastructure: The generated continuum infrastructure. + + Raises: + ValueError: If any tier count is negative or the total number of nodes is zero. + """ + counts = { + "device": device_count, + "edge": edge_count, + "fog": fog_count, + "cloud": cloud_count, + } + if any(count < 0 for count in counts.values()): + raise ValueError("Tier counts must be non-negative.") + + non_empty_tiers = [(name, count) for name, count in counts.items() if count > 0] + total_nodes = sum(count for _, count in non_empty_tiers) + if total_nodes == 0: + raise ValueError("At least one tier must contain nodes.") + + if connectivity is None: + connectivity = [1.0] * max(len(non_empty_tiers) - 1, 0) + if cross_level_connectivity is None: + cross_level_connectivity = [ + ( + 0.0 + if name == "device" + else 0.25 + if name == "edge" + else 0.5 + if name == "fog" + else 1.0 + ) + for name, _ in non_empty_tiers + ] + + infrastructure = get_hierarchical( + n=total_nodes, + infrastructure_id=infrastructure_id, + symmetric=symmetric, + node_partitioning=[count / total_nodes for _, count in non_empty_tiers], + connectivity=connectivity, + cross_level_connectivity=cross_level_connectivity, + update_policies=update_policies, + node_assets=node_assets, + link_assets=link_assets, + include_default_assets=include_default_assets, + strict=strict, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + relabel_hierarchical_levels( + infrastructure, + [name for name, _ in non_empty_tiers], + ) + + tier_profiles = { + "device": dict( + cpu=1.0, + gpu=0.0, + ram=1.0, + storage=1.0, + availability=0.95, + processing_time=8.0, + ), + "edge": dict( + cpu=8.0, + gpu=1.0, + ram=16.0, + storage=64.0, + availability=0.98, + processing_time=3.0, + ), + "fog": dict( + cpu=16.0, + gpu=2.0, + ram=32.0, + storage=256.0, + availability=0.99, + processing_time=2.0, + ), + "cloud": dict( + cpu=32.0, + gpu=4.0, + ram=128.0, + storage=1024.0, + availability=0.999, + processing_time=1.0, + ), + } + for tier_name, _ in non_empty_tiers: + profile = tier_node_assets(infrastructure, **tier_profiles[tier_name]) + for node_id in infrastructure.nodes: + if node_id.startswith(f"{tier_name}_"): + infrastructure.nodes[node_id].update(profile) + + return infrastructure diff --git a/eclypse/builders/infrastructure/patterns/factory_cells.py b/eclypse/builders/infrastructure/patterns/factory_cells.py new file mode 100644 index 0000000..bfc3812 --- /dev/null +++ b/eclypse/builders/infrastructure/patterns/factory_cells.py @@ -0,0 +1,222 @@ +"""Factory-cells infrastructure pattern. + +The factory-cells pattern models repeated production cells connected to plant-edge +compute, suitable for industrial monitoring and assembly workloads. Each cell is +organised around a local controller with its machines and sensors, while one or +more plant-edge nodes aggregate traffic across cells and optionally uplink to a +cloud tier. + +The pattern combines two QoS domains: short, high-quality local links inside a +cell for operational traffic, and slower uplinks from cells to plant-edge or +cloud resources for coordination, analytics, or historical storage. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.builders.infrastructure._helpers import ( + add_nodes, + connect_clique, + connect_round_robin, + tier_link_assets, + tier_node_assets, +) +from eclypse.graph import Infrastructure + +if TYPE_CHECKING: + from collections.abc import Callable + + import networkx as nx + + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_factory_cells( + cell_count: int, + machines_per_cell: int, + sensors_per_cell: int, + plant_edge_count: int = 1, + cloud_count: int = 1, + infrastructure_id: str = "factory_cells", + symmetric: bool = True, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + strict: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create a smart-factory topology organised into repeated production cells. + + Args: + cell_count (int): + Number of production cells. + machines_per_cell (int): + Number of machine nodes per cell. + sensors_per_cell (int): + Number of sensor nodes per cell. + plant_edge_count (int): + Number of plant-edge nodes shared across cells. + cloud_count (int): + Number of cloud nodes attached to the plant edge. + infrastructure_id (str): + Identifier assigned to the infrastructure. + symmetric (bool): + Whether generated links should be mirrored. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + strict (bool): + Whether inconsistent asset values should raise. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. + + Returns: + Infrastructure: The generated factory-cells infrastructure. + + Raises: + ValueError: If no production cell is requested. + """ + if cell_count <= 0: + raise ValueError("The factory-cells pattern requires at least one cell.") + + infrastructure = Infrastructure( + infrastructure_id=infrastructure_id, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + plant_edges = [f"plant_edge_{index}" for index in range(plant_edge_count)] + clouds = [f"cloud_{index}" for index in range(cloud_count)] + add_nodes( + infrastructure, + plant_edges, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=24.0, + gpu=2.0, + ram=64.0, + storage=512.0, + availability=0.995, + processing_time=2.0, + ), + ) + add_nodes( + infrastructure, + clouds, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=32.0, + gpu=4.0, + ram=128.0, + storage=1024.0, + availability=0.999, + processing_time=1.0, + ), + ) + + cell_link = tier_link_assets(infrastructure, latency=1.0, bandwidth=1000.0) + uplink = tier_link_assets(infrastructure, latency=8.0, bandwidth=500.0) + for cell_index in range(cell_count): + controller = [f"cell_{cell_index}_controller"] + machines = [ + f"cell_{cell_index}_machine_{index}" for index in range(machines_per_cell) + ] + sensors = [ + f"cell_{cell_index}_sensor_{index}" for index in range(sensors_per_cell) + ] + add_nodes( + infrastructure, + controller, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=8.0, + ram=8.0, + storage=32.0, + availability=0.99, + processing_time=3.0, + ), + ) + add_nodes( + infrastructure, + machines, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=4.0, + ram=4.0, + storage=16.0, + availability=0.98, + processing_time=4.0, + ), + ) + add_nodes( + infrastructure, + sensors, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=1.0, + ram=1.0, + storage=2.0, + availability=0.97, + processing_time=6.0, + ), + ) + connect_round_robin( + infrastructure, + machines + sensors, + controller, + symmetric=symmetric, + strict=strict, + **cell_link, + ) + connect_round_robin( + infrastructure, + controller, + plant_edges, + symmetric=symmetric, + strict=strict, + **uplink, + ) + + connect_clique( + infrastructure, + plant_edges, + symmetric=symmetric, + strict=strict, + **tier_link_assets(infrastructure, latency=2.0, bandwidth=2000.0), + ) + connect_round_robin( + infrastructure, + plant_edges, + clouds, + symmetric=symmetric, + strict=strict, + **tier_link_assets(infrastructure, latency=15.0, bandwidth=1000.0), + ) + return infrastructure diff --git a/eclypse/builders/infrastructure/patterns/industrial_tsn.py b/eclypse/builders/infrastructure/patterns/industrial_tsn.py new file mode 100644 index 0000000..51a20e7 --- /dev/null +++ b/eclypse/builders/infrastructure/patterns/industrial_tsn.py @@ -0,0 +1,200 @@ +"""Industrial TSN infrastructure pattern. + +The industrial TSN pattern models a deterministic switched LAN for controllers, +field devices, and local edge compute in industrial automation settings. It +proposes a switching fabric at the centre, with endpoints, controllers, and +plant-edge compute attached to the same deterministic network. + +The featured QoS assumption is strict and predictable service quality: TSN links +represent low-latency, high-bandwidth paths intended for bounded-delay control +traffic, while the switching fabric provides deterministic connectivity between +control and production nodes instead of best-effort routing. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.builders.infrastructure._helpers import ( + add_nodes, + connect_clique, + connect_round_robin, + tier_link_assets, + tier_node_assets, +) +from eclypse.graph import Infrastructure + +if TYPE_CHECKING: + from collections.abc import Callable + + import networkx as nx + + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_industrial_tsn( + endpoint_count: int, + switch_count: int = 2, + controller_count: int = 2, + edge_count: int = 1, + infrastructure_id: str = "industrial_tsn", + symmetric: bool = True, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + strict: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create an industrial TSN LAN with switches, controllers, and endpoints. + + Args: + endpoint_count (int): + Number of field endpoints connected to the TSN network. + switch_count (int): + Number of industrial switches. + controller_count (int): + Number of control nodes. + edge_count (int): + Number of edge-compute nodes on the plant LAN. + infrastructure_id (str): + Identifier assigned to the infrastructure. + symmetric (bool): + Whether generated links should be mirrored. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + strict (bool): + Whether inconsistent asset values should raise. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. + + Returns: + Infrastructure: The generated industrial TSN infrastructure. + + Raises: + ValueError: If the topology misses its switching fabric. + """ + if switch_count <= 0: + raise ValueError("The industrial TSN pattern requires at least one switch.") + + infrastructure = Infrastructure( + infrastructure_id=infrastructure_id, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + endpoints = [f"endpoint_{index}" for index in range(endpoint_count)] + switches = [f"switch_{index}" for index in range(switch_count)] + controllers = [f"controller_{index}" for index in range(controller_count)] + edges = [f"edge_{index}" for index in range(edge_count)] + + add_nodes( + infrastructure, + endpoints, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=1.0, + ram=1.0, + storage=2.0, + availability=0.97, + processing_time=6.0, + ), + ) + add_nodes( + infrastructure, + switches, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=2.0, + ram=2.0, + storage=8.0, + availability=0.999, + processing_time=1.0, + ), + ) + add_nodes( + infrastructure, + controllers, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=8.0, + ram=8.0, + storage=32.0, + availability=0.995, + processing_time=2.0, + ), + ) + add_nodes( + infrastructure, + edges, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=16.0, + gpu=1.0, + ram=32.0, + storage=128.0, + availability=0.995, + processing_time=2.0, + ), + ) + + tsn_link = tier_link_assets(infrastructure, latency=0.5, bandwidth=1000.0) + backbone_link = tier_link_assets(infrastructure, latency=1.0, bandwidth=2000.0) + + connect_round_robin( + infrastructure, + endpoints, + switches, + symmetric=symmetric, + strict=strict, + **tsn_link, + ) + connect_round_robin( + infrastructure, + controllers, + switches, + symmetric=symmetric, + strict=strict, + **backbone_link, + ) + connect_round_robin( + infrastructure, + edges, + switches, + symmetric=symmetric, + strict=strict, + **backbone_link, + ) + connect_clique( + infrastructure, + switches, + symmetric=symmetric, + strict=strict, + **backbone_link, + ) + return infrastructure diff --git a/eclypse/builders/infrastructure/patterns/mec_5g.py b/eclypse/builders/infrastructure/patterns/mec_5g.py new file mode 100644 index 0000000..85df92c --- /dev/null +++ b/eclypse/builders/infrastructure/patterns/mec_5g.py @@ -0,0 +1,206 @@ +"""MEC 5G infrastructure pattern. + +The MEC 5G pattern models user equipment attached to radio sites, each backed by +edge compute close to the access network and connected onward to a cloud tier. +The proposed layers are user equipment, radio access nodes, MEC hosts, and +cloud nodes. Vehicles or mobile users attach to the radio tier, the radio tier +uplinks to nearby MEC compute, and MEC nodes connect to a more distant cloud. + +The built-in QoS profile reflects this structure: access links favour low +latency and moderate bandwidth, backhaul links are faster and more stable, and +the cloud uplink trades proximity for larger aggregate capacity. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.builders.infrastructure._helpers import ( + add_nodes, + connect_clique, + connect_round_robin, + tier_link_assets, + tier_node_assets, +) +from eclypse.graph import Infrastructure + +if TYPE_CHECKING: + from collections.abc import Callable + + import networkx as nx + + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_mec_5g( + user_count: int, + ran_count: int, + mec_count: int | None = None, + cloud_count: int = 1, + infrastructure_id: str = "mec_5g", + symmetric: bool = True, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + strict: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create a 5G edge infrastructure with radio access, MEC, and cloud tiers. + + Args: + user_count (int): + Number of user-equipment nodes. + ran_count (int): + Number of radio access sites. + mec_count (int | None): + Number of MEC hosts. Defaults to ``ran_count``. + cloud_count (int): + Number of cloud nodes. + infrastructure_id (str): + Identifier assigned to the infrastructure. + symmetric (bool): + Whether generated links should be mirrored. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + strict (bool): + Whether inconsistent asset values should raise. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. + + Returns: + Infrastructure: The generated MEC 5G infrastructure. + + Raises: + ValueError: If the required radio access or MEC tiers are missing. + """ + mec_total = ran_count if mec_count is None else mec_count + if ran_count <= 0: + raise ValueError("The MEC 5G pattern requires at least one RAN site.") + if mec_total <= 0: + raise ValueError("The MEC 5G pattern requires at least one MEC host.") + + infrastructure = Infrastructure( + infrastructure_id=infrastructure_id, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + users = [f"user_{index}" for index in range(user_count)] + rans = [f"ran_{index}" for index in range(ran_count)] + mecs = [f"mec_{index}" for index in range(mec_total)] + clouds = [f"cloud_{index}" for index in range(cloud_count)] + + add_nodes( + infrastructure, + users, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=1.0, + ram=1.0, + storage=1.0, + availability=0.95, + processing_time=8.0, + ), + ) + add_nodes( + infrastructure, + rans, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=2.0, + ram=4.0, + storage=8.0, + availability=0.98, + processing_time=4.0, + ), + ) + add_nodes( + infrastructure, + mecs, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=16.0, + gpu=1.0, + ram=32.0, + storage=256.0, + availability=0.99, + processing_time=2.0, + ), + ) + add_nodes( + infrastructure, + clouds, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=32.0, + gpu=4.0, + ram=128.0, + storage=1024.0, + availability=0.999, + processing_time=1.0, + ), + ) + + access_link = tier_link_assets(infrastructure, latency=5.0, bandwidth=200.0) + backhaul_link = tier_link_assets(infrastructure, latency=2.0, bandwidth=1000.0) + wan_link = tier_link_assets(infrastructure, latency=20.0, bandwidth=500.0) + + connect_round_robin( + infrastructure, + users, + rans, + symmetric=symmetric, + strict=strict, + **access_link, + ) + connect_round_robin( + infrastructure, + rans, + mecs, + symmetric=symmetric, + strict=strict, + **backhaul_link, + ) + connect_round_robin( + infrastructure, + mecs, + clouds, + symmetric=symmetric, + strict=strict, + **wan_link, + ) + connect_clique( + infrastructure, + clouds, + symmetric=symmetric, + strict=strict, + **tier_link_assets(infrastructure, latency=5.0, bandwidth=2000.0), + ) + + return infrastructure diff --git a/eclypse/builders/infrastructure/patterns/multi_region_wan.py b/eclypse/builders/infrastructure/patterns/multi_region_wan.py new file mode 100644 index 0000000..c9ae930 --- /dev/null +++ b/eclypse/builders/infrastructure/patterns/multi_region_wan.py @@ -0,0 +1,152 @@ +"""Multi-region WAN infrastructure pattern. + +The multi-region WAN pattern models several compute regions connected by a slower +backbone, with each region containing local compute nodes behind a regional +gateway. The architecture proposes two layers inside each region, a gateway and +its attached regional nodes, and then a WAN backbone between gateways. + +This captures the QoS asymmetry of geo-distributed deployments: intra-region +links are relatively fast and high-bandwidth, while inter-region communication +is costlier in latency and capacity, making the pattern useful for placement and +replication studies across distant sites. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.builders.infrastructure._helpers import ( + add_nodes, + connect_clique, + connect_round_robin, + tier_link_assets, + tier_node_assets, +) +from eclypse.graph import Infrastructure + +if TYPE_CHECKING: + from collections.abc import Callable + + import networkx as nx + + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_multi_region_wan( + region_count: int, + nodes_per_region: int, + infrastructure_id: str = "multi_region_wan", + symmetric: bool = True, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + strict: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create a multi-region WAN with per-region gateways and local compute nodes. + + Args: + region_count (int): + Number of regions in the topology. + nodes_per_region (int): + Number of compute nodes attached to each region. + infrastructure_id (str): + Identifier assigned to the infrastructure. + symmetric (bool): + Whether generated links should be mirrored. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + strict (bool): + Whether inconsistent asset values should raise. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. + + Returns: + Infrastructure: The generated multi-region WAN. + + Raises: + ValueError: If no region is requested. + """ + if region_count <= 0: + raise ValueError("The multi-region WAN requires at least one region.") + + infrastructure = Infrastructure( + infrastructure_id=infrastructure_id, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + gateways = [f"region_{region}_gateway" for region in range(region_count)] + add_nodes( + infrastructure, + gateways, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=8.0, + ram=16.0, + storage=64.0, + availability=0.995, + processing_time=2.0, + ), + ) + + local_link = tier_link_assets(infrastructure, latency=2.0, bandwidth=1000.0) + wan_link = tier_link_assets(infrastructure, latency=35.0, bandwidth=300.0) + for region in range(region_count): + region_nodes = [ + f"region_{region}_node_{index}" for index in range(nodes_per_region) + ] + add_nodes( + infrastructure, + region_nodes, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=16.0, + gpu=1.0, + ram=32.0, + storage=256.0, + availability=0.99, + processing_time=3.0, + ), + ) + connect_round_robin( + infrastructure, + region_nodes, + [gateways[region]], + symmetric=symmetric, + strict=strict, + **local_link, + ) + + connect_clique( + infrastructure, + gateways, + symmetric=symmetric, + strict=strict, + **wan_link, + ) + return infrastructure diff --git a/eclypse/builders/infrastructure/patterns/vehicular_edge.py b/eclypse/builders/infrastructure/patterns/vehicular_edge.py new file mode 100644 index 0000000..dd10d6f --- /dev/null +++ b/eclypse/builders/infrastructure/patterns/vehicular_edge.py @@ -0,0 +1,201 @@ +"""Vehicular-edge infrastructure pattern. + +The vehicular-edge pattern models vehicles attached to roadside units, backed by MEC +hosts and an optional cloud tier. The proposed layers are vehicles, roadside +units, MEC hosts, and cloud nodes. Vehicles attach to nearby roadside units, +roadside units forward traffic to edge compute, and MEC nodes connect onward to +the cloud for less latency-sensitive processing. + +Its QoS profile reflects connected-mobility assumptions: access links favour low +latency but limited bandwidth, roadside and MEC links provide stronger local +capacity, and the cloud uplink captures the longer-latency control or archival +path. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.builders.infrastructure._helpers import ( + add_nodes, + connect_clique, + connect_round_robin, + tier_link_assets, + tier_node_assets, +) +from eclypse.graph import Infrastructure + +if TYPE_CHECKING: + from collections.abc import Callable + + import networkx as nx + + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_vehicular_edge( + vehicle_count: int, + rsu_count: int, + mec_count: int = 1, + cloud_count: int = 1, + infrastructure_id: str = "vehicular_edge", + symmetric: bool = True, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + strict: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create a vehicular-edge topology with RSUs, MEC, and cloud tiers. + + Args: + vehicle_count (int): + Number of vehicle nodes. + rsu_count (int): + Number of roadside units. + mec_count (int): + Number of MEC hosts serving the roadside tier. + cloud_count (int): + Number of cloud nodes attached to MEC. + infrastructure_id (str): + Identifier assigned to the infrastructure. + symmetric (bool): + Whether generated links should be mirrored. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + strict (bool): + Whether inconsistent asset values should raise. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. + + Returns: + Infrastructure: The generated vehicular-edge infrastructure. + + Raises: + ValueError: If the topology misses roadside or MEC nodes. + """ + if rsu_count <= 0: + raise ValueError("The vehicular-edge pattern requires at least one RSU.") + if mec_count <= 0: + raise ValueError("The vehicular-edge pattern requires at least one MEC host.") + + infrastructure = Infrastructure( + infrastructure_id=infrastructure_id, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + vehicles = [f"vehicle_{index}" for index in range(vehicle_count)] + rsus = [f"rsu_{index}" for index in range(rsu_count)] + mecs = [f"mec_{index}" for index in range(mec_count)] + clouds = [f"cloud_{index}" for index in range(cloud_count)] + + add_nodes( + infrastructure, + vehicles, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=2.0, + ram=2.0, + storage=4.0, + availability=0.94, + processing_time=7.0, + ), + ) + add_nodes( + infrastructure, + rsus, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=4.0, + ram=8.0, + storage=16.0, + availability=0.98, + processing_time=4.0, + ), + ) + add_nodes( + infrastructure, + mecs, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=16.0, + gpu=1.0, + ram=32.0, + storage=128.0, + availability=0.99, + processing_time=2.0, + ), + ) + add_nodes( + infrastructure, + clouds, + strict=strict, + **tier_node_assets( + infrastructure, + cpu=32.0, + gpu=4.0, + ram=128.0, + storage=1024.0, + availability=0.999, + processing_time=1.0, + ), + ) + + connect_round_robin( + infrastructure, + vehicles, + rsus, + symmetric=symmetric, + strict=strict, + **tier_link_assets(infrastructure, latency=3.0, bandwidth=200.0), + ) + connect_round_robin( + infrastructure, + rsus, + mecs, + symmetric=symmetric, + strict=strict, + **tier_link_assets(infrastructure, latency=5.0, bandwidth=500.0), + ) + connect_round_robin( + infrastructure, + mecs, + clouds, + symmetric=symmetric, + strict=strict, + **tier_link_assets(infrastructure, latency=20.0, bandwidth=500.0), + ) + connect_clique( + infrastructure, + rsus, + symmetric=symmetric, + strict=strict, + **tier_link_assets(infrastructure, latency=8.0, bandwidth=250.0), + ) + return infrastructure diff --git a/eclypse/builders/infrastructure/references/__init__.py b/eclypse/builders/infrastructure/references/__init__.py new file mode 100644 index 0000000..61f49b8 --- /dev/null +++ b/eclypse/builders/infrastructure/references/__init__.py @@ -0,0 +1,15 @@ +"""Infrastructure references (e.g. get_orion_cev, topohub). + +The package groups concrete named topologies reconstructed from papers, +datasets, or standards-oriented reference material. It includes both specific +reference builders and dataset-backed loader families, such as the TopoHub +subpackage for SNDlib, CAIDA, Gabriel, backbone, and Topology Zoo references. +""" + +from .orion_cev import get_orion_cev +from . import topohub + +__all__ = [ + "get_orion_cev", + "topohub", +] diff --git a/eclypse/builders/infrastructure/orion_cev.py b/eclypse/builders/infrastructure/references/orion_cev.py similarity index 74% rename from eclypse/builders/infrastructure/orion_cev.py rename to eclypse/builders/infrastructure/references/orion_cev.py index 2b62f04..12dc19a 100644 --- a/eclypse/builders/infrastructure/orion_cev.py +++ b/eclypse/builders/infrastructure/references/orion_cev.py @@ -1,22 +1,26 @@ -"""Factory for the Orion CEV infrastructure topology. - -Defines the Orion Crew Exploration Vehicle (CEV) network as an -Infrastructure object, including switches and end systems such as -sensors, controllers, and processing units. Links and node resources -(CPU, RAM, availability, etc.) are assigned based on realistic values -for mixed-criticality embedded platforms. - -The topology and resource model are inspired by: -Berisa et al., "AVB-aware Routing and Scheduling for Critical Traffic in -Time-sensitive Networks with Preemption", RTNS 2022, -https://dl.acm.org/doi/10.1145/3534879.3534926. +"""Reference topology for the Orion Crew Exploration Vehicle network. + +The Orion Crew Exploration Vehicle (CEV) reference models a published mixed-criticality +embedded network with end systems and switching nodes. The topology includes +multiple layers of avionics switches, distributed units, controllers, and +mission subsystems, all connected through a fixed embedded communication +backbone. + +Its featured capabilities are those of a structured mixed-criticality network: +deterministic switch-centric connectivity, explicit separation between end +systems and network switches, and link/node resources suitable for analysing QoS +constraints such as latency, bandwidth, and availability in a safety-oriented +setting. + +Source: + Berisa et al., "AVB-aware Routing and Scheduling for Critical Traffic in + Time-sensitive Networks with Preemption", RTNS 2022, + https://dl.acm.org/doi/10.1145/3534879.3534926 """ from __future__ import annotations -from typing import ( - TYPE_CHECKING, -) +from typing import TYPE_CHECKING from eclypse.builders._helpers import prune_assets from eclypse.graph import Infrastructure @@ -27,7 +31,6 @@ import networkx as nx from eclypse.graph.assets import Asset - from eclypse.placement.strategies import PlacementStrategy from eclypse.utils.types import ( InitPolicy, UpdatePolicies, @@ -42,30 +45,27 @@ def get_orion_cev( include_default_assets: bool = False, resource_init: InitPolicy = "max", path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, - placement_strategy: PlacementStrategy | None = None, seed: int | None = None, ) -> Infrastructure: - """Create the Orion CEV infrastructure. + """Create the Orion CEV reference infrastructure. Args: infrastructure_id (str): - The ID of the infrastructure. Defaults to "OrionCEV". - update_policies (Callable | list[Callable] | None): - Graph update policies. Defaults to None. + Identifier assigned to the infrastructure. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. node_assets (dict[str, Asset] | None): - The assets for the nodes. Defaults to None. + Node asset definitions available to the infrastructure. link_assets (dict[str, Asset] | None): - The assets for the links. Defaults to None. + Edge asset definitions available to the infrastructure. include_default_assets (bool): - Whether to include the default assets. Defaults to False. + Whether to include default ECLYPSE assets. resource_init (InitPolicy): - The initialization policy for the resources. Defaults to "max". + Initialisation policy used for graph assets. path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): - The algorithm to compute the paths between nodes. Defaults to - None. - placement_strategy (PlacementStrategy | None): - The strategy to place the resources. Defaults to None. - seed (int | None): The seed for the random number generator. Defaults to None. + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. Returns: Infrastructure: The Orion CEV infrastructure. @@ -78,7 +78,6 @@ def get_orion_cev( include_default_assets=include_default_assets, resource_init=resource_init, path_algorithm=path_algorithm, - placement_strategy=placement_strategy, seed=seed, ) @@ -115,7 +114,6 @@ def get_orion_cev( "SM2CA", "SM2CB", ] - network_switches = [ "NS11", "NS12", @@ -147,7 +145,6 @@ def get_orion_cev( processing_time=10, ), ) - for ns in network_switches: infra.add_node( ns, @@ -219,7 +216,6 @@ def get_orion_cev( ("NS8", "NS51"), ("NS8", "NS52"), ] - for source, target in edges: infra.add_edge( source, @@ -227,5 +223,4 @@ def get_orion_cev( symmetric=True, **prune_assets(infra.edge_assets, latency=10, bandwidth=100), ) - return infra diff --git a/eclypse/builders/infrastructure/references/topohub/__init__.py b/eclypse/builders/infrastructure/references/topohub/__init__.py new file mode 100644 index 0000000..f096054 --- /dev/null +++ b/eclypse/builders/infrastructure/references/topohub/__init__.py @@ -0,0 +1,23 @@ +"""TopoHub references (e.g. get_topohub, get_sndlib, get_topology_zoo). + +The package groups dataset-backed infrastructure references loaded through the +``topohub`` Python wrapper. It includes a generic ``get_topohub`` entrypoint +for arbitrary TopoHub paths together with family-specific helpers for SNDlib, +the Internet Topology Zoo, CAIDA, synthetic backbones, and Gabriel graphs. +""" + +from ._helpers import get_topohub +from .backbone import get_backbone +from .caida import get_caida +from .gabriel import get_gabriel +from .sndlib import get_sndlib +from .topology_zoo import get_topology_zoo + +__all__ = [ + "get_backbone", + "get_caida", + "get_gabriel", + "get_sndlib", + "get_topohub", + "get_topology_zoo", +] diff --git a/eclypse/builders/infrastructure/references/topohub/_helpers.py b/eclypse/builders/infrastructure/references/topohub/_helpers.py new file mode 100644 index 0000000..4587656 --- /dev/null +++ b/eclypse/builders/infrastructure/references/topohub/_helpers.py @@ -0,0 +1,205 @@ +"""Helper functions shared by TopoHub-backed infrastructure references.""" + +from __future__ import annotations + +from importlib import import_module +from typing import ( + TYPE_CHECKING, + Any, +) + +import networkx as nx + +from eclypse.builders._helpers import prune_assets +from eclypse.graph import Infrastructure +from eclypse.simulation.config import _require_module + +if TYPE_CHECKING: + from collections.abc import Callable + + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_topohub( + topology: str, + use_names: bool = False, + infrastructure_id: str | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create an infrastructure from any TopoHub topology path. + + The ``topology`` value must be a valid TopoHub dataset path. Available paths + can be inspected from the TopoHub repository catalogue and documentation. + + Args: + topology (str): + Full TopoHub topology path, such as ``"sndlib/polska"``, + ``"topozoo/Abilene"``, or ``"gabriel/25/0"``. + use_names (bool): + Whether TopoHub should use node names as node identifiers. + infrastructure_id (str | None): + Identifier assigned to the infrastructure. If omitted, a dataset-based + identifier is derived from the topology path. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. + + Returns: + Infrastructure: The converted TopoHub infrastructure. + """ + _require_module("topohub") + + topohub = import_module("topohub") + + topo = topohub.get(topology, use_names=use_names) + graph = nx.node_link_graph(topo, edges="edges") + default_id = topology.replace("/", "_").replace("-", "_") + infrastructure = Infrastructure( + infrastructure_id=infrastructure_id or default_id, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + _copy_graph_metadata(infrastructure, graph, topology) + + node_mapping = { + node_id: _normalise_node_id(node_id, data, use_names=use_names) + for node_id, data in graph.nodes(data=True) + } + + for source_id, attrs in graph.nodes(data=True): + node_id = node_mapping[source_id] + metadata = _node_metadata(attrs, use_names=use_names) + metadata.setdefault("topohub_id", source_id) + infrastructure.add_node( + node_id, + strict=False, + **metadata, + ) + + for source_id, target_id, attrs in graph.edges(data=True): + source = node_mapping[source_id] + target = node_mapping[target_id] + edge_attrs = _edge_attributes(infrastructure, attrs) + edge_attrs.update( + {key: value for key, value in attrs.items() if key not in edge_attrs} + ) + infrastructure.add_edge( + source, + target, + symmetric=True, + strict=False, + **edge_attrs, + ) + + return infrastructure + + +def _copy_graph_metadata( + infrastructure: Infrastructure, + graph: nx.Graph, + dataset_path: str, +) -> None: + """Copy graph-level metadata from TopoHub into the infrastructure.""" + infrastructure.graph.update(graph.graph) + infrastructure.graph["dataset_path"] = dataset_path + + +def _normalise_node_id( + node_id: Any, + attrs: dict[str, Any], + use_names: bool, +) -> str: + """Return a stable string node identifier for ECLYPSE.""" + if use_names and isinstance(attrs.get("name"), str) and attrs["name"]: + return attrs["name"] + if isinstance(node_id, str): + return node_id + if isinstance(attrs.get("name"), str) and attrs["name"] and not use_names: + return f"n{node_id}" + return f"n{node_id}" + + +def _node_metadata( + attrs: dict[str, Any], + use_names: bool, +) -> dict[str, Any]: + """Return node metadata according to the selected identifier policy.""" + metadata = dict(attrs) + if use_names: + metadata.pop("name", None) + return metadata + + +def _edge_attributes( + infrastructure: Infrastructure, + attrs: dict[str, Any], +) -> dict[str, Any]: + """Map TopoHub edge metadata into ECLYPSE edge assets.""" + edge_assets: dict[str, Any] = {} + + latency = next( + ( + float(attrs[key]) + for key in ("latency", "delay") + if isinstance(attrs.get(key), (int, float)) + ), + None, + ) + if latency is None and isinstance(attrs.get("dist"), (int, float)): + latency = float(attrs["dist"]) / 200.0 + + if latency is not None: + edge_assets.update( + prune_assets( + infrastructure.edge_assets, + latency=latency, + ) + ) + + for key in ( + "bandwidth", + "capacity", + "preinstalled_capacity", + "preinstalled_cap", + ): + value = attrs.get(key) + if isinstance(value, (int, float)): + edge_assets.update( + prune_assets( + infrastructure.edge_assets, + bandwidth=float(value), + ) + ) + break + + return edge_assets + + +__all__ = ["get_topohub"] diff --git a/eclypse/builders/infrastructure/references/topohub/backbone.py b/eclypse/builders/infrastructure/references/topohub/backbone.py new file mode 100644 index 0000000..0c2d3d4 --- /dev/null +++ b/eclypse/builders/infrastructure/references/topohub/backbone.py @@ -0,0 +1,91 @@ +"""Synthetic backbone reference infrastructures. + +The backbone family models synthetic long-haul backbones distributed through +TopoHub. These topologies emphasise WAN structure, geographic spread, and link +distance rather than node compute heterogeneity, making them a good reference +for routing, latency, and placement experiments over region-scale networks. + +Example: + .. code-block:: python + + get_backbone("africa", **kwargs) + +Source: + TopoHub, https://www.topohub.org +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ._helpers import get_topohub + +if TYPE_CHECKING: + from collections.abc import Callable + + import networkx as nx + + from eclypse.graph import Infrastructure + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_backbone( + topology: str, + infrastructure_id: str | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create a synthetic backbone infrastructure from TopoHub. + + The ``topology`` value must be a valid topology name from + `TopoHub `_'s ``backbone`` family catalogue. + + Args: + topology (str): + Backbone topology identifier, such as ``"africa"``. + infrastructure_id (str | None): + Identifier assigned to the infrastructure. If omitted, a dataset-based + identifier is used. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. + + Returns: + Infrastructure: The converted backbone infrastructure. + """ + dataset_path = f"backbone/{topology}" + return get_topohub( + topology=dataset_path, + infrastructure_id=infrastructure_id or f"backbone_{topology}", + use_names=False, + update_policies=update_policies, + node_assets=node_assets, + link_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + +__all__ = ["get_backbone"] diff --git a/eclypse/builders/infrastructure/references/topohub/caida.py b/eclypse/builders/infrastructure/references/topohub/caida.py new file mode 100644 index 0000000..b4a26f3 --- /dev/null +++ b/eclypse/builders/infrastructure/references/topohub/caida.py @@ -0,0 +1,92 @@ +"""CAIDA-backed reference infrastructures. + +The CAIDA family models Internet-scale AS connectivity snapshots processed by +TopoHub from CAIDA Ark data. These references focus on large-scale structural +properties and peering relationships rather than node compute capabilities, so +ECLYPSE keeps node resources implicit and relies on the infrastructure +initialisation policy for unspecified assets. + +Example: + .. code-block:: python + + get_caida("2024-01", **kwargs) + +Source: + CAIDA Ark, https://www.caida.org/catalog/datasets/ark/ +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ._helpers import get_topohub + +if TYPE_CHECKING: + from collections.abc import Callable + + import networkx as nx + + from eclypse.graph import Infrastructure + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_caida( + snapshot: str, + infrastructure_id: str | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create a CAIDA-backed infrastructure from TopoHub. + + The ``snapshot`` value must be a valid topology name from + `TopoHub `_'s ``caida`` family catalogue. + + Args: + snapshot (str): + CAIDA snapshot identifier within TopoHub. + infrastructure_id (str | None): + Identifier assigned to the infrastructure. If omitted, a dataset-based + identifier is used. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. + + Returns: + Infrastructure: The converted CAIDA infrastructure. + """ + dataset_path = f"caida/{snapshot}" + return get_topohub( + topology=dataset_path, + infrastructure_id=infrastructure_id or f"caida_{snapshot}", + use_names=False, + update_policies=update_policies, + node_assets=node_assets, + link_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + +__all__ = ["get_caida"] diff --git a/eclypse/builders/infrastructure/references/topohub/gabriel.py b/eclypse/builders/infrastructure/references/topohub/gabriel.py new file mode 100644 index 0000000..7efb09c --- /dev/null +++ b/eclypse/builders/infrastructure/references/topohub/gabriel.py @@ -0,0 +1,98 @@ +"""Synthetic Gabriel-graph reference infrastructures. + +The Gabriel family models synthetic long-haul and optical-style backbone +topologies generated by TopoHub with controlled graph size and reproducible +samples. These graphs include node coordinates, link distances, and ECMP load +metadata, making them useful as scalable WAN references with realistic +distance-driven QoS properties. + +Example: + .. code-block:: python + + get_gabriel(25, sample=2, **kwargs) + +Source: + Jurkiewicz, "TopoHub: A repository of reference Gabriel graph and + real-world topologies for networking research", SoftwareX 2023, + https://doi.org/10.1016/j.softx.2023.101540 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ._helpers import get_topohub + +if TYPE_CHECKING: + from collections.abc import Callable + + import networkx as nx + + from eclypse.graph import Infrastructure + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_gabriel( + size: int, + sample: int = 0, + infrastructure_id: str | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create a Gabriel-graph infrastructure from TopoHub. + + The ``size`` and ``sample`` values must correspond to a valid topology in + `TopoHub `_'s ``gabriel`` family catalogue. + + Args: + size (int): + Number of nodes requested from the Gabriel family. + sample (int): + Reproducible sample index for the chosen size. + infrastructure_id (str | None): + Identifier assigned to the infrastructure. If omitted, a dataset-based + identifier is used. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. + + Returns: + Infrastructure: The converted Gabriel infrastructure. + """ + dataset_path = f"gabriel/{size}/{sample}" + default_id = f"gabriel_{size}_{sample}" + return get_topohub( + topology=dataset_path, + infrastructure_id=infrastructure_id or default_id, + use_names=False, + update_policies=update_policies, + node_assets=node_assets, + link_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + +__all__ = ["get_gabriel"] diff --git a/eclypse/builders/infrastructure/references/topohub/sndlib.py b/eclypse/builders/infrastructure/references/topohub/sndlib.py new file mode 100644 index 0000000..a5ded74 --- /dev/null +++ b/eclypse/builders/infrastructure/references/topohub/sndlib.py @@ -0,0 +1,92 @@ +"""SNDlib-backed reference infrastructures. + +The SNDlib family models published backbone and traffic-engineering benchmark +topologies from the Survivable Network Design Library. These references provide +realistic inter-site connectivity, geographic link lengths, and traffic-related +metadata such as demand matrices and ECMP load statistics when present in +TopoHub. + +Example: + .. code-block:: python + + get_sndlib("polska", **kwargs) + +Source: + SNDlib, https://sndlib.put.poznan.pl/ +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ._helpers import get_topohub + +if TYPE_CHECKING: + from collections.abc import Callable + + import networkx as nx + + from eclypse.graph import Infrastructure + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_sndlib( + topology: str, + infrastructure_id: str | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create a SNDlib-backed infrastructure from TopoHub. + + The ``topology`` value must be a valid topology name from + `TopoHub `_'s ``sndlib`` family catalogue. + + Args: + topology (str): + SNDlib topology identifier, such as ``"polska"`` or ``"geant"``. + infrastructure_id (str | None): + Identifier assigned to the infrastructure. If omitted, a dataset-based + identifier is used. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. + + Returns: + Infrastructure: The converted SNDlib infrastructure. + """ + dataset_path = f"sndlib/{topology}" + return get_topohub( + topology=dataset_path, + infrastructure_id=infrastructure_id or f"sndlib_{topology}", + use_names=True, + update_policies=update_policies, + node_assets=node_assets, + link_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + +__all__ = ["get_sndlib"] diff --git a/eclypse/builders/infrastructure/references/topohub/topology_zoo.py b/eclypse/builders/infrastructure/references/topohub/topology_zoo.py new file mode 100644 index 0000000..78837d1 --- /dev/null +++ b/eclypse/builders/infrastructure/references/topohub/topology_zoo.py @@ -0,0 +1,92 @@ +"""Internet Topology Zoo-backed reference infrastructures. + +The Topology Zoo family models published real-world backbone and research +network topologies curated by the Internet Topology Zoo and redistributed +through TopoHub. These references primarily contribute realistic node +placement, inter-site connectivity, and link distances, making them suitable +for WAN latency and geographic-placement studies. + +Example: + .. code-block:: python + + get_topology_zoo("Abilene", **kwargs) + +Source: + Internet Topology Zoo, https://topology-zoo.org/ +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ._helpers import get_topohub + +if TYPE_CHECKING: + from collections.abc import Callable + + import networkx as nx + + from eclypse.graph import Infrastructure + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +def get_topology_zoo( + topology: str, + infrastructure_id: str | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + link_assets: dict[str, Asset] | None = None, + include_default_assets: bool = False, + resource_init: InitPolicy = "max", + path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, + seed: int | None = None, +) -> Infrastructure: + """Create a Topology Zoo-backed infrastructure from TopoHub. + + The ``topology`` value must be a valid topology name from + `TopoHub `_'s ``topozoo`` family catalogue. + + Args: + topology (str): + Topology Zoo identifier, such as ``"Abilene"``. + infrastructure_id (str | None): + Identifier assigned to the infrastructure. If omitted, a dataset-based + identifier is used. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Node asset definitions available to the infrastructure. + link_assets (dict[str, Asset] | None): + Edge asset definitions available to the infrastructure. + include_default_assets (bool): + Whether to include default ECLYPSE assets. + resource_init (InitPolicy): + Initialisation policy used for graph assets. + path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): + Path computation function for infrastructure routing. + seed (int | None): + Seed forwarded to the infrastructure random generator. + + Returns: + Infrastructure: The converted Topology Zoo infrastructure. + """ + dataset_path = f"topozoo/{topology}" + return get_topohub( + topology=dataset_path, + infrastructure_id=infrastructure_id or f"topology_zoo_{topology.lower()}", + use_names=True, + update_policies=update_policies, + node_assets=node_assets, + link_assets=link_assets, + include_default_assets=include_default_assets, + resource_init=resource_init, + path_algorithm=path_algorithm, + seed=seed, + ) + + +__all__ = ["get_topology_zoo"] diff --git a/eclypse/builders/workflow/__init__.py b/eclypse/builders/workflow/__init__.py new file mode 100644 index 0000000..e03ce8b --- /dev/null +++ b/eclypse/builders/workflow/__init__.py @@ -0,0 +1,13 @@ +"""Workflow builders using `WfCommons `_ library. + +The package groups *simulation-only* workflow builders that generate task DAGs +and map their metadata onto ECLYPSE applications. These builders do not expose +service implementations for emulation; instead, they target workflow-oriented +simulation studies where task structure, runtime, and data movement matter. +When WfCommons file sizes are mapped onto ECLYPSE assets, workflow ``storage`` +and dependency ``bandwidth`` are normalised from bytes to MiB. +""" + +from .workflow import get_workflow + +__all__ = ["get_workflow"] diff --git a/eclypse/builders/workflow/_helpers.py b/eclypse/builders/workflow/_helpers.py new file mode 100644 index 0000000..0e7e4d0 --- /dev/null +++ b/eclypse/builders/workflow/_helpers.py @@ -0,0 +1,454 @@ +"""Helper functions shared by workflow builders.""" + +from __future__ import annotations + +import random +from contextlib import contextmanager +from importlib import import_module +from typing import ( + TYPE_CHECKING, + Any, + cast, +) + +import networkx as nx + +from eclypse.builders._helpers import prune_assets +from eclypse.graph import Application +from eclypse.graph.assets.defaults import ( + get_default_edge_assets, + get_default_node_assets, +) +from eclypse.simulation.config import _require_module + +from .base_method import WorkflowBaseMethod +from .workflow_family import WorkflowFamily + +if TYPE_CHECKING: + from collections.abc import Iterator + + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + +_MIN_TASKS = { + WorkflowFamily.BLAST: 45, + WorkflowFamily.BWA: 106, + WorkflowFamily.CYCLES: 69, + WorkflowFamily.EPIGENOMICS: 43, + WorkflowFamily.GENOME: 54, + WorkflowFamily.MONTAGE: 60, + WorkflowFamily.RNASEQ: 65, + WorkflowFamily.SEISMOLOGY: 103, + WorkflowFamily.SOYKB: 98, + WorkflowFamily.SRASEARCH: 24, +} + +_BYTES_PER_MIB = 2**20 + + +def coerce_workflow_family( + workflow: WorkflowFamily | str, +) -> WorkflowFamily: + """Return a workflow family enum from user input.""" + if isinstance(workflow, WorkflowFamily): + return workflow + + normalized = workflow.strip().lower() + for candidate in WorkflowFamily: + if candidate.value == normalized: + return candidate + + valid = ", ".join(member.value for member in WorkflowFamily) + raise ValueError(f"Unknown workflow family: {workflow}. Expected one of: {valid}") + + +def coerce_workflow_base_method( + base_method: WorkflowBaseMethod | str, +) -> WorkflowBaseMethod: + """Return a workflow base-method enum from user input.""" + if isinstance(base_method, WorkflowBaseMethod): + return base_method + + normalized = base_method.strip().lower() + for candidate in WorkflowBaseMethod: + if candidate.value == normalized: + return candidate + + valid = ", ".join(member.value for member in WorkflowBaseMethod) + raise ValueError( + f"Unknown workflow base method: {base_method}. Expected one of: {valid}", + ) + + +def build_workflow_application( + workflow: WorkflowFamily | str, + num_tasks: int | None = None, + data_footprint: int | None = 0, + exclude_graphs: set[str] | None = None, + runtime_factor: float | None = 1.0, + input_file_size_factor: float | None = 1.0, + output_file_size_factor: float | None = 1.0, + base_method: WorkflowBaseMethod | str = WorkflowBaseMethod.ERROR_TABLE, + workflow_name: str | None = None, + application_id: str | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + edge_assets: dict[str, Asset] | None = None, + requirement_init: InitPolicy = "min", + flows: list[list[str]] | str = "default", + seed: int | None = None, +) -> Application: + """Build a simulation-only workflow application from WfCommons.""" + workflow_family = coerce_workflow_family(workflow) + workflow_base_method = coerce_workflow_base_method(base_method) + resolved_num_tasks = validate_num_tasks(workflow_family, num_tasks) + + recipe_class, wfcommons_base_method, workflow_generator_class = _load_wfcommons( + workflow_family, + workflow_base_method, + ) + + recipe = recipe_class( + data_footprint=data_footprint, + num_tasks=resolved_num_tasks, + exclude_graphs=exclude_graphs, + runtime_factor=runtime_factor, + input_file_size_factor=input_file_size_factor, + output_file_size_factor=output_file_size_factor, + base_method=wfcommons_base_method, + ) + + generator = workflow_generator_class(recipe) + with _temporary_random_seed(seed): + generated_workflow = generator.build_workflow(workflow_name) + + _application_id = application_id or generated_workflow.name + _flows = ( + derive_workflow_flows(generated_workflow) + if flows == "default" + else cast("list[list[str]]", flows) + ) + default_node_assets = get_default_node_assets(with_init=False) + default_edge_assets = get_default_edge_assets(with_init=False) + default_node_assets.update(node_assets or {}) + default_edge_assets.update(edge_assets or {}) + + application = Application( + application_id=_application_id, + update_policies=update_policies, + node_assets=default_node_assets, + edge_assets=default_edge_assets, + include_default_assets=False, + requirement_init=requirement_init, + flows=_flows, + seed=seed, + ) + + _copy_workflow_metadata(application, generated_workflow, workflow_family) + + for node_id, node_data in generated_workflow.nodes(data=True): + task = node_data.get("task") + node_requirements = _task_asset_values(task) + metadata = _task_metadata(task) + metadata.update( + {key: value for key, value in node_data.items() if key != "task"} + ) + application.add_node( + str(node_id), + strict=False, + **prune_assets(application.node_assets, **node_requirements), + **metadata, + ) + + for source, target, edge_data in generated_workflow.edges(data=True): + source_task = generated_workflow.nodes[source].get("task") + target_task = generated_workflow.nodes[target].get("task") + edge_requirements = _edge_asset_values(source_task, target_task) + metadata = _edge_metadata(source_task, target_task, edge_data) + application.add_edge( + str(source), + str(target), + symmetric=False, + strict=False, + **prune_assets(application.edge_assets, **edge_requirements), + **metadata, + ) + + return application + + +def derive_workflow_flows(workflow: nx.DiGraph) -> list[list[str]]: + """Derive default root-to-leaf flows from a workflow DAG.""" + roots = list(_workflow_roots(workflow)) + leaves = list(_workflow_leaves(workflow)) + resolved: set[tuple[str, ...]] = set() + + for root in roots: + for leaf in leaves: + if not nx.has_path(workflow, root, leaf): + continue + for path in nx.all_simple_paths(workflow, root, leaf): + resolved.add(tuple(str(node_id) for node_id in path)) + + return [list(path) for path in sorted(resolved)] + + +def validate_num_tasks( + workflow_family: WorkflowFamily, + num_tasks: int | None, +) -> int: + """Resolve and validate the number of tasks for a workflow family.""" + minimum = _MIN_TASKS[workflow_family] + resolved = minimum if num_tasks is None else num_tasks + + if resolved < minimum: + raise ValueError( + f"Workflow family '{workflow_family.value}' requires " + f"num_tasks >= {minimum}, got {resolved}.", + ) + + return resolved + + +def _load_wfcommons( + workflow_family: WorkflowFamily, + base_method: WorkflowBaseMethod, +) -> tuple[type[Any], Any, type[Any]]: + """Resolve the WfCommons recipe, base method, and generator classes.""" + _require_module("wfcommons") + + wfcommons = import_module("wfcommons") + wfcommons_abstract_recipe = import_module("wfcommons.wfchef.wfchef_abstract_recipe") + WfCommonsBaseMethod = wfcommons_abstract_recipe.BaseMethod + + recipe_names = { + WorkflowFamily.BLAST: "BlastRecipe", + WorkflowFamily.BWA: "BwaRecipe", + WorkflowFamily.CYCLES: "CyclesRecipe", + WorkflowFamily.EPIGENOMICS: "EpigenomicsRecipe", + WorkflowFamily.GENOME: "GenomeRecipe", + WorkflowFamily.MONTAGE: "MontageRecipe", + WorkflowFamily.RNASEQ: "RnaseqRecipe", + WorkflowFamily.SEISMOLOGY: "SeismologyRecipe", + WorkflowFamily.SOYKB: "SoykbRecipe", + WorkflowFamily.SRASEARCH: "SrasearchRecipe", + } + + recipe_class = getattr(wfcommons, recipe_names[workflow_family]) + wfcommons_base_method = getattr(WfCommonsBaseMethod, base_method.name) + return recipe_class, wfcommons_base_method, wfcommons.WorkflowGenerator + + +@contextmanager +def _temporary_random_seed(seed: int | None) -> Iterator[None]: + """Temporarily seed the Python random module for WfCommons generation.""" + if seed is None: + yield + return + + state = random.getstate() + random.seed(seed) + try: + yield + finally: + random.setstate(state) + + +def _workflow_roots(workflow: nx.DiGraph) -> list[Any]: + """Return the roots of a workflow graph.""" + if hasattr(workflow, "roots") and callable(workflow.roots): + return list(workflow.roots()) + return [node_id for node_id, degree in workflow.in_degree() if degree == 0] + + +def _workflow_leaves(workflow: nx.DiGraph) -> list[Any]: + """Return the leaves of a workflow graph.""" + if hasattr(workflow, "leaves") and callable(workflow.leaves): + return list(workflow.leaves()) + return [node_id for node_id, degree in workflow.out_degree() if degree == 0] + + +def _copy_workflow_metadata( + application: Application, + workflow: Any, + workflow_family: WorkflowFamily, +) -> None: + """Copy graph-level workflow metadata into the application graph.""" + application.graph["workflow_backend"] = "wfcommons" + application.graph["workflow_family"] = workflow_family.value + + for key in ( + "name", + "description", + "workflow_id", + "created_at", + "executed_at", + "makespan", + "schema_version", + "runtime_system_name", + "runtime_system_url", + "runtime_system_version", + "author_name", + "author_email", + "author_institution", + "author_country", + ): + value = getattr(workflow, key, None) + if value is not None: + application.graph[key] = value + + graph_metadata = getattr(workflow, "graph", {}) + if isinstance(graph_metadata, dict): + application.graph.update( + {key: value for key, value in graph_metadata.items() if key not in {"task"}} + ) + + +def _task_asset_values(task: Any) -> dict[str, float]: + """Map task metadata onto default ECLYPSE node assets. + + Workflow storage requirements are normalised from raw WfCommons byte counts + to MiB before they are assigned to the ECLYPSE ``storage`` asset. + """ + if task is None: + return {} + + task_storage = _task_storage(task) + values: dict[str, float] = {} + + if getattr(task, "cores", None) is not None: + values["cpu"] = float(task.cores) + if getattr(task, "memory", None) is not None: + values["ram"] = float(task.memory) + if task_storage is not None: + values["storage"] = float(task_storage) + if getattr(task, "runtime", None) is not None: + values["processing_time"] = float(task.runtime) + + return values + + +def _task_metadata(task: Any) -> dict[str, Any]: + """Return task metadata that does not map to default assets.""" + if task is None: + return {} + + input_files = list(getattr(task, "input_files", [])) + output_files = list(getattr(task, "output_files", [])) + task_type = getattr(task, "type", None) + + metadata: dict[str, Any] = { + "workflow_task_name": getattr(task, "name", None), + "workflow_task_program": getattr(task, "program", None), + "workflow_task_category": getattr(task, "category", None), + "workflow_task_priority": getattr(task, "priority", None), + "workflow_task_start_time": getattr(task, "start_time", None), + "workflow_task_type": getattr(task_type, "name", None), + "workflow_input_files": [ + getattr(file, "file_id", None) for file in input_files + ], + "workflow_output_files": [ + getattr(file, "file_id", None) for file in output_files + ], + "workflow_input_size_mib": _bytes_to_mib(_total_file_size(input_files)), + "workflow_output_size_mib": _bytes_to_mib(_total_file_size(output_files)), + "workflow_args": list(getattr(task, "args", [])), + "workflow_machines": list(getattr(task, "machines", [])), + "workflow_bytes_read": getattr(task, "bytes_read", None), + "workflow_bytes_written": getattr(task, "bytes_written", None), + } + return {key: value for key, value in metadata.items() if value is not None} + + +def _edge_asset_values(source_task: Any, target_task: Any) -> dict[str, float]: + """Map workflow dependency metadata onto default ECLYPSE edge assets. + + Dependency transfer sizes are normalised from raw WfCommons byte counts to + MiB before they are assigned to the ECLYPSE ``bandwidth`` asset. + """ + transferred_size = _transferred_size(source_task, target_task) + if transferred_size is None: + return {} + return {"bandwidth": float(transferred_size)} + + +def _edge_metadata( + source_task: Any, + target_task: Any, + edge_data: dict[str, Any], +) -> dict[str, Any]: + """Return edge metadata for a workflow dependency.""" + transferred_files = _shared_output_input_files(source_task, target_task) + metadata = dict(edge_data) + metadata["workflow_transferred_files"] = list(transferred_files) + metadata["workflow_transferred_size_mib"] = _bytes_to_mib( + sum(transferred_files.values()), + ) + return metadata + + +def _task_storage(task: Any) -> float | None: + """Return the storage footprint associated with a workflow task in MiB.""" + input_size = _total_file_size(getattr(task, "input_files", [])) + output_size = _total_file_size(getattr(task, "output_files", [])) + bytes_read = getattr(task, "bytes_read", None) or 0 + bytes_written = getattr(task, "bytes_written", None) or 0 + storage = max(int(input_size + output_size), int(bytes_read + bytes_written)) + return _bytes_to_mib(storage) + + +def _transferred_size(source_task: Any, target_task: Any) -> float | None: + """Return the total size of files transferred across a workflow edge in MiB.""" + shared_files = _shared_output_input_files(source_task, target_task) + transferred_size = sum(shared_files.values()) + return _bytes_to_mib(transferred_size) + + +def _shared_output_input_files(source_task: Any, target_task: Any) -> dict[str, int]: + """Return the files produced by the source and consumed by the target.""" + if source_task is None or target_task is None: + return {} + + source_outputs: dict[str, int] = {} + for file in getattr(source_task, "output_files", []): + file_id = getattr(file, "file_id", None) + if file_id is None: + continue + source_outputs[str(file_id)] = int(getattr(file, "size", 0) or 0) + + target_inputs = { + str(getattr(file, "file_id", None)) + for file in getattr(target_task, "input_files", []) + if getattr(file, "file_id", None) + } + + return { + file_id: size + for file_id, size in source_outputs.items() + if file_id in target_inputs + } + + +def _total_file_size(files: list[Any]) -> int: + """Return the total size of a list of workflow files.""" + return sum(int(getattr(file, "size", 0) or 0) for file in files) + + +def _bytes_to_mib(size_in_bytes: int) -> float | None: + """Convert a byte count into MiB, returning ``None`` for zero values.""" + return (size_in_bytes / _BYTES_PER_MIB) or None + + +__all__ = [ + "WorkflowBaseMethod", + "WorkflowFamily", + "build_workflow_application", + "coerce_workflow_base_method", + "coerce_workflow_family", + "derive_workflow_flows", + "validate_num_tasks", +] diff --git a/eclypse/builders/workflow/base_method.py b/eclypse/builders/workflow/base_method.py new file mode 100644 index 0000000..6ccf06e --- /dev/null +++ b/eclypse/builders/workflow/base_method.py @@ -0,0 +1,17 @@ +"""Workflow generation base methods.""" + +from __future__ import annotations + +from enum import StrEnum + + +class WorkflowBaseMethod(StrEnum): + """Base graph selection strategies supported by WfCommons recipes.""" + + ERROR_TABLE = "error_table" + SMALLEST = "smallest" + BIGGEST = "biggest" + RANDOM = "random" + + +__all__ = ["WorkflowBaseMethod"] diff --git a/eclypse/builders/workflow/workflow.py b/eclypse/builders/workflow/workflow.py new file mode 100644 index 0000000..466abaf --- /dev/null +++ b/eclypse/builders/workflow/workflow.py @@ -0,0 +1,128 @@ +"""Simulation-only workflow applications generated with `WfCommons `_. + +The builder generates a task-dependency DAG and converts it into an ECLYPSE +application graph for simulation. It does not expose MPI/REST service logic or +emulation support. Workflow task metadata is instead mapped directly onto the +default ECLYPSE node and edge assets, which are always included. +WfCommons file-size metadata is normalised from bytes to MiB when it is mapped +onto ECLYPSE ``storage`` and ``bandwidth`` assets. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ._helpers import build_workflow_application +from .base_method import WorkflowBaseMethod + +if TYPE_CHECKING: + from eclypse.graph import Application + from eclypse.graph.assets import Asset + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) + + from .workflow_family import WorkflowFamily + + +def get_workflow( + workflow: WorkflowFamily | str, + num_tasks: int | None = None, + data_footprint: int | None = 0, + exclude_graphs: set[str] | None = None, + runtime_factor: float | None = 1.0, + input_file_size_factor: float | None = 1.0, + output_file_size_factor: float | None = 1.0, + base_method: WorkflowBaseMethod | str = WorkflowBaseMethod.ERROR_TABLE, + workflow_name: str | None = None, + application_id: str | None = None, + update_policies: UpdatePolicies = None, + node_assets: dict[str, Asset] | None = None, + edge_assets: dict[str, Asset] | None = None, + requirement_init: InitPolicy = "min", + flows: list[list[str]] | str = "default", + seed: int | None = None, +) -> Application: + """Create a simulation-only workflow application generated by `WfCommons `_. + + The returned application is graph-only: it contains task nodes, dependency + edges, and root-to-leaf flows for simulation, but it does not instantiate + executable services or communication interfaces for emulation. + WfCommons file-size metadata is mapped onto ECLYPSE ``storage`` and + ``bandwidth`` assets in MiB. + + Args: + workflow (WorkflowFamily | str): + Workflow family to generate. Supported values are the members of + :class:`~eclypse.builders.workflow.workflow_family.WorkflowFamily`, + such as ``WorkflowFamily.MONTAGE`` or ``"montage"``. + num_tasks (int | None): + Target number of tasks for the generated workflow. When omitted, + the minimum valid size supported by the selected WfCommons + workflow family is used. + data_footprint (int | None): + Total workflow data footprint requested from WfCommons. + exclude_graphs (set[str] | None): + Optional set of WfCommons graph identifiers to exclude during + generation. + runtime_factor (float | None): + Scaling factor applied to generated task runtimes. + input_file_size_factor (float | None): + Scaling factor applied to generated task input file sizes. + output_file_size_factor (float | None): + Scaling factor applied to generated task output file sizes. + base_method (WorkflowBaseMethod | str): + Base graph selection strategy used by WfCommons. Supported values + are the members of + :class:`~eclypse.builders.workflow.base_method.WorkflowBaseMethod`, + such as ``WorkflowBaseMethod.ERROR_TABLE`` or ``"error_table"``. + workflow_name (str | None): + Optional workflow instance name forwarded to WfCommons. + application_id (str | None): + Optional ECLYPSE application identifier. If omitted, the generated + workflow name is reused. + update_policies (UpdatePolicies): + Graph update policies executed during ``evolve()``. + node_assets (dict[str, Asset] | None): + Additional node asset definitions merged with the default ECLYPSE + node assets. Default assets are always included, and workflow + ``storage`` is normalised to MiB from WfCommons byte counts. + edge_assets (dict[str, Asset] | None): + Additional edge asset definitions merged with the default ECLYPSE + edge assets. Default assets are always included, and workflow + dependency ``bandwidth`` is normalised to MiB from WfCommons byte + counts. + requirement_init (InitPolicy): + Initialisation strategy applied to workflow node and edge assets. + flows (list[list[str]] | str): + Workflow flows to install on the application. Use ``"default"`` to + derive root-to-leaf flows from the generated DAG. + seed (int | None): + Seed used both for ECLYPSE graph initialisation and to stabilise + WfCommons workflow generation through Python's random module. + + Returns: + Application: The generated workflow application. + """ + return build_workflow_application( + workflow=workflow, + num_tasks=num_tasks, + data_footprint=data_footprint, + exclude_graphs=exclude_graphs, + runtime_factor=runtime_factor, + input_file_size_factor=input_file_size_factor, + output_file_size_factor=output_file_size_factor, + base_method=base_method, + workflow_name=workflow_name, + application_id=application_id, + update_policies=update_policies, + node_assets=node_assets, + edge_assets=edge_assets, + requirement_init=requirement_init, + flows=flows, + seed=seed, + ) + + +__all__ = ["get_workflow"] diff --git a/eclypse/builders/workflow/workflow_family.py b/eclypse/builders/workflow/workflow_family.py new file mode 100644 index 0000000..6238712 --- /dev/null +++ b/eclypse/builders/workflow/workflow_family.py @@ -0,0 +1,23 @@ +"""Workflow families supported by the WfCommons-backed builder.""" + +from __future__ import annotations + +from enum import StrEnum + + +class WorkflowFamily(StrEnum): + """Workflow families that can be generated by WfCommons.""" + + BLAST = "blast" + BWA = "bwa" + CYCLES = "cycles" + EPIGENOMICS = "epigenomics" + GENOME = "genome" + MONTAGE = "montage" + RNASEQ = "rnaseq" + SEISMOLOGY = "seismology" + SOYKB = "soykb" + SRASEARCH = "srasearch" + + +__all__ = ["WorkflowFamily"] diff --git a/eclypse/graph/application.py b/eclypse/graph/application.py index 8b29a2a..de40285 100644 --- a/eclypse/graph/application.py +++ b/eclypse/graph/application.py @@ -38,7 +38,7 @@ def __init__( update_policies: UpdatePolicies = None, node_assets: dict[str, Asset] | None = None, edge_assets: dict[str, Asset] | None = None, - include_default_assets: bool = False, + include_default_assets: bool = True, requirement_init: InitPolicy = "min", flows: list[list[str]] | None = None, seed: int | None = None, @@ -52,7 +52,7 @@ def __init__( node_assets (dict[str, Asset] | None): The assets of the nodes. edge_assets (dict[str, Asset] | None): The assets of the edges. include_default_assets (bool): Whether to include the default assets. \ - Defaults to False. + Defaults to True. requirement_init (InitPolicy): The initialization of the requirements. flows (list[list[str]] | None): The flows of the application. @@ -60,8 +60,8 @@ def __init__( """ _node_assets = get_default_node_assets() if include_default_assets else {} _edge_assets = get_default_edge_assets() if include_default_assets else {} - _node_assets.update(node_assets if node_assets is not None else {}) - _edge_assets.update(edge_assets if edge_assets is not None else {}) + _node_assets.update(node_assets or {}) + _edge_assets.update(edge_assets or {}) super().__init__( graph_id=application_id, @@ -100,16 +100,19 @@ def add_service(self, service: Service, **assets): self.add_node(service.id, **assets) - def set_flows(self): + def set_flows(self, ingress: str = "gateway"): """Set the flows of the application, using the following rules. - If the flows are already set, do nothing. - - If the flows are not set, use the gateway as the source and all\ + - If the flows are not set, use the `ingress` as the source and all the other nodes as the target. - - If there is no gateway, set the flows to an empty list. + - If there is no ingress, set the flows to an empty list. + + Args: + ingress (str): The name of the ingress node. Defaults to "gateway". """ if self.flows == []: - gateway_name = next((s for s in self.nodes if "gateway" in s.lower()), None) + gateway_name = next((s for s in self.nodes if s == ingress), None) if gateway_name is not None: self.flows = [] for target in self.nodes: @@ -123,7 +126,7 @@ def set_flows(self): continue @cached_property - def has_logic(self) -> bool: + def has_service_implementations(self) -> bool: """Check if the application has a logic for each service. This property requires to be True for the remote execution. diff --git a/eclypse/graph/asset_graph.py b/eclypse/graph/asset_graph.py index b76ac9c..8188e4f 100644 --- a/eclypse/graph/asset_graph.py +++ b/eclypse/graph/asset_graph.py @@ -69,8 +69,8 @@ def __init__( self.id = graph_id self.update_policies = _normalize_update_policies(update_policies) - _node_assets = node_assets if node_assets is not None else {} - _edge_assets = edge_assets if edge_assets is not None else {} + _node_assets = node_assets or {} + _edge_assets = edge_assets or {} self.node_assets = AssetBucket(**_node_assets) self.edge_assets = AssetBucket(**_edge_assets) @@ -118,7 +118,7 @@ def add_node(self, node_for_adding: str, strict: bool = True, **assets): violations = self.node_assets.is_consistent(_assets, violations=True) if isinstance(violations, dict) and violations: - msg = f"Node {node_for_adding} has inconsistent assets | " + format_log_kv( + msg = f"{node_for_adding} has inconsistent assets | " + format_log_kv( assets=",".join(sorted(violations)) ) if strict: @@ -168,7 +168,7 @@ def add_edge( violations = self.edge_assets.is_consistent(_assets, violations=True) if isinstance(violations, dict) and violations: msg = ( - f"Edge {u_of_edge} -> {v_of_edge} has inconsistent assets | " + f"({u_of_edge} -> {v_of_edge}) has inconsistent assets | " + format_log_kv(assets=",".join(sorted(violations))) ) if strict: diff --git a/eclypse/graph/assets/defaults.py b/eclypse/graph/assets/defaults.py index 4b6eb80..ad9edb7 100644 --- a/eclypse/graph/assets/defaults.py +++ b/eclypse/graph/assets/defaults.py @@ -59,12 +59,7 @@ def cpu( Returns: Additive: The CPU asset. """ - _init_fn = ( - Choice([2**i for i in range(1, 9)]) - if init_fn_or_value is None - else init_fn_or_value - ) - return Additive(lower_bound, upper_bound, _init_fn) + return Additive(lower_bound, upper_bound, init_fn_or_value) def ram( @@ -83,12 +78,7 @@ def ram( Returns: Additive: The RAM asset. """ - _init_fn = ( - Choice([2**i for i in range(1, 11)]) - if init_fn_or_value is None - else init_fn_or_value - ) - return Additive(lower_bound, upper_bound, _init_fn) + return Additive(lower_bound, upper_bound, init_fn_or_value) def storage( @@ -107,12 +97,7 @@ def storage( Returns: Additive: The storage asset. """ - _init_fn = ( - Choice([2**i for i in range(1, 13)]) - if init_fn_or_value is None - else init_fn_or_value - ) - return Additive(lower_bound, upper_bound, _init_fn) + return Additive(lower_bound, upper_bound, init_fn_or_value) def gpu( @@ -131,12 +116,7 @@ def gpu( Returns: Additive: The GPU asset. """ - _init_fn = ( - Choice([2**i for i in range(1, 9)]) - if init_fn_or_value is None - else init_fn_or_value - ) - return Additive(lower_bound, upper_bound, _init_fn) + return Additive(lower_bound, upper_bound, init_fn_or_value) def availability( @@ -155,8 +135,7 @@ def availability( Returns: Multiplicative: The availability asset. """ - _init_fn = Uniform(0.99, 1) if init_fn_or_value is None else init_fn_or_value - return Multiplicative(lower_bound, upper_bound, _init_fn) + return Multiplicative(lower_bound, upper_bound, init_fn_or_value) def processing_time( @@ -175,8 +154,7 @@ def processing_time( Returns: Concave: The processing time asset. """ - _init_fn = IntUniform(1, 25) if init_fn_or_value is None else init_fn_or_value - return Concave(lower_bound, upper_bound, _init_fn, functional=False) + return Concave(lower_bound, upper_bound, init_fn_or_value, functional=False) def latency( @@ -195,8 +173,7 @@ def latency( Returns: Concave: The latency asset. """ - _init_fn = IntUniform(1, 40) if init_fn_or_value is None else init_fn_or_value - return Concave(lower_bound, upper_bound, _init_fn) + return Concave(lower_bound, upper_bound, init_fn_or_value) def bandwidth( @@ -215,36 +192,62 @@ def bandwidth( Returns: Additive: The bandwidth asset. """ - _init_fn = IntUniform(50, 1500) if init_fn_or_value is None else init_fn_or_value - return Additive(lower_bound, upper_bound, _init_fn) + return Additive(lower_bound, upper_bound, init_fn_or_value) + + +_DEFAULT_NODE_ASSETS_INIT_FN = { + "cpu": Choice([2**i for i in range(1, 9)]), + "ram": Choice([2**i for i in range(1, 11)]), + "storage": Choice([2**i for i in range(1, 13)]), + "gpu": Choice([2**i for i in range(1, 9)]), + "availability": Uniform(0.99, 1), + "processing_time": IntUniform(1, 25), +} +_DEFAULT_EDGE_ASSETS_INIT_FN = { + "latency": IntUniform(1, 40), + "bandwidth": IntUniform(50, 1500), +} -def get_default_node_assets(): + +def get_default_node_assets(with_init: bool = True): """Get the set of default node assets. + Args: + with_init (bool): + Whether to attach the bundled default initialisers to the assets. + Returns: dict[str, Any]: The default node assets: cpu, ram, storage, gpu, availability, processing_time. """ + init_fns = _DEFAULT_NODE_ASSETS_INIT_FN if with_init else {} return { - "cpu": cpu(), - "ram": ram(), - "storage": storage(), - "gpu": gpu(), - "availability": availability(), - "processing_time": processing_time(), + "cpu": cpu(init_fn_or_value=init_fns.get("cpu")), + "ram": ram(init_fn_or_value=init_fns.get("ram")), + "storage": storage(init_fn_or_value=init_fns.get("storage")), + "gpu": gpu(init_fn_or_value=init_fns.get("gpu")), + "availability": availability(init_fn_or_value=init_fns.get("availability")), + "processing_time": processing_time( + init_fn_or_value=init_fns.get("processing_time") + ), } -def get_default_edge_assets(): +def get_default_edge_assets(with_init: bool = True): """Get the set of default edge assets. + Args: + with_init (bool): + Whether to attach the bundled default initialisers to the assets. + Returns: dict[str, Any]: The default edge assets: latency, bandwidth. """ + init_fns = _DEFAULT_EDGE_ASSETS_INIT_FN if with_init else {} return { - "latency": latency(), - "bandwidth": bandwidth(), + "latency": latency(init_fn_or_value=init_fns.get("latency")), + "bandwidth": bandwidth(init_fn_or_value=init_fns.get("bandwidth")), } diff --git a/eclypse/graph/infrastructure.py b/eclypse/graph/infrastructure.py index 7d63ffc..49baafd 100644 --- a/eclypse/graph/infrastructure.py +++ b/eclypse/graph/infrastructure.py @@ -4,7 +4,6 @@ edges representing links between them. The infrastructure also stores: -- A global placement strategy (optional). - A set of path assets aggregators, one per edge asset. - A path algorithm to compute the paths between nodes. - A view of the available nodes and edges. @@ -42,7 +41,6 @@ from collections.abc import Callable from eclypse.graph.assets.asset import Asset - from eclypse.placement.strategies import PlacementStrategy from eclypse.utils.types import ( InitPolicy, UpdatePolicies, @@ -55,11 +53,10 @@ class Infrastructure(AssetGraph): # pylint: disable=too-few-public-methods def __init__( self, infrastructure_id: str = "Infrastructure", - placement_strategy: PlacementStrategy | None = None, update_policies: UpdatePolicies = None, node_assets: dict[str, Asset] | None = None, edge_assets: dict[str, Asset] | None = None, - include_default_assets: bool = False, + include_default_assets: bool = True, path_assets_aggregators: dict[str, Callable[[list[Any]], Any]] | None = None, path_algorithm: Callable[[nx.Graph, str, str], list[str]] | None = None, resource_init: InitPolicy = "min", @@ -69,14 +66,12 @@ def __init__( Args: infrastructure_id (str): The ID of the infrastructure. - placement_strategy (PlacementStrategy | None): The placement \ - strategy to use. update_policies (Callable | list[Callable] | None):\ Graph update policies executed during ``evolve()``. node_assets (dict[str, Asset] | None): The assets of the nodes. edge_assets (dict[str, Asset] | None): The assets of the edges. include_default_assets (bool): Whether to include the default assets. \ - Defaults to False. + Defaults to True. path_assets_aggregators (dict[str, Callable[[list[Any]], Any]] | None): \ The aggregators to use for the path assets. path_algorithm (Callable[[nx.Graph, str, str], list[str]] | None): \ @@ -87,8 +82,8 @@ def __init__( """ _node_assets = get_default_node_assets() if include_default_assets else {} _edge_assets = get_default_edge_assets() if include_default_assets else {} - _node_assets.update(node_assets if node_assets is not None else {}) - _edge_assets.update(edge_assets if edge_assets is not None else {}) + _node_assets.update(node_assets or {}) + _edge_assets.update(edge_assets or {}) super().__init__( graph_id=infrastructure_id, @@ -129,8 +124,6 @@ def __init__( else _get_default_path_algorithm ) - self.strategy = placement_strategy - self._available: nx.DiGraph | None = None self._paths: dict[str, dict[str, list[str]]] = {} self._costs: dict[str, dict[str, list[tuple[str, str, Any]]]] = {} @@ -194,8 +187,8 @@ def remove_edge(self, u: str, v: str): super().remove_edge(u, v) self._invalidate_cache() - def contains(self, other: nx.DiGraph) -> list[str]: - """Comparison between requirements and infrastructure resources. + def validate(self, other: nx.DiGraph) -> list[str]: + """Validate the infrastructure against a set of requirements. Compares the requirements of the nodes and edges in the PlacementView with the resources of the nodes and edges in the Infrastructure. @@ -391,15 +384,6 @@ def is_available(self, n: str): """ return self.nodes[n].get("availability", 1) > 0 - @property - def has_strategy(self) -> bool: - """Check if the infrastructure has a placement strategy. - - Returns: - bool: True if the infrastructure has a placement strategy, False otherwise. - """ - return self.strategy is not None - def _default_weight_function(_: str, __: str, eattr: dict[str, Any]) -> float: """Function to compute the weight of an edge in the shortest path algorithm. diff --git a/eclypse/placement/_manager.py b/eclypse/placement/_manager.py index 71b868b..b9349cf 100644 --- a/eclypse/placement/_manager.py +++ b/eclypse/placement/_manager.py @@ -8,10 +8,7 @@ from __future__ import annotations from random import shuffle -from typing import ( - TYPE_CHECKING, - Any, -) +from typing import TYPE_CHECKING from eclypse.placement import Placement from eclypse.utils._logging import ( @@ -22,28 +19,34 @@ from .view import PlacementView if TYPE_CHECKING: - from collections.abc import ( - Generator, - ) + from collections.abc import Generator from eclypse.graph import ( Application, Infrastructure, ) from eclypse.placement import PlacementStrategy + from eclypse.utils._logging import Logger class PlacementManager: """PlacementManager manages the placement of applications in the infrastructure.""" - def __init__(self, infrastructure: Infrastructure): + def __init__( + self, + infrastructure: Infrastructure, + default_strategy: PlacementStrategy | None = None, + ): """Initializes the PlacementManager. Args: infrastructure (Infrastructure): The infrastructure to place the applications onto. + default_strategy (PlacementStrategy | None): + Strategy used when an application is registered without one. """ self.infrastructure = infrastructure + self.default_strategy = default_strategy self.placements: dict[str, Placement] = {} self.placement_view: PlacementView = PlacementView(self.infrastructure) @@ -99,25 +102,27 @@ def generate_mapping(self, placement: Placement): Args: placement (Placement): The placement to generate the mapping for. """ + strategy = placement.strategy or self.default_strategy + + if strategy is None: + raise ValueError( + f"No placement strategy provided for {placement.application.id}" + ) + if placement.strategy is None: self.logger.trace( - f"Using {self.infrastructure.strategy.__class__.__name__} " + f"Using {strategy.__class__.__name__} " f" strategy for {placement.application.id}", ) - if self.infrastructure.has_strategy: - placement.mapping = self.infrastructure.strategy.place( # type: ignore[union-attr] - self.infrastructure, - placement.application, - self.placements, - self.placement_view, - ) - else: - raise ValueError( - f"No placement strategy provided for {placement.application.id}" - ) + placement.mapping = strategy.place( + self.infrastructure, + placement.application, + self.placements, + self.placement_view, + ) else: self.logger.trace( - f"Using {placement.strategy.__class__.__name__} " + f"Using {strategy.__class__.__name__} " f"strategy for {placement.application.id}" ) placement._generate_mapping(self.placements, self.placement_view) @@ -175,7 +180,7 @@ def mapping_phase( self.placement_view._update_view(p) - not_respected = self.infrastructure.contains(self.placement_view) + not_respected = self.infrastructure.validate(self.placement_view) yield (p, not_respected) def register( @@ -216,7 +221,7 @@ def get(self, application_id: str) -> Placement: return self.placements[application_id] @property - def logger(self) -> Any: + def logger(self) -> Logger: """Get a logger for the PlacementManager. Returns: diff --git a/eclypse/placement/strategies/best_fit.py b/eclypse/placement/strategies/best_fit.py index 2e562ff..3793cec 100644 --- a/eclypse/placement/strategies/best_fit.py +++ b/eclypse/placement/strategies/best_fit.py @@ -41,7 +41,7 @@ def place( application: Application, _: dict[str, Placement], placement_view: PlacementView, - ) -> dict[Any, Any]: + ) -> dict[str, str]: """Performs the placement according to a best-fit logic. Places the services of an application on the infrastructure nodes based on @@ -80,9 +80,9 @@ def place( best_fit = node best_nattr = nattr best_idx = idx - mapping[service] = best_fit if best_fit is None or best_nattr is None or best_idx is None: continue + mapping[service] = best_fit infrastructure_nodes[best_idx] = ( best_fit, diff --git a/eclypse/placement/strategies/random.py b/eclypse/placement/strategies/random.py index 6871db4..a168458 100644 --- a/eclypse/placement/strategies/random.py +++ b/eclypse/placement/strategies/random.py @@ -11,7 +11,6 @@ import random as rnd from typing import ( TYPE_CHECKING, - Any, ) from eclypse.utils.constants import RND_SEED @@ -53,7 +52,7 @@ def place( application: Application, __: dict[str, Placement], placement_view: PlacementView, - ) -> dict[Any, Any]: + ) -> dict[str, str]: """Places the services of an application on the infrastructure nodes, randomly. Args: diff --git a/eclypse/placement/strategies/round_robin.py b/eclypse/placement/strategies/round_robin.py index 057ec26..00a0cea 100644 --- a/eclypse/placement/strategies/round_robin.py +++ b/eclypse/placement/strategies/round_robin.py @@ -51,7 +51,7 @@ def place( application: Application, __: dict[str, Placement], placement_view: PlacementView, - ) -> dict[Any, Any]: + ) -> dict[str, str]: """Performs the placement according to a round-robin logic. Places the services of an application on the infrastructure nodes, attempting diff --git a/eclypse/placement/strategies/static.py b/eclypse/placement/strategies/static.py index efc3395..77639d0 100644 --- a/eclypse/placement/strategies/static.py +++ b/eclypse/placement/strategies/static.py @@ -7,10 +7,7 @@ from __future__ import annotations -from typing import ( - TYPE_CHECKING, - Any, -) +from typing import TYPE_CHECKING from .strategy import PlacementStrategy @@ -51,7 +48,7 @@ def place( application: Application, _: dict[str, Placement], __: PlacementView, - ) -> dict[Any, Any]: + ) -> dict[str, str]: """Returns the static mapping of services to nodes, given at initialization. Returns: diff --git a/eclypse/placement/strategies/strategy.py b/eclypse/placement/strategies/strategy.py index e439607..608afb3 100644 --- a/eclypse/placement/strategies/strategy.py +++ b/eclypse/placement/strategies/strategy.py @@ -10,10 +10,7 @@ ABC, abstractmethod, ) -from typing import ( - TYPE_CHECKING, - Any, -) +from typing import TYPE_CHECKING if TYPE_CHECKING: from eclypse.graph import ( @@ -40,7 +37,7 @@ def place( application: Application, placements: dict[str, Placement], placement_view: PlacementView, - ) -> dict[Any, Any]: + ) -> dict[str, str]: """Defines the placement logic. Given an infrastructure, an application, a dictionary of placements, and a @@ -58,9 +55,8 @@ def place( placement_view (PlacementView): The placement view to use for the placement. Returns: - dict[Any, Any]: - A dictionary mapping service IDs to node IDs, or None if - the application cannot be placed onto the infrastructure. + dict[str, str]: + A dictionary mapping service IDs to node IDs. """ def is_feasible( diff --git a/eclypse/placement/view.py b/eclypse/placement/view.py index 61afce9..d40d9fd 100644 --- a/eclypse/placement/view.py +++ b/eclypse/placement/view.py @@ -23,6 +23,7 @@ Concave, Convex, ) +from eclypse.utils._logging import format_log_kv if TYPE_CHECKING: from collections.abc import ( @@ -221,11 +222,19 @@ def _update_view(self, placement: Placement): self.add_edge(node_s, node_t, **_int_reqs) else: + # placement.infrastructure.logger.warning( + # f"Stopping placement search for {placement.application.id}" + # ) + # placement.infrastructure.logger.warning( + # f" [Path not found] {s} ({node_s}) -> {t} ({node_t})" + # ) placement.infrastructure.logger.warning( - f"Stopping placement search for {placement.application.id}" - ) - placement.infrastructure.logger.warning( - f" [Path not found] {s} ({node_s}) -> {t} ({node_t})" + "Path not found | " + + format_log_kv( + app=placement.application.id, + source=f"{s} ({node_s})", + target=f"{t} ({node_t})", + ) ) placement.mark_for_reset() break diff --git a/eclypse/policies/__init__.py b/eclypse/policies/__init__.py index baf46d7..ff356cd 100644 --- a/eclypse/policies/__init__.py +++ b/eclypse/policies/__init__.py @@ -7,23 +7,33 @@ from __future__ import annotations -from eclypse.policies import ( +from . import ( + compose, + constraints, degrade, distribution, failure, noise, replay, schedule, + topology, + workload, ) -from eclypse.policies._filters import ( +from ._filters import ( EdgeFilter, NodeFilter, ) -from eclypse.policies.schedule import ( +from .schedule import ( after, + at, between, + cooldown, every, + jittered_every, once_at, + repeat, + until, + with_probability, ) from eclypse.utils.types import ( UpdatePolicies, @@ -37,13 +47,23 @@ "UpdatePolicies", "UpdatePolicy", "after", + "at", "between", + "compose", + "constraints", + "cooldown", "degrade", "distribution", "every", "failure", + "jittered_every", "noise", "once_at", + "repeat", "replay", "schedule", + "topology", + "until", + "with_probability", + "workload", ] diff --git a/eclypse/policies/_filters.py b/eclypse/policies/_filters.py index d4a44a9..4f352f3 100644 --- a/eclypse/policies/_filters.py +++ b/eclypse/policies/_filters.py @@ -22,7 +22,16 @@ def iter_selected_nodes( node_ids: list[str] | None = None, node_filter: NodeFilter | None = None, ) -> list[tuple[str, dict[str, Any]]]: - """Yield nodes matching the provided selectors.""" + """Return nodes matching the provided selectors. + + Args: + graph (AssetGraph): Asset graph to inspect. + node_ids (list[str] | None): Optional explicit node identifiers to keep. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + + Returns: + Matching ``(node_id, data)`` pairs. + """ selected_node_ids = set(node_ids) if node_ids is not None else None selected_nodes: list[tuple[str, dict[str, Any]]] = [] @@ -42,7 +51,17 @@ def iter_selected_edges( edge_ids: list[tuple[str, str]] | None = None, edge_filter: EdgeFilter | None = None, ) -> list[tuple[str, str, dict[str, Any]]]: - """Yield edges matching the provided selectors.""" + """Return edges matching the provided selectors. + + Args: + graph (AssetGraph): Asset graph to inspect. + edge_ids (list[tuple[str, str]] | None): + Optional explicit ``(source, target)`` pairs to keep. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Matching ``(source, target, data)`` triples. + """ selected_edge_ids = set(edge_ids) if edge_ids is not None else None selected_edges: list[tuple[str, str, dict[str, Any]]] = [] @@ -60,7 +79,16 @@ def iter_selected_keys( data: dict[str, Any], keys: str | list[str] | None = None, ) -> list[str]: - """Yield existing keys selected for a policy operation.""" + """Return existing keys selected for a policy operation. + + Args: + data (dict[str, Any]): Asset mapping to inspect. + keys (str | list[str] | None): + Optional asset key or list of keys. ``None`` selects all keys. + + Returns: + Selected keys that exist in ``data``. + """ selected = normalize_selected_keys(keys) if selected is None: return list(data.keys()) @@ -76,7 +104,14 @@ def iter_selected_keys( def normalize_selected_keys( keys: str | list[str] | None, ) -> list[str] | None: - """Normalise a string-or-list selector to a list of keys.""" + """Normalise a string-or-list selector to a list of keys. + + Args: + keys (str | list[str] | None): Optional asset key selector. + + Returns: + ``None`` when no selector is provided, otherwise a list of keys. + """ if keys is None: return None if isinstance(keys, str): @@ -88,7 +123,15 @@ def effective_assets( assets: str | list[str] | None, per_asset_values: dict[str, Any] | None = None, ) -> list[str]: - """Resolve selected asset keys from explicit selectors and per-asset maps.""" + """Resolve selected asset keys from explicit selectors and per-asset maps. + + Args: + assets (str | list[str] | None): Optional asset key selector. + per_asset_values (dict[str, Any] | None): Optional mapping keyed by asset name. + + Returns: + Ordered asset keys selected by either source. + """ selected_assets = list(normalize_selected_keys(assets) or []) for key in per_asset_values or {}: @@ -99,7 +142,15 @@ def effective_assets( def ensure_numeric_value(key: str, value: Any) -> float: - """Return a numeric value or raise a clear error for unsupported assets.""" + """Return a numeric value or raise a clear error for unsupported assets. + + Args: + key (str): Asset name used in error messages. + value (Any): Candidate value to validate. + + Returns: + ``value`` converted to ``float``. + """ if isinstance(value, bool) or not isinstance(value, int | float): raise TypeError( f'Policy expected numeric asset "{key}", got {type(value).__name__}.' @@ -112,7 +163,16 @@ def clamp( lower: float | None = None, upper: float | None = None, ) -> float: - """Clamp a numeric value between optional bounds.""" + """Clamp a numeric value between optional bounds. + + Args: + value (float): Numeric value to clamp. + lower (float | None): Optional lower bound. + upper (float | None): Optional upper bound. + + Returns: + Clamped value. + """ if lower is not None: value = max(lower, value) if upper is not None: @@ -121,7 +181,15 @@ def clamp( def coerce_numeric_like(original: Any, value: float) -> int | float: - """Cast a computed value back to the original numeric kind when possible.""" + """Cast a computed value back to the original numeric kind when possible. + + Args: + original (Any): Original asset value. + value (float): Computed numeric value. + + Returns: + Rounded integer when ``original`` is an integer, otherwise ``value``. + """ if isinstance(original, bool): return value if isinstance(original, int): @@ -129,9 +197,77 @@ def coerce_numeric_like(original: Any, value: float) -> int | float: return value +def apply_numeric_transform( + graph: AssetGraph, + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, + transform: Callable[[str, float], float], +) -> None: + """Apply a numeric transform to selected node and edge assets. + + Args: + graph (AssetGraph): Asset graph to mutate. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + transform (Callable[[str, float], float]): + Callable receiving ``(asset_key, current_value)``. + + Returns: + None. + """ + if node_assets is not None: + for _, data in iter_selected_nodes( + graph, + node_ids=node_ids, + node_filter=node_filter, + ): + apply_numeric_transform_to_values(data, node_assets, transform=transform) + + if edge_assets is not None: + for _, _, data in iter_selected_edges( + graph, + edge_ids=edge_ids, + edge_filter=edge_filter, + ): + apply_numeric_transform_to_values(data, edge_assets, transform=transform) + + +def apply_numeric_transform_to_values( + data: dict[str, Any], + assets: str | list[str] | None, + *, + transform: Callable[[str, float], float], +) -> None: + """Apply a numeric transform to selected keys in one asset mapping. + + Args: + data (dict[str, Any]): Asset mapping to mutate. + assets (str | list[str] | None): Asset key selector. ``None`` selects all existing keys. + transform (Callable[[str, float], float]): + Callable receiving ``(asset_key, current_value)``. + + Returns: + None. + """ + for key in iter_selected_keys(data, assets): + current = ensure_numeric_value(key, data[key]) + data[key] = coerce_numeric_like(data[key], transform(key, current)) + + __all__ = [ "EdgeFilter", "NodeFilter", + "apply_numeric_transform", + "apply_numeric_transform_to_values", "clamp", "coerce_numeric_like", "effective_assets", diff --git a/eclypse/policies/_helpers.py b/eclypse/policies/_helpers.py new file mode 100644 index 0000000..7f788e9 --- /dev/null +++ b/eclypse/policies/_helpers.py @@ -0,0 +1,38 @@ +"""Shared helpers for built-in policies.""" + +from __future__ import annotations + + +def validate_probability(name: str, value: float | None) -> None: + """Validate an optional probability value. + + Args: + name (str): Parameter name used in validation errors. + value (float | None): Probability value to validate. ``None`` is accepted. + + Returns: + None. + """ + if value is None: + return + if not 0 <= value <= 1: + raise ValueError(f"{name} must be between 0 and 1.") + + +def validate_missing_behaviour(missing: str) -> None: + """Validate the behaviour used for missing graph items. + + Args: + missing (str): Missing-item behaviour to validate. + + Returns: + None. + """ + if missing not in {"ignore", "error"}: + raise ValueError('missing must be either "ignore" or "error".') + + +__all__ = [ + "validate_missing_behaviour", + "validate_probability", +] diff --git a/eclypse/policies/compose/__init__.py b/eclypse/policies/compose/__init__.py new file mode 100644 index 0000000..14bd708 --- /dev/null +++ b/eclypse/policies/compose/__init__.py @@ -0,0 +1,15 @@ +"""Policy composition helpers.""" + +from .all_of import all_of +from .chain import chain +from .conditional import conditional +from .one_of import one_of +from .weighted_choice import weighted_choice + +__all__ = [ + "all_of", + "chain", + "conditional", + "one_of", + "weighted_choice", +] diff --git a/eclypse/policies/compose/all_of.py b/eclypse/policies/compose/all_of.py new file mode 100644 index 0000000..686775b --- /dev/null +++ b/eclypse/policies/compose/all_of.py @@ -0,0 +1,22 @@ +"""Composition policy that applies all children.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.compose.chain import chain + +if TYPE_CHECKING: + from eclypse.utils.types import UpdatePolicy + + +def all_of(*policies: UpdatePolicy) -> UpdatePolicy: + """Run all policies in order. + + Args: + policies (UpdatePolicy): Policies to call in order. + + Returns: + Composed policy. + """ + return chain(*policies) diff --git a/eclypse/policies/compose/chain.py b/eclypse/policies/compose/chain.py new file mode 100644 index 0000000..a665f30 --- /dev/null +++ b/eclypse/policies/compose/chain.py @@ -0,0 +1,26 @@ +"""Sequential policy composition.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +def chain(*policies: UpdatePolicy) -> UpdatePolicy: + """Run policies in the provided order. + + Args: + policies (UpdatePolicy): Policies to call in order. + + Returns: + Composed policy. + """ + + def policy(graph: AssetGraph): + for child_policy in policies: + child_policy(graph) + + return policy diff --git a/eclypse/policies/compose/conditional.py b/eclypse/policies/compose/conditional.py new file mode 100644 index 0000000..d8ed848 --- /dev/null +++ b/eclypse/policies/compose/conditional.py @@ -0,0 +1,33 @@ +"""Conditional policy composition.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +def conditional( + predicate: Callable[[AssetGraph], bool], + policy: UpdatePolicy, +) -> UpdatePolicy: + """Run ``policy`` only when ``predicate(graph)`` is true. + + Args: + predicate (Callable[[AssetGraph], bool]): + Callable receiving the graph and returning a truthy value. + policy (UpdatePolicy): Wrapped policy to call when ``predicate`` passes. + + Returns: + Conditional policy. + """ + + def wrapped(graph: AssetGraph): + if predicate(graph): + policy(graph) + + return wrapped diff --git a/eclypse/policies/compose/one_of.py b/eclypse/policies/compose/one_of.py new file mode 100644 index 0000000..7cbb7d4 --- /dev/null +++ b/eclypse/policies/compose/one_of.py @@ -0,0 +1,27 @@ +"""Random uniform policy composition.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +def one_of(*policies: UpdatePolicy) -> UpdatePolicy: + """Run one uniformly sampled policy. + + Args: + policies (UpdatePolicy): Candidate policies to sample from. + + Returns: + Policy that calls one sampled child policy. + """ + if not policies: + raise ValueError("At least one policy must be provided.") + + def policy(graph: AssetGraph): + graph.rnd.choice(policies)(graph) + + return policy diff --git a/eclypse/policies/compose/weighted_choice.py b/eclypse/policies/compose/weighted_choice.py new file mode 100644 index 0000000..bc0752b --- /dev/null +++ b/eclypse/policies/compose/weighted_choice.py @@ -0,0 +1,35 @@ +"""Random weighted policy composition.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +def weighted_choice( + policies: list[UpdatePolicy] | tuple[UpdatePolicy, ...], + weights: list[float] | tuple[float, ...], +) -> UpdatePolicy: + """Run one policy sampled from explicit weights. + + Args: + policies (list[UpdatePolicy] | tuple[UpdatePolicy, ...]): Candidate policies to sample from. + weights (list[float] | tuple[float, ...]): Sampling weights aligned with ``policies``. + + Returns: + Policy that calls one weighted-sampled child policy. + """ + if len(policies) != len(weights): + raise ValueError("weights must match policies length.") + if any(weight < 0 for weight in weights): + raise ValueError("weights must be non-negative.") + if sum(weights) <= 0: + raise ValueError("at least one weight must be positive.") + + def policy(graph: AssetGraph): + graph.rnd.choices(policies, weights=weights, k=1)[0](graph) + + return policy diff --git a/eclypse/policies/constraints/__init__.py b/eclypse/policies/constraints/__init__.py new file mode 100644 index 0000000..b56a980 --- /dev/null +++ b/eclypse/policies/constraints/__init__.py @@ -0,0 +1,13 @@ +"""Constraint-enforcing policies.""" + +from .clamp_values import clamp_values +from .ensure_capacity_floor import ensure_capacity_floor +from .normalise import normalise +from .round_int import round_int + +__all__ = [ + "clamp_values", + "ensure_capacity_floor", + "normalise", + "round_int", +] diff --git a/eclypse/policies/constraints/_helpers.py b/eclypse/policies/constraints/_helpers.py new file mode 100644 index 0000000..c0c2207 --- /dev/null +++ b/eclypse/policies/constraints/_helpers.py @@ -0,0 +1,63 @@ +"""Shared helpers for constraint policies.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import apply_numeric_transform + +if TYPE_CHECKING: + from collections.abc import Callable + + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def build_numeric_constraint_policy( + *, + transform: Callable[[str, float], float], + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Build a constraint policy from a numeric transform. + + Args: + transform (Callable[[str, float], float]): + Callable receiving ``(asset_key, current_value)``. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that mutates selected numeric assets. + """ + if node_assets is None and edge_assets is None: + raise ValueError("At least one of node_assets or edge_assets must be provided.") + + def policy(graph: AssetGraph): + apply_numeric_transform( + graph, + node_assets=node_assets, + edge_assets=edge_assets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + transform=transform, + ) + + return policy + + +__all__ = ["build_numeric_constraint_policy"] diff --git a/eclypse/policies/constraints/clamp_values.py b/eclypse/policies/constraints/clamp_values.py new file mode 100644 index 0000000..45123de --- /dev/null +++ b/eclypse/policies/constraints/clamp_values.py @@ -0,0 +1,55 @@ +"""Clamp constraint policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import clamp +from eclypse.policies.constraints._helpers import build_numeric_constraint_policy + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def clamp_values( + *, + lower: float | None = None, + upper: float | None = None, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Clamp selected assets to optional bounds. + + Args: + lower (float | None): Optional lower bound. + upper (float | None): Optional upper bound. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that clamps selected numeric assets. + """ + if lower is not None and upper is not None and lower > upper: + raise ValueError("lower must be less than or equal to upper.") + + return build_numeric_constraint_policy( + node_assets=node_assets, + edge_assets=edge_assets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + transform=lambda _key, value: clamp(value, lower, upper), + ) diff --git a/eclypse/policies/constraints/ensure_capacity_floor.py b/eclypse/policies/constraints/ensure_capacity_floor.py new file mode 100644 index 0000000..764d0eb --- /dev/null +++ b/eclypse/policies/constraints/ensure_capacity_floor.py @@ -0,0 +1,29 @@ +"""Capacity floor constraint policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.constraints.clamp_values import clamp_values + +if TYPE_CHECKING: + from eclypse.utils.types import UpdatePolicy + + +def ensure_capacity_floor( + floor: float, + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, +) -> UpdatePolicy: + """Ensure selected capacity-like assets do not go below ``floor``. + + Args: + floor (float): Minimum allowed numeric value. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + + Returns: + Policy that enforces the configured floor. + """ + return clamp_values(lower=floor, node_assets=node_assets, edge_assets=edge_assets) diff --git a/eclypse/policies/constraints/normalise.py b/eclypse/policies/constraints/normalise.py new file mode 100644 index 0000000..cafc492 --- /dev/null +++ b/eclypse/policies/constraints/normalise.py @@ -0,0 +1,60 @@ +"""Normalisation constraint policy.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Any, +) + +from eclypse.policies._filters import ( + coerce_numeric_like, + ensure_numeric_value, + iter_selected_keys, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +def normalise( + total: float, + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, +) -> UpdatePolicy: + """Scale selected values so their graph-wide sum equals ``total``. + + Args: + total (float): Desired sum across all selected assets. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + + Returns: + Policy that rescales selected numeric assets. + """ + + def policy(graph: AssetGraph): + selected: list[tuple[dict[str, Any], str]] = [] + if node_assets is not None: + for _, data in graph.nodes.data(): + selected.extend( + (data, key) for key in iter_selected_keys(data, node_assets) + ) + if edge_assets is not None: + for _, _, data in graph.edges.data(): + selected.extend( + (data, key) for key in iter_selected_keys(data, edge_assets) + ) + current_total = 0.0 + for data, key in selected: + current_total += ensure_numeric_value(key, data[key]) + if current_total == 0: + return + factor = total / current_total + for data, key in selected: + value = ensure_numeric_value(key, data[key]) * factor + data[key] = coerce_numeric_like(data[key], value) + + return policy diff --git a/eclypse/policies/constraints/round_int.py b/eclypse/policies/constraints/round_int.py new file mode 100644 index 0000000..d181322 --- /dev/null +++ b/eclypse/policies/constraints/round_int.py @@ -0,0 +1,31 @@ +"""Integer rounding constraint policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.constraints._helpers import build_numeric_constraint_policy + +if TYPE_CHECKING: + from eclypse.utils.types import UpdatePolicy + + +def round_int( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, +) -> UpdatePolicy: + """Round selected numeric values to integers. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + + Returns: + Policy that rounds selected numeric assets. + """ + return build_numeric_constraint_policy( + node_assets=node_assets, + edge_assets=edge_assets, + transform=lambda _key, value: round(value), + ) diff --git a/eclypse/policies/degrade/__init__.py b/eclypse/policies/degrade/__init__.py index 68eebf0..cc6d284 100644 --- a/eclypse/policies/degrade/__init__.py +++ b/eclypse/policies/degrade/__init__.py @@ -2,10 +2,22 @@ from __future__ import annotations +from .clamp_values import clamp_values +from .decay import decay from .increase import increase +from .ramp_to import ramp_to from .reduce import reduce +from .restore import restore +from .scale import scale +from .set_value import set_value __all__ = [ + "clamp_values", + "decay", "increase", + "ramp_to", "reduce", + "restore", + "scale", + "set_value", ] diff --git a/eclypse/policies/degrade/_helpers.py b/eclypse/policies/degrade/_helpers.py index d0813fd..e02422a 100644 --- a/eclypse/policies/degrade/_helpers.py +++ b/eclypse/policies/degrade/_helpers.py @@ -11,6 +11,7 @@ ) from eclypse.policies._filters import ( + apply_numeric_transform, coerce_numeric_like, effective_assets, ensure_numeric_value, @@ -35,7 +36,7 @@ @dataclass(slots=True) -class ValueAdjustmentPolicy: +class _ValueAdjustmentPolicy: """Adjust selected asset values over a fixed number of epochs.""" direction: ValueAdjustmentDirection @@ -53,7 +54,7 @@ class ValueAdjustmentPolicy: def __post_init__(self): """Validate the value-adjustment configuration.""" - validate_adjustment_parameters( + _validate_adjustment_parameters( self.direction, factor=self.factor, target=self.target, @@ -99,7 +100,7 @@ def __call__(self, graph: AssetGraph): self.step += 1 -def build_value_adjustment_policy( +def _build_value_adjustment_policy( direction: ValueAdjustmentDirection, *, factor: float | None = None, @@ -112,11 +113,28 @@ def build_value_adjustment_policy( edge_ids: list[tuple[str, str]] | None = None, edge_filter: EdgeFilter | None = None, ) -> UpdatePolicy: - """Build a stateful value-adjustment policy.""" + """Build a stateful value-adjustment policy. + + Args: + direction (ValueAdjustmentDirection): + Adjustment direction, either ``"increase"`` or ``"reduce"``. + factor (float | None): Optional multiplicative target factor. + target (float | None): Optional absolute target value. + epochs (int): Number of calls used to complete the adjustment. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Stateful value-adjustment policy. + """ if node_assets is None and edge_assets is None: raise ValueError("At least one of node_assets or edge_assets must be provided.") - return ValueAdjustmentPolicy( + return _ValueAdjustmentPolicy( direction=direction, factor=factor, target=target, @@ -145,7 +163,28 @@ def build_configured_value_adjustment_policy( edge_ids: list[tuple[str, str]] | None = None, edge_filter: EdgeFilter | None = None, ) -> UpdatePolicy: - """Build a value-adjustment policy with defaults and per-asset overrides.""" + """Build a value-adjustment policy with defaults and per-asset overrides. + + Args: + direction (ValueAdjustmentDirection): + Adjustment direction, either ``"increase"`` or ``"reduce"``. + factor (float | None): Optional default multiplicative target factor. + target (float | None): Optional default absolute target value. + epochs (int | None): Optional default number of calls used for adjustment. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_asset_overrides (ValueAdjustmentOverrides | None): + Optional per-node-asset adjustment overrides. + edge_asset_overrides (ValueAdjustmentOverrides | None): + Optional per-edge-asset adjustment overrides. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that applies one child adjustment per selected asset. + """ effective_node_assets = effective_assets(node_assets, node_asset_overrides) effective_edge_assets = effective_assets(edge_assets, edge_asset_overrides) @@ -155,18 +194,18 @@ def build_configured_value_adjustment_policy( "node_asset_overrides, or edge_asset_overrides must be provided." ) - validate_overrides( + _validate_overrides( direction, { - **normalize_overrides("node_asset_overrides", node_asset_overrides), - **normalize_overrides("edge_asset_overrides", edge_asset_overrides), + **_normalize_overrides("node_asset_overrides", node_asset_overrides), + **_normalize_overrides("edge_asset_overrides", edge_asset_overrides), }, ) child_policies: list[UpdatePolicy] = [] for asset in effective_node_assets: - adjustment = resolve_adjustment( + adjustment = _resolve_adjustment( direction, asset_name=asset, factor=factor, @@ -175,7 +214,7 @@ def build_configured_value_adjustment_policy( per_asset_overrides=node_asset_overrides, ) child_policies.append( - build_value_adjustment_policy( + _build_value_adjustment_policy( direction, factor=adjustment.get("factor"), target=adjustment.get("target"), @@ -187,7 +226,7 @@ def build_configured_value_adjustment_policy( ) for asset in effective_edge_assets: - adjustment = resolve_adjustment( + adjustment = _resolve_adjustment( direction, asset_name=asset, factor=factor, @@ -196,7 +235,7 @@ def build_configured_value_adjustment_policy( per_asset_overrides=edge_asset_overrides, ) child_policies.append( - build_value_adjustment_policy( + _build_value_adjustment_policy( direction, factor=adjustment.get("factor"), target=adjustment.get("target"), @@ -216,14 +255,24 @@ def policy(graph: AssetGraph): return policy -def validate_adjustment_parameters( +def _validate_adjustment_parameters( direction: ValueAdjustmentDirection, *, factor: float | None, target: float | None, epochs: int | None, ) -> None: - """Validate a value-adjustment policy configuration.""" + """Validate a value-adjustment policy configuration. + + Args: + direction (ValueAdjustmentDirection): Adjustment direction to validate. + factor (float | None): Optional multiplicative target factor. + target (float | None): Optional absolute target value. + epochs (int | None): Optional number of calls used for adjustment. + + Returns: + None. + """ if epochs is None: raise ValueError("epochs must be provided.") if epochs <= 0: @@ -243,11 +292,19 @@ def validate_adjustment_parameters( raise ValueError("target must be non-negative.") -def normalize_overrides( +def _normalize_overrides( name: str, overrides: ValueAdjustmentOverrides | None, ) -> dict[str, ValueAdjustmentOverride]: - """Normalise one or more named overrides into a flat mapping.""" + """Normalise one or more named overrides into a flat mapping. + + Args: + name (str): Name assigned to the override mapping. + overrides (ValueAdjustmentOverrides | None): Optional mapping of asset names to overrides. + + Returns: + Flat mapping from display names to overrides. + """ if overrides is None: return {} @@ -257,14 +314,22 @@ def normalize_overrides( } -def validate_overrides( +def _validate_overrides( direction: ValueAdjustmentDirection, overrides: dict[str, ValueAdjustmentOverride], ) -> None: - """Validate one or more named value-adjustment overrides.""" + """Validate one or more named value-adjustment overrides. + + Args: + direction (ValueAdjustmentDirection): Adjustment direction to validate against. + overrides (dict[str, ValueAdjustmentOverride]): Mapping from display names to overrides. + + Returns: + None. + """ for name, adjustment in overrides.items(): _ensure_only_supported_adjustment_fields(name, adjustment) - validate_adjustment_parameters( + _validate_adjustment_parameters( direction, factor=adjustment.get("factor"), target=adjustment.get("target"), @@ -272,7 +337,7 @@ def validate_overrides( ) -def resolve_adjustment( +def _resolve_adjustment( direction: ValueAdjustmentDirection, *, asset_name: str, @@ -281,7 +346,19 @@ def resolve_adjustment( epochs: int | None, per_asset_overrides: ValueAdjustmentOverrides | None, ) -> ValueAdjustmentOverride: - """Merge default and per-asset override settings for a selected asset.""" + """Merge default and per-asset override settings for a selected asset. + + Args: + direction (ValueAdjustmentDirection): Adjustment direction to validate against. + asset_name (str): Asset whose settings are being resolved. + factor (float | None): Optional default multiplicative target factor. + target (float | None): Optional default absolute target value. + epochs (int | None): Optional default number of calls used for adjustment. + per_asset_overrides (ValueAdjustmentOverrides | None): Optional per-asset override mapping. + + Returns: + Resolved value-adjustment override. + """ adjustment: ValueAdjustmentOverride = {} if factor is not None: @@ -297,7 +374,7 @@ def resolve_adjustment( if "target" in override: adjustment.pop("factor", None) adjustment.update(override) - validate_overrides(direction, {asset_name: adjustment}) + _validate_overrides(direction, {asset_name: adjustment}) return adjustment @@ -317,7 +394,7 @@ def _adjust_value( original: object, current: float, state_key: tuple[str, ...], - policy: ValueAdjustmentPolicy, + policy: _ValueAdjustmentPolicy, ) -> int | float: if policy.factor is not None: step_factor = policy.factor ** (1 / policy.epochs) @@ -350,7 +427,62 @@ def interpolate_value( target_value: float, progress: float, ) -> float: - """Interpolate smoothly between an initial value and a target.""" + """Interpolate smoothly between an initial value and a target. + + Args: + initial_value (float): Value at progress ``0``. + target_value (float): Value at progress ``1``. + progress (float): Interpolation progress between ``0`` and ``1``. + + Returns: + Interpolated value. + """ if initial_value > 0 and target_value > 0: return initial_value * ((target_value / initial_value) ** progress) return initial_value + ((target_value - initial_value) * progress) + + +def build_asset_transform_policy( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, + transform, + label: str, +) -> UpdatePolicy: + """Build a stateless value-transform policy for selected assets. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + transform (Any): Callable receiving ``(asset_key, current_value)``. + label (str): Trace-log label for the generated policy. + + Returns: + Policy that mutates selected numeric assets. + """ + if node_assets is None and edge_assets is None: + raise ValueError("At least one of node_assets or edge_assets must be provided.") + + def policy(graph: AssetGraph): + apply_numeric_transform( + graph, + node_assets=node_assets, + edge_assets=edge_assets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + transform=transform, + ) + + graph.logger.trace(f"Applied {label} value policy.") + + return policy diff --git a/eclypse/policies/degrade/clamp_values.py b/eclypse/policies/degrade/clamp_values.py new file mode 100644 index 0000000..4f9c2cb --- /dev/null +++ b/eclypse/policies/degrade/clamp_values.py @@ -0,0 +1,55 @@ +"""Clamp selected asset values.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import clamp +from eclypse.policies.degrade._helpers import build_asset_transform_policy + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def clamp_values( + *, + lower: float | None = None, + upper: float | None = None, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Clamp selected asset values to optional lower and upper bounds. + + Args: + lower (float | None): Optional lower bound. + upper (float | None): Optional upper bound. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that clamps selected numeric assets. + """ + if lower is not None and upper is not None and lower > upper: + raise ValueError("lower must be less than or equal to upper.") + return build_asset_transform_policy( + node_assets=node_assets, + edge_assets=edge_assets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + transform=lambda _key, current: clamp(current, lower=lower, upper=upper), + label="clamp_values", + ) diff --git a/eclypse/policies/degrade/decay.py b/eclypse/policies/degrade/decay.py new file mode 100644 index 0000000..4af27f8 --- /dev/null +++ b/eclypse/policies/degrade/decay.py @@ -0,0 +1,51 @@ +"""Decay selected asset values.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.degrade.scale import scale + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def decay( + factor: float, + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Multiply selected asset values by a decay factor on each call. + + Args: + factor (float): Multiplicative decay factor between ``0`` and ``1``. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that decays selected numeric assets. + """ + if not 0 <= factor <= 1: + raise ValueError("factor must be between 0 and 1.") + return scale( + factor, + node_assets=node_assets, + edge_assets=edge_assets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/degrade/increase.py b/eclypse/policies/degrade/increase.py index 030ed83..3a99061 100644 --- a/eclypse/policies/degrade/increase.py +++ b/eclypse/policies/degrade/increase.py @@ -41,23 +41,25 @@ def increase( ``edge_asset_overrides`` for specific assets. Args: - factor: Relative multiplicative factor applied to each selected asset. + factor (float | None): Relative multiplicative factor applied to each selected asset. Provide either ``factor`` or ``target``. - target: Absolute value reached by each selected asset at the end of the + target (float | None): Absolute value reached by each selected asset at the end of the adjustment horizon. Provide either ``factor`` or ``target``. - epochs: Number of evolution steps over which the increase is applied. - node_assets: Node asset names using the default adjustment + epochs (int | None): Number of evolution steps over which the increase is applied. + node_assets (str | list[str] | None): Node asset names using the default adjustment configuration. - edge_assets: Edge asset names using the default adjustment + edge_assets (str | list[str] | None): Edge asset names using the default adjustment configuration. - node_asset_overrides: Per-node-asset overrides for ``factor``, + node_asset_overrides (ValueAdjustmentOverrides | None): + Per-node-asset overrides for ``factor``, ``target``, or ``epochs``. - edge_asset_overrides: Per-edge-asset overrides for ``factor``, + edge_asset_overrides (ValueAdjustmentOverrides | None): + Per-edge-asset overrides for ``factor``, ``target``, or ``epochs``. - node_ids: Optional subset of node identifiers to update. - node_filter: Optional predicate used to select nodes dynamically. - edge_ids: Optional subset of edge identifiers to update. - edge_filter: Optional predicate used to select edges dynamically. + node_ids (list[str] | None): Optional subset of node identifiers to update. + node_filter (NodeFilter | None): Optional predicate used to select nodes dynamically. + edge_ids (list[tuple[str, str]] | None): Optional subset of edge identifiers to update. + edge_filter (EdgeFilter | None): Optional predicate used to select edges dynamically. Returns: A graph update policy that increases the selected asset values. diff --git a/eclypse/policies/degrade/ramp_to.py b/eclypse/policies/degrade/ramp_to.py new file mode 100644 index 0000000..aaa5b62 --- /dev/null +++ b/eclypse/policies/degrade/ramp_to.py @@ -0,0 +1,154 @@ +"""Ramp selected assets to a target.""" + +from __future__ import annotations + +from dataclasses import ( + dataclass, + field, +) +from typing import ( + TYPE_CHECKING, + Any, +) + +from eclypse.policies._filters import ( + coerce_numeric_like, + ensure_numeric_value, + iter_selected_edges, + iter_selected_keys, + iter_selected_nodes, +) +from eclypse.policies.degrade._helpers import interpolate_value + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class RampToPolicy: + """Move selected asset values to a target over a fixed number of epochs.""" + + target: float + epochs: int + node_assets: str | list[str] | None = None + edge_assets: str | list[str] | None = None + node_targets: dict[str, float] | None = None + edge_targets: dict[str, float] | None = None + node_ids: list[str] | None = None + node_filter: NodeFilter | None = None + edge_ids: list[tuple[str, str]] | None = None + edge_filter: EdgeFilter | None = None + step: int = 0 + initial_values: dict[tuple[str, ...], float] = field(default_factory=dict) + + def __post_init__(self): + """Validate the ramp configuration.""" + if self.epochs <= 0: + raise ValueError("epochs must be strictly positive.") + if self.node_assets is None and self.edge_assets is None: + raise ValueError( + "At least one of node_assets or edge_assets must be provided." + ) + + def __call__(self, graph: AssetGraph): + """Apply one ramp step to selected assets.""" + if self.step >= self.epochs: + return + + if self.node_assets is not None: + for node_id, data in iter_selected_nodes( + graph, + node_ids=self.node_ids, + node_filter=self.node_filter, + ): + for key in iter_selected_keys(data, self.node_assets): + _ramp_value( + data, key, ("node", node_id, key), self.node_targets, self + ) + + if self.edge_assets is not None: + for source, target, data in iter_selected_edges( + graph, + edge_ids=self.edge_ids, + edge_filter=self.edge_filter, + ): + for key in iter_selected_keys(data, self.edge_assets): + _ramp_value( + data, + key, + ("edge", source, target, key), + self.edge_targets, + self, + ) + + self.step += 1 + graph.logger.trace("Applied ramp_to value policy.") + + +def ramp_to( + target: float, + *, + epochs: int, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_targets: dict[str, float] | None = None, + edge_targets: dict[str, float] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Move selected assets linearly toward ``target`` over ``epochs`` calls. + + Args: + target (float): Default target value reached after ``epochs`` calls. + epochs (int): Number of calls used to complete the ramp. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_targets (dict[str, float] | None): Optional per-node-asset target overrides. + edge_targets (dict[str, float] | None): Optional per-edge-asset target overrides. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Stateful policy that ramps selected numeric assets. + """ + return RampToPolicy( + target=target, + epochs=epochs, + node_assets=node_assets, + edge_assets=edge_assets, + node_targets=node_targets, + edge_targets=edge_targets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) + + +def _ramp_value( + data: dict[str, Any], + key: str, + state_key: tuple[str, ...], + configured_targets: dict[str, float] | None, + policy: RampToPolicy, +): + current = ensure_numeric_value(key, data[key]) + initial = policy.initial_values.setdefault(state_key, current) + target = ( + configured_targets[key] + if configured_targets is not None and key in configured_targets + else policy.target + ) + progress = min(policy.step + 1, policy.epochs) / policy.epochs + data[key] = coerce_numeric_like( + data[key], interpolate_value(initial, target, progress) + ) diff --git a/eclypse/policies/degrade/reduce.py b/eclypse/policies/degrade/reduce.py index faee0af..1e9de04 100644 --- a/eclypse/policies/degrade/reduce.py +++ b/eclypse/policies/degrade/reduce.py @@ -41,23 +41,25 @@ def reduce( ``edge_asset_overrides`` for specific assets. Args: - factor: Relative multiplicative factor applied to each selected asset. + factor (float | None): Relative multiplicative factor applied to each selected asset. Provide either ``factor`` or ``target``. - target: Absolute value reached by each selected asset at the end of the + target (float | None): Absolute value reached by each selected asset at the end of the adjustment horizon. Provide either ``factor`` or ``target``. - epochs: Number of evolution steps over which the reduction is applied. - node_assets: Node asset names using the default adjustment + epochs (int | None): Number of evolution steps over which the reduction is applied. + node_assets (str | list[str] | None): Node asset names using the default adjustment configuration. - edge_assets: Edge asset names using the default adjustment + edge_assets (str | list[str] | None): Edge asset names using the default adjustment configuration. - node_asset_overrides: Per-node-asset overrides for ``factor``, + node_asset_overrides (ValueAdjustmentOverrides | None): + Per-node-asset overrides for ``factor``, ``target``, or ``epochs``. - edge_asset_overrides: Per-edge-asset overrides for ``factor``, + edge_asset_overrides (ValueAdjustmentOverrides | None): + Per-edge-asset overrides for ``factor``, ``target``, or ``epochs``. - node_ids: Optional subset of node identifiers to update. - node_filter: Optional predicate used to select nodes dynamically. - edge_ids: Optional subset of edge identifiers to update. - edge_filter: Optional predicate used to select edges dynamically. + node_ids (list[str] | None): Optional subset of node identifiers to update. + node_filter (NodeFilter | None): Optional predicate used to select nodes dynamically. + edge_ids (list[tuple[str, str]] | None): Optional subset of edge identifiers to update. + edge_filter (EdgeFilter | None): Optional predicate used to select edges dynamically. Returns: A graph update policy that reduces the selected asset values. diff --git a/eclypse/policies/degrade/restore.py b/eclypse/policies/degrade/restore.py new file mode 100644 index 0000000..a607d9e --- /dev/null +++ b/eclypse/policies/degrade/restore.py @@ -0,0 +1,151 @@ +"""Restore selected asset values to baselines.""" + +from __future__ import annotations + +from dataclasses import ( + dataclass, + field, +) +from typing import ( + TYPE_CHECKING, + Any, +) + +from eclypse.policies._filters import ( + coerce_numeric_like, + ensure_numeric_value, + iter_selected_edges, + iter_selected_keys, + iter_selected_nodes, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class RestorePolicy: + """Restore selected asset values to captured or provided baselines.""" + + epochs: int = 1 + node_assets: str | list[str] | None = None + edge_assets: str | list[str] | None = None + node_values: dict[str, float] | None = None + edge_values: dict[str, float] | None = None + node_ids: list[str] | None = None + node_filter: NodeFilter | None = None + edge_ids: list[tuple[str, str]] | None = None + edge_filter: EdgeFilter | None = None + step: int = 0 + baselines: dict[tuple[str, ...], float] = field(default_factory=dict) + + def __post_init__(self): + """Validate the restore configuration.""" + if self.epochs <= 0: + raise ValueError("epochs must be strictly positive.") + if self.node_assets is None and self.edge_assets is None: + raise ValueError( + "At least one of node_assets or edge_assets must be provided." + ) + + def __call__(self, graph: AssetGraph): + """Move selected values towards their baseline.""" + if self.step >= self.epochs: + return + + if self.node_assets is not None: + for node_id, data in iter_selected_nodes( + graph, + node_ids=self.node_ids, + node_filter=self.node_filter, + ): + for key in iter_selected_keys(data, self.node_assets): + _restore_value( + data, + key, + ("node", node_id, key), + self.node_values, + self, + ) + + if self.edge_assets is not None: + for source, target, data in iter_selected_edges( + graph, + edge_ids=self.edge_ids, + edge_filter=self.edge_filter, + ): + for key in iter_selected_keys(data, self.edge_assets): + _restore_value( + data, + key, + ("edge", source, target, key), + self.edge_values, + self, + ) + + self.step += 1 + graph.logger.trace("Applied restore value policy.") + + +def restore( + *, + epochs: int = 1, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_values: dict[str, float] | None = None, + edge_values: dict[str, float] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Restore selected asset values to captured or provided baselines. + + Args: + epochs (int): Number of calls used to complete the restore operation. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_values (dict[str, float] | None): Optional per-node-asset baseline overrides. + edge_values (dict[str, float] | None): Optional per-edge-asset baseline overrides. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Stateful policy that restores selected numeric assets. + """ + return RestorePolicy( + epochs=epochs, + node_assets=node_assets, + edge_assets=edge_assets, + node_values=node_values, + edge_values=edge_values, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) + + +def _restore_value( + data: dict[str, Any], + key: str, + state_key: tuple[str, ...], + configured_values: dict[str, float] | None, + policy: RestorePolicy, +): + current = ensure_numeric_value(key, data[key]) + baseline = ( + configured_values[key] + if configured_values is not None and key in configured_values + else policy.baselines.setdefault(state_key, current) + ) + remaining = policy.epochs - policy.step + new_value = current + (baseline - current) / remaining + data[key] = coerce_numeric_like(data[key], new_value) diff --git a/eclypse/policies/degrade/scale.py b/eclypse/policies/degrade/scale.py new file mode 100644 index 0000000..61a2f8a --- /dev/null +++ b/eclypse/policies/degrade/scale.py @@ -0,0 +1,50 @@ +"""Scale selected asset values.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.degrade._helpers import build_asset_transform_policy + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def scale( + factor: float, + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Multiply selected asset values by ``factor`` immediately. + + Args: + factor (float): Multiplicative factor to apply. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that scales selected numeric assets. + """ + return build_asset_transform_policy( + node_assets=node_assets, + edge_assets=edge_assets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + transform=lambda _key, current: current * factor, + label="scale", + ) diff --git a/eclypse/policies/degrade/set_value.py b/eclypse/policies/degrade/set_value.py new file mode 100644 index 0000000..1466052 --- /dev/null +++ b/eclypse/policies/degrade/set_value.py @@ -0,0 +1,64 @@ +"""Set selected asset values.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.degrade._helpers import build_asset_transform_policy + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def set_value( + value: float, + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_values: dict[str, float] | None = None, + edge_values: dict[str, float] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Assign selected assets to a fixed value or per-asset override. + + Args: + value (float): Default value assigned to each selected asset. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_values (dict[str, float] | None): Optional per-node-asset value overrides. + edge_values (dict[str, float] | None): Optional per-edge-asset value overrides. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that sets selected numeric assets. + """ + selected_node_assets = node_assets or list((node_values or {}).keys()) or None + selected_edge_assets = edge_assets or list((edge_values or {}).keys()) or None + + def transform(key: str, _current: float) -> float: + if node_values is not None and key in node_values: + return node_values[key] + if edge_values is not None and key in edge_values: + return edge_values[key] + return value + + return build_asset_transform_policy( + node_assets=selected_node_assets, + edge_assets=selected_edge_assets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + transform=transform, + label="set_value", + ) diff --git a/eclypse/policies/distribution/__init__.py b/eclypse/policies/distribution/__init__.py index 51a40b4..9a8081d 100644 --- a/eclypse/policies/distribution/__init__.py +++ b/eclypse/policies/distribution/__init__.py @@ -1,21 +1,37 @@ """Distribution-based built-in policies.""" from .beta import beta +from .bernoulli import bernoulli from .categorical import categorical +from .constant import constant +from .discrete import discrete +from .empirical import empirical +from .exponential import exponential from .gamma import gamma from .lognormal import lognormal from .normal import normal +from .pareto import pareto +from .poisson import poisson from .triangular import triangular from .truncated_normal import truncated_normal from .uniform import uniform +from .weibull import weibull __all__ = [ + "bernoulli", "beta", "categorical", + "constant", + "discrete", + "empirical", + "exponential", "gamma", "lognormal", "normal", + "pareto", + "poisson", "triangular", "truncated_normal", "uniform", + "weibull", ] diff --git a/eclypse/policies/distribution/_helpers.py b/eclypse/policies/distribution/_helpers.py index d74823e..5cb59ff 100644 --- a/eclypse/policies/distribution/_helpers.py +++ b/eclypse/policies/distribution/_helpers.py @@ -44,6 +44,12 @@ "must use strictly positive parameters.", ), ], + "exponential": [ + ( + lambda distribution: distribution > 0, + "must use a strictly positive rate.", + ), + ], "lognormal": [ ( lambda distribution: distribution[1] >= 0, @@ -62,6 +68,12 @@ "must be ordered as (low, high).", ), ], + "weibull": [ + ( + lambda distribution: distribution[0] > 0 and distribution[1] > 0, + "must use strictly positive parameters.", + ), + ], } @@ -80,7 +92,29 @@ def build_distribution_policy( edge_ids: list[tuple[str, str]] | None, edge_filter: EdgeFilter | None, ) -> UpdatePolicy: - """Build a distribution-based multiplicative update policy.""" + """Build a distribution-based multiplicative update policy. + + Args: + kind (Distribution): Built-in distribution name. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_distribution (tuple[float, float]): Default node distribution parameters. + edge_distribution (tuple[float, float] | None): + Default edge distribution parameters. When omitted, + ``node_distribution`` is reused. + node_asset_distributions (dict[str, tuple[float, float]] | None): + Optional per-node-asset distributions. + edge_asset_distributions (dict[str, tuple[float, float]] | None): + Optional per-edge-asset distributions. + minimum (float): Lower bound after applying sampled multipliers. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that multiplies selected numeric assets by sampled values. + """ effective_edge_distribution = ( node_distribution if edge_distribution is None else edge_distribution ) @@ -120,7 +154,7 @@ def build_distribution_policy( node_filter=node_filter, edge_ids=edge_ids, edge_filter=edge_filter, - sampler=lambda rnd, distribution: sample_distribution(rnd, kind, distribution), + sampler=lambda rnd, distribution: _sample_distribution(rnd, kind, distribution), ) @@ -140,7 +174,26 @@ def build_sampled_distribution_policy( edge_filter: EdgeFilter | None, sampler: Any, ) -> UpdatePolicy: - """Build a multiplicative update policy from a custom distribution sampler.""" + """Build a multiplicative update policy from a custom distribution sampler. + + Args: + kind (str): Distribution name used in trace logs. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_distribution (Any): Default node distribution configuration. + edge_distribution (Any): Default edge distribution configuration. + node_asset_distributions (dict[str, Any] | None): Optional per-node-asset distributions. + edge_asset_distributions (dict[str, Any] | None): Optional per-edge-asset distributions. + minimum (float): Lower bound after applying sampled multipliers. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + sampler (Any): Callable receiving ``(random, distribution)``. + + Returns: + Policy that multiplies selected numeric assets by sampled values. + """ effective_node_assets = effective_assets(node_assets, node_asset_distributions) effective_edge_assets = effective_assets(edge_assets, edge_asset_distributions) @@ -150,7 +203,7 @@ def build_sampled_distribution_policy( "node_asset_distributions, or edge_asset_distributions must be provided." ) - log_message = build_distribution_log_message( + log_message = _build_distribution_log_message( kind, node_distribution=node_distribution, edge_distribution=edge_distribution, @@ -200,7 +253,7 @@ def policy(graph: AssetGraph): return policy -def build_distribution_log_message( +def _build_distribution_log_message( kind: str, *, node_distribution: Any, @@ -208,18 +261,37 @@ def build_distribution_log_message( node_asset_distributions: dict[str, Any] | None, edge_asset_distributions: dict[str, Any] | None, ) -> str: - """Build a compact trace message describing a distribution policy.""" + """Build a compact trace message describing a distribution policy. + + Args: + kind (str): Distribution name. + node_distribution (Any): Default node distribution configuration. + edge_distribution (Any): Default edge distribution configuration. + node_asset_distributions (dict[str, Any] | None): Optional per-node-asset distributions. + edge_asset_distributions (dict[str, Any] | None): Optional per-edge-asset distributions. + + Returns: + Trace-log message for the policy. + """ has_overrides = bool(node_asset_distributions or edge_asset_distributions) return ( f"Applied {kind} distribution policy " - f"[node=({describe_distribution(kind, node_distribution)}), " - f"edge=({describe_distribution(kind, edge_distribution)}), " + f"[node=({_describe_distribution(kind, node_distribution)}), " + f"edge=({_describe_distribution(kind, edge_distribution)}), " f"overrides={'yes' if has_overrides else 'no'}]." ) -def describe_distribution(kind: str, distribution: Any) -> str: - """Describe a distribution with kind-appropriate parameter names.""" +def _describe_distribution(kind: str, distribution: Any) -> str: # noqa: C901 + """Describe a distribution with kind-appropriate parameter names. + + Args: + kind (str): Distribution name. + distribution (Any): Distribution configuration to describe. + + Returns: + Human-readable distribution description. + """ description: str if kind == "uniform": @@ -237,11 +309,25 @@ def describe_distribution(kind: str, distribution: Any) -> str: elif kind == "gamma": shape, scale = distribution description = f"shape={shape}, scale={scale}" + elif kind == "exponential": + description = f"lambda={distribution}" + elif kind == "weibull": + alpha, beta_param = distribution + description = f"alpha={alpha}, beta={beta_param}" + elif kind == "pareto": + description = f"alpha={distribution}" + elif kind == "poisson": + description = f"lambda={distribution}" + elif kind == "bernoulli": + probability, success, failure = distribution + description = f"p={probability}, success={success}, failure={failure}" elif kind == "triangular": low, high, mode = distribution description = f"low={low}, high={high}, mode={mode}" elif kind == "categorical": description = f"choices={len(distribution[0])}" + elif kind in {"empirical", "discrete"}: + description = f"choices={len(distribution)}" else: description = str(distribution) @@ -252,7 +338,16 @@ def normalize_distributions( name: str, distributions: Any | dict[str, Any] | None, ) -> dict[str, Any]: - """Normalise one or more named distributions into a flat mapping.""" + """Normalise one or more named distributions into a flat mapping. + + Args: + name (str): Name assigned to scalar distribution values. + distributions (Any | dict[str, Any] | None): + Distribution value, mapping of distributions, or ``None``. + + Returns: + Mapping from display names to distributions. + """ if distributions is None: return {} @@ -270,19 +365,36 @@ def validate_distributions( *, checks: list[tuple[Any, str]], ) -> None: - """Validate one or more named distributions against predicate-based checks.""" + """Validate one or more named distributions against predicate-based checks. + + Args: + distributions (dict[str, Any]): Mapping from display names to distributions. + checks (list[tuple[Any, str]]): Predicate and error-message pairs. + + Returns: + None. + """ for name, distribution in distributions.items(): for predicate, message in checks: if not predicate(distribution): raise ValueError(f"{name} {message}") -def sample_distribution( +def _sample_distribution( rnd: Random, kind: Distribution, distribution: tuple[float, float], ) -> float: - """Sample a multiplier from the requested distribution.""" + """Sample a multiplier from the requested distribution. + + Args: + rnd (Random): Random number generator. + kind (Distribution): Built-in distribution name. + distribution (tuple[float, float]): Distribution parameters. + + Returns: + Sampled multiplier. + """ first, second = distribution if kind == "normal": @@ -297,14 +409,15 @@ def sample_distribution( if kind == "gamma": return rnd.gammavariate(first, second) + if kind == "weibull": + return rnd.weibullvariate(first, second) + return rnd.uniform(first, second) __all__ = [ - "build_distribution_log_message", "build_distribution_policy", "build_sampled_distribution_policy", - "describe_distribution", "normalize_distributions", "validate_distributions", ] diff --git a/eclypse/policies/distribution/bernoulli.py b/eclypse/policies/distribution/bernoulli.py new file mode 100644 index 0000000..854d4f7 --- /dev/null +++ b/eclypse/policies/distribution/bernoulli.py @@ -0,0 +1,104 @@ +"""Bernoulli multiplier distribution policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import ( + build_sampled_distribution_policy, + normalize_distributions, + validate_distributions, +) +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from random import Random + + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + +BernoulliDistribution = tuple[float, float, float] + + +def bernoulli( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: BernoulliDistribution = (0.5, 1.0, 0.0), + edge_distribution: BernoulliDistribution | None = None, + node_asset_distributions: dict[str, BernoulliDistribution] | None = None, + edge_asset_distributions: dict[str, BernoulliDistribution] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Sample success or failure multipliers with a Bernoulli trial. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_distribution (BernoulliDistribution): + Default ``(probability, success, failure)`` tuple for + selected node assets. + edge_distribution (BernoulliDistribution | None): + Default distribution for selected edge assets. When + omitted, ``node_distribution`` is reused. + node_asset_distributions (dict[str, BernoulliDistribution] | None): + Optional per-node-asset distributions. + edge_asset_distributions (dict[str, BernoulliDistribution] | None): + Optional per-edge-asset distributions. + minimum (float): Lower bound after applying the sampled multiplier. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that multiplies selected numeric assets by Bernoulli samples. + """ + effective_edge_distribution = ( + node_distribution if edge_distribution is None else edge_distribution + ) + validate_distributions( + { + **normalize_distributions("node_distribution", node_distribution), + **normalize_distributions("edge_distribution", effective_edge_distribution), + **normalize_distributions( + "node_asset_distributions", node_asset_distributions + ), + **normalize_distributions( + "edge_asset_distributions", edge_asset_distributions + ), + }, + checks=[ + ( + lambda distribution: 0 <= distribution[0] <= 1, + "must use a probability between 0 and 1.", + ), + ], + ) + return build_sampled_distribution_policy( + kind="bernoulli", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=effective_edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + sampler=_sample_bernoulli, + ) + + +def _sample_bernoulli(rnd: Random, distribution: BernoulliDistribution) -> float: + probability, success, failure = distribution + return success if rnd.random() < probability else failure diff --git a/eclypse/policies/distribution/categorical.py b/eclypse/policies/distribution/categorical.py index 74fab4b..fe5b6f2 100644 --- a/eclypse/policies/distribution/categorical.py +++ b/eclypse/policies/distribution/categorical.py @@ -153,7 +153,19 @@ def normalize_weight_sets( asset_distributions: dict[str, list[float]] | None, asset_weights: dict[str, list[float]] | None, ) -> dict[str, tuple[list[float], list[float] | None]]: - """Normalise default and per-asset categorical weights into one mapping.""" + """Normalise default and per-asset categorical weights into one mapping. + + Args: + default_name (str): Display name for the default distribution. + default_distribution (list[float]): Default categorical choices. + default_weights (list[float] | None): Optional default categorical weights. + asset_name (str): Display name for per-asset distributions. + asset_distributions (dict[str, list[float]] | None): Optional per-asset categorical choices. + asset_weights (dict[str, list[float]] | None): Optional per-asset categorical weights. + + Returns: + Mapping from display names to ``(choices, weights)`` pairs. + """ normalized_weights = { default_name: (default_distribution, default_weights), } @@ -176,7 +188,15 @@ def normalize_weight_sets( def validate_weights( weight_sets: dict[str, tuple[list[float], list[float] | None]], ) -> None: - """Validate one or more named categorical weight sets.""" + """Validate one or more named categorical weight sets. + + Args: + weight_sets (dict[str, tuple[list[float], list[float] | None]]): + Mapping from display names to ``(choices, weights)`` pairs. + + Returns: + None. + """ for name, (distribution, weights) in weight_sets.items(): if weights is None: continue diff --git a/eclypse/policies/distribution/constant.py b/eclypse/policies/distribution/constant.py new file mode 100644 index 0000000..279584f --- /dev/null +++ b/eclypse/policies/distribution/constant.py @@ -0,0 +1,68 @@ +"""Constant multiplier distribution policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import build_sampled_distribution_policy +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def constant( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: float = 1.0, + edge_distribution: float | None = None, + node_asset_distributions: dict[str, float] | None = None, + edge_asset_distributions: dict[str, float] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply deterministic multiplicative factors to selected assets. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_distribution (float): Default multiplier for selected node assets. + edge_distribution (float | None): Default multiplier for selected edge assets. When + omitted, ``node_distribution`` is reused. + node_asset_distributions (dict[str, float] | None): Optional per-node-asset multipliers. + edge_asset_distributions (dict[str, float] | None): Optional per-edge-asset multipliers. + minimum (float): Lower bound after applying the multiplier. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that multiplies selected numeric assets. + """ + effective_edge_distribution = ( + node_distribution if edge_distribution is None else edge_distribution + ) + return build_sampled_distribution_policy( + kind="constant", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=effective_edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + sampler=lambda _rnd, distribution: distribution, + ) diff --git a/eclypse/policies/distribution/discrete.py b/eclypse/policies/distribution/discrete.py new file mode 100644 index 0000000..522656b --- /dev/null +++ b/eclypse/policies/distribution/discrete.py @@ -0,0 +1,108 @@ +"""Weighted discrete numeric multiplier distribution policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import ( + build_sampled_distribution_policy, + normalize_distributions, + validate_distributions, +) +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from random import Random + + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + +DiscreteDistribution = list[tuple[float, float]] | tuple[tuple[float, float], ...] + + +def discrete( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: DiscreteDistribution = ((1.0, 1.0),), + edge_distribution: DiscreteDistribution | None = None, + node_asset_distributions: dict[str, DiscreteDistribution] | None = None, + edge_asset_distributions: dict[str, DiscreteDistribution] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Sample numeric multipliers from weighted choices. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_distribution (DiscreteDistribution): + Default ``(value, weight)`` choices for node assets. + edge_distribution (DiscreteDistribution | None): + Default choices for edge assets. When omitted, + ``node_distribution`` is reused. + node_asset_distributions (dict[str, DiscreteDistribution] | None): + Optional per-node-asset choices. + edge_asset_distributions (dict[str, DiscreteDistribution] | None): + Optional per-edge-asset choices. + minimum (float): Lower bound after applying the sampled multiplier. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that multiplies selected numeric assets by discrete samples. + """ + effective_edge_distribution = ( + node_distribution if edge_distribution is None else edge_distribution + ) + validate_distributions( + { + **normalize_distributions("node_distribution", node_distribution), + **normalize_distributions("edge_distribution", effective_edge_distribution), + **normalize_distributions( + "node_asset_distributions", node_asset_distributions + ), + **normalize_distributions( + "edge_asset_distributions", edge_asset_distributions + ), + }, + checks=[ + (lambda distribution: len(distribution) > 0, "must not be empty."), + ( + lambda distribution: all(weight >= 0 for _, weight in distribution), + "must use non-negative weights.", + ), + ( + lambda distribution: sum(weight for _, weight in distribution) > 0, + "must include at least one positive weight.", + ), + ], + ) + return build_sampled_distribution_policy( + kind="discrete", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=tuple(node_distribution), + edge_distribution=tuple(effective_edge_distribution), + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + sampler=_sample_discrete, + ) + + +def _sample_discrete(rnd: Random, distribution: DiscreteDistribution) -> float: + values, weights = zip(*distribution, strict=True) + return rnd.choices(values, weights=weights, k=1)[0] diff --git a/eclypse/policies/distribution/empirical.py b/eclypse/policies/distribution/empirical.py new file mode 100644 index 0000000..bd33486 --- /dev/null +++ b/eclypse/policies/distribution/empirical.py @@ -0,0 +1,95 @@ +"""Empirical multiplier distribution policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import ( + build_sampled_distribution_policy, + normalize_distributions, + validate_distributions, +) +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from random import Random + + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def empirical( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: list[float] | tuple[float, ...] = (1.0,), + edge_distribution: list[float] | tuple[float, ...] | None = None, + node_asset_distributions: dict[str, list[float] | tuple[float, ...]] | None = None, + edge_asset_distributions: dict[str, list[float] | tuple[float, ...]] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Sample multiplicative factors from observed values. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_distribution (list[float] | tuple[float, ...]): + Default observed multipliers for selected node assets. + edge_distribution (list[float] | tuple[float, ...] | None): + Default observed multipliers for edge assets. When + omitted, ``node_distribution`` is reused. + node_asset_distributions (dict[str, list[float] | tuple[float, ...]] | None): + Optional per-node-asset observations. + edge_asset_distributions (dict[str, list[float] | tuple[float, ...]] | None): + Optional per-edge-asset observations. + minimum (float): Lower bound after applying the sampled multiplier. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that multiplies selected numeric assets by empirical samples. + """ + effective_edge_distribution = ( + node_distribution if edge_distribution is None else edge_distribution + ) + validate_distributions( + { + **normalize_distributions("node_distribution", node_distribution), + **normalize_distributions("edge_distribution", effective_edge_distribution), + **normalize_distributions( + "node_asset_distributions", node_asset_distributions + ), + **normalize_distributions( + "edge_asset_distributions", edge_asset_distributions + ), + }, + checks=[(lambda distribution: len(distribution) > 0, "must not be empty.")], + ) + return build_sampled_distribution_policy( + kind="empirical", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=tuple(node_distribution), + edge_distribution=tuple(effective_edge_distribution), + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + sampler=_sample_empirical, + ) + + +def _sample_empirical(rnd: Random, distribution) -> float: + return rnd.choice(tuple(distribution)) diff --git a/eclypse/policies/distribution/exponential.py b/eclypse/policies/distribution/exponential.py new file mode 100644 index 0000000..542ec39 --- /dev/null +++ b/eclypse/policies/distribution/exponential.py @@ -0,0 +1,86 @@ +"""Exponential multiplier distribution policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import ( + _BUILTIN_DISTRIBUTION_CHECKS, + build_sampled_distribution_policy, + normalize_distributions, + validate_distributions, +) +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def exponential( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: float = 1.0, + edge_distribution: float | None = None, + node_asset_distributions: dict[str, float] | None = None, + edge_asset_distributions: dict[str, float] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Sample exponential multiplicative factors from ``lambda`` rates. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_distribution (float): Default rate for selected node assets. + edge_distribution (float | None): Default rate for selected edge assets. When omitted, + ``node_distribution`` is reused. + node_asset_distributions (dict[str, float] | None): Optional per-node-asset rates. + edge_asset_distributions (dict[str, float] | None): Optional per-edge-asset rates. + minimum (float): Lower bound after applying the sampled multiplier. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that multiplies selected numeric assets by exponential samples. + """ + effective_edge_distribution = ( + node_distribution if edge_distribution is None else edge_distribution + ) + validate_distributions( + { + **normalize_distributions("node_distribution", node_distribution), + **normalize_distributions("edge_distribution", effective_edge_distribution), + **normalize_distributions( + "node_asset_distributions", node_asset_distributions + ), + **normalize_distributions( + "edge_asset_distributions", edge_asset_distributions + ), + }, + checks=_BUILTIN_DISTRIBUTION_CHECKS["exponential"], + ) + return build_sampled_distribution_policy( + kind="exponential", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=effective_edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + sampler=lambda rnd, distribution: rnd.expovariate(distribution), + ) diff --git a/eclypse/policies/distribution/pareto.py b/eclypse/policies/distribution/pareto.py new file mode 100644 index 0000000..12d1737 --- /dev/null +++ b/eclypse/policies/distribution/pareto.py @@ -0,0 +1,85 @@ +"""Pareto multiplier distribution policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import ( + build_sampled_distribution_policy, + normalize_distributions, + validate_distributions, +) +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def pareto( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: float = 1.0, + edge_distribution: float | None = None, + node_asset_distributions: dict[str, float] | None = None, + edge_asset_distributions: dict[str, float] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Sample Pareto multiplicative factors. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_distribution (float): Default Pareto alpha for selected node assets. + edge_distribution (float | None): Default alpha for selected edge assets. When omitted, + ``node_distribution`` is reused. + node_asset_distributions (dict[str, float] | None): Optional per-node-asset alphas. + edge_asset_distributions (dict[str, float] | None): Optional per-edge-asset alphas. + minimum (float): Lower bound after applying the sampled multiplier. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that multiplies selected numeric assets by Pareto samples. + """ + effective_edge_distribution = ( + node_distribution if edge_distribution is None else edge_distribution + ) + validate_distributions( + { + **normalize_distributions("node_distribution", node_distribution), + **normalize_distributions("edge_distribution", effective_edge_distribution), + **normalize_distributions( + "node_asset_distributions", node_asset_distributions + ), + **normalize_distributions( + "edge_asset_distributions", edge_asset_distributions + ), + }, + checks=[(lambda distribution: distribution > 0, "must be strictly positive.")], + ) + return build_sampled_distribution_policy( + kind="pareto", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=effective_edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + sampler=lambda rnd, distribution: rnd.paretovariate(distribution), + ) diff --git a/eclypse/policies/distribution/poisson.py b/eclypse/policies/distribution/poisson.py new file mode 100644 index 0000000..7ee29f4 --- /dev/null +++ b/eclypse/policies/distribution/poisson.py @@ -0,0 +1,105 @@ +"""Poisson multiplier distribution policy.""" + +from __future__ import annotations + +import math +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import ( + build_sampled_distribution_policy, + normalize_distributions, + validate_distributions, +) +from eclypse.utils.constants import MIN_FLOAT + +_NORMAL_APPROXIMATION_THRESHOLD = 30 + +if TYPE_CHECKING: + from random import Random + + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def poisson( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: float = 1.0, + edge_distribution: float | None = None, + node_asset_distributions: dict[str, float] | None = None, + edge_asset_distributions: dict[str, float] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Sample Poisson multiplicative factors without NumPy. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_distribution (float): Default Poisson lambda for selected node assets. + edge_distribution (float | None): Default lambda for selected edge assets. When omitted, + ``node_distribution`` is reused. + node_asset_distributions (dict[str, float] | None): Optional per-node-asset lambdas. + edge_asset_distributions (dict[str, float] | None): Optional per-edge-asset lambdas. + minimum (float): Lower bound after applying the sampled multiplier. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that multiplies selected numeric assets by Poisson samples. + """ + effective_edge_distribution = ( + node_distribution if edge_distribution is None else edge_distribution + ) + validate_distributions( + { + **normalize_distributions("node_distribution", node_distribution), + **normalize_distributions("edge_distribution", effective_edge_distribution), + **normalize_distributions( + "node_asset_distributions", node_asset_distributions + ), + **normalize_distributions( + "edge_asset_distributions", edge_asset_distributions + ), + }, + checks=[(lambda distribution: distribution >= 0, "must be non-negative.")], + ) + return build_sampled_distribution_policy( + kind="poisson", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=effective_edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + sampler=_sample_poisson, + ) + + +def _sample_poisson(rnd: Random, lam: float) -> int: + if lam == 0: + return 0 + if lam > _NORMAL_APPROXIMATION_THRESHOLD: + return max(0, round(rnd.gauss(lam, math.sqrt(lam)))) + + threshold = math.exp(-lam) + product = 1.0 + value = -1 + while product > threshold: + value += 1 + product *= rnd.random() + return value diff --git a/eclypse/policies/distribution/weibull.py b/eclypse/policies/distribution/weibull.py new file mode 100644 index 0000000..f08a8be --- /dev/null +++ b/eclypse/policies/distribution/weibull.py @@ -0,0 +1,66 @@ +"""Weibull multiplier distribution policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import build_distribution_policy +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def weibull( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: tuple[float, float] = (1.0, 1.0), + edge_distribution: tuple[float, float] | None = None, + node_asset_distributions: dict[str, tuple[float, float]] | None = None, + edge_asset_distributions: dict[str, tuple[float, float]] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Sample Weibull multiplicative factors. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_distribution (tuple[float, float]): Default ``(alpha, beta)`` tuple for node assets. + edge_distribution (tuple[float, float] | None): Default tuple for edge assets. When omitted, + ``node_distribution`` is reused. + node_asset_distributions (dict[str, tuple[float, float]] | None): + Optional per-node-asset distributions. + edge_asset_distributions (dict[str, tuple[float, float]] | None): + Optional per-edge-asset distributions. + minimum (float): Lower bound after applying the sampled multiplier. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that multiplies selected numeric assets by Weibull samples. + """ + return build_distribution_policy( + "weibull", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/failure/__init__.py b/eclypse/policies/failure/__init__.py index b165911..253ef47 100644 --- a/eclypse/policies/failure/__init__.py +++ b/eclypse/policies/failure/__init__.py @@ -3,13 +3,27 @@ from __future__ import annotations from .availability_flap import availability_flap +from .brownout import brownout +from .correlated_failure import correlated_failure +from .edge_availability_flap import edge_availability_flap +from .kill_edges import kill_edges from .kill_nodes import kill_nodes from .latency_spike import latency_spike +from .network_partition import network_partition +from .resource_exhaustion import resource_exhaustion +from .revive_edges import revive_edges from .revive_nodes import revive_nodes __all__ = [ "availability_flap", + "brownout", + "correlated_failure", + "edge_availability_flap", + "kill_edges", "kill_nodes", "latency_spike", + "network_partition", + "resource_exhaustion", + "revive_edges", "revive_nodes", ] diff --git a/eclypse/policies/failure/_helpers.py b/eclypse/policies/failure/_helpers.py index 2c780f8..3df0087 100644 --- a/eclypse/policies/failure/_helpers.py +++ b/eclypse/policies/failure/_helpers.py @@ -1,11 +1,86 @@ -"""Shared helpers for failure-oriented update policies.""" +"""Shared helpers for failure policies.""" from __future__ import annotations +from typing import TYPE_CHECKING -def validate_probability(name: str, value: float | None): - """Validate an optional probability value.""" - if value is None: +from eclypse.policies._filters import ensure_numeric_value + +if TYPE_CHECKING: + from random import Random + + +def set_availability_with_probability( + data: dict[str, object], + *, + probability: float, + availability_key: str, + target_availability: float, + random: Random, +) -> None: + """Assign an availability value when a Bernoulli trial succeeds. + + Args: + data (dict[str, object]): Asset mapping to mutate. + probability (float): Probability of applying the availability change. + availability_key (str): Asset key storing availability. + target_availability (float): Availability value to assign on success. + random (Random): Random number generator. + + Returns: + None. + """ + if random.random() < probability: + data[availability_key] = target_availability + + +def flap_availability( + data: dict[str, object], + *, + down_probability: float, + up_probability: float, + down_availability: float, + up_availability: float, + availability_key: str, + unavailable_at_or_below: float, + random: Random, +) -> None: + """Toggle an availability value up or down according to its current state. + + Args: + data (dict[str, object]): Asset mapping to mutate. + down_probability (float): Probability of moving an available asset down. + up_probability (float): Probability of restoring an unavailable asset. + down_availability (float): Availability value assigned when moving down. + up_availability (float): Availability value assigned when moving up. + availability_key (str): Asset key storing availability. + unavailable_at_or_below (float): Threshold below which the asset is unavailable. + random (Random): Random number generator. + + Returns: + None. + """ + current = ensure_numeric_value(availability_key, data[availability_key]) + if current <= unavailable_at_or_below: + set_availability_with_probability( + data, + probability=up_probability, + availability_key=availability_key, + target_availability=up_availability, + random=random, + ) return - if not 0 <= value <= 1: - raise ValueError(f"{name} must be between 0 and 1.") + + set_availability_with_probability( + data, + probability=down_probability, + availability_key=availability_key, + target_availability=down_availability, + random=random, + ) + + +__all__ = [ + "flap_availability", + "set_availability_with_probability", +] diff --git a/eclypse/policies/failure/availability_flap.py b/eclypse/policies/failure/availability_flap.py index adfd0c3..5fa90d3 100644 --- a/eclypse/policies/failure/availability_flap.py +++ b/eclypse/policies/failure/availability_flap.py @@ -4,11 +4,9 @@ from typing import TYPE_CHECKING -from eclypse.policies._filters import ( - ensure_numeric_value, - iter_selected_nodes, -) -from eclypse.policies.failure._helpers import validate_probability +from eclypse.policies._filters import iter_selected_nodes +from eclypse.policies._helpers import validate_probability +from eclypse.policies.failure._helpers import flap_availability from eclypse.utils.constants import ( MAX_AVAILABILITY, MIN_AVAILABILITY, @@ -60,12 +58,16 @@ def policy(graph: AssetGraph): node_ids=node_ids, node_filter=node_filter, ): - current = ensure_numeric_value(availability_key, data[availability_key]) - if current <= unavailable_at_or_below: - if graph.rnd.random() < effective_up_probability: - data[availability_key] = up_availability - elif graph.rnd.random() < down_probability: - data[availability_key] = down_availability + flap_availability( + data, + down_probability=down_probability, + up_probability=effective_up_probability, + down_availability=down_availability, + up_availability=up_availability, + availability_key=availability_key, + unavailable_at_or_below=unavailable_at_or_below, + random=graph.rnd, + ) graph.logger.trace("Applied availability_flap policy.") diff --git a/eclypse/policies/failure/brownout.py b/eclypse/policies/failure/brownout.py new file mode 100644 index 0000000..51e6497 --- /dev/null +++ b/eclypse/policies/failure/brownout.py @@ -0,0 +1,52 @@ +"""Brownout policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.failure.resource_exhaustion import resource_exhaustion + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def brownout( + probability: float = 1.0, + *, + factor: float = 0.7, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply partial service degradation without a hard failure. + + Args: + probability (float): Per-asset probability of applying the brownout. + factor (float): Multiplicative reduction factor. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that partially reduces selected capacity-like assets. + """ + return resource_exhaustion( + probability, + factor=factor, + node_assets=node_assets, + edge_assets=edge_assets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/failure/correlated_failure.py b/eclypse/policies/failure/correlated_failure.py new file mode 100644 index 0000000..8ac669f --- /dev/null +++ b/eclypse/policies/failure/correlated_failure.py @@ -0,0 +1,62 @@ +"""Correlated node failure policy.""" + +from __future__ import annotations + +from collections import defaultdict +from typing import ( + TYPE_CHECKING, + Any, +) + +from eclypse.policies._filters import iter_selected_nodes +from eclypse.policies._helpers import validate_probability +from eclypse.utils.constants import MIN_AVAILABILITY + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import NodeFilter + from eclypse.utils.types import UpdatePolicy + + +def correlated_failure( + probability: float, + *, + group_key: str, + availability_key: str = "availability", + failed_availability: float = MIN_AVAILABILITY, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, +) -> UpdatePolicy: + """Fail all selected nodes sharing a group value together. + + Args: + probability (float): Per-group probability of applying the failure. + group_key (str): Node asset used to identify correlated groups. + availability_key (str): Node asset used to store availability. + failed_availability (float): Value written when a group fails. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + + Returns: + Policy that fails whole selected node groups. + """ + validate_probability("probability", probability) + + def policy(graph: AssetGraph): + groups: dict[Any, list[dict[str, Any]]] = defaultdict(list) + for _, data in iter_selected_nodes( + graph, + node_ids=node_ids, + node_filter=node_filter, + ): + if group_key in data: + groups[data[group_key]].append(data) + + for nodes in groups.values(): + if graph.rnd.random() < probability: + for data in nodes: + data[availability_key] = failed_availability + + graph.logger.trace("Applied correlated_failure policy.") + + return policy diff --git a/eclypse/policies/failure/edge_availability_flap.py b/eclypse/policies/failure/edge_availability_flap.py new file mode 100644 index 0000000..4d9d23a --- /dev/null +++ b/eclypse/policies/failure/edge_availability_flap.py @@ -0,0 +1,73 @@ +"""Edge availability flapping policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import iter_selected_edges +from eclypse.policies._helpers import validate_probability +from eclypse.policies.failure._helpers import flap_availability +from eclypse.utils.constants import ( + MAX_AVAILABILITY, + MIN_AVAILABILITY, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import EdgeFilter + from eclypse.utils.types import UpdatePolicy + + +def edge_availability_flap( + down_probability: float, + *, + up_probability: float | None = None, + down_availability: float = MIN_AVAILABILITY, + up_availability: float = MAX_AVAILABILITY, + availability_key: str = "availability", + unavailable_at_or_below: float = MIN_AVAILABILITY, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Toggle edge availability up and down according to probabilities. + + Args: + down_probability (float): Probability of moving an available edge down. + up_probability (float | None): Probability of moving an unavailable edge up. When + omitted, ``down_probability`` is reused. + down_availability (float): Availability value for unavailable edges. + up_availability (float): Availability value for recovered edges. + availability_key (str): Edge asset used to store availability. + unavailable_at_or_below (float): Threshold for considering an edge unavailable. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that flips selected edge availability. + """ + validate_probability("down_probability", down_probability) + validate_probability("up_probability", up_probability) + effective_up_probability = ( + down_probability if up_probability is None else up_probability + ) + + def policy(graph: AssetGraph): + for _, _, data in iter_selected_edges( + graph, + edge_ids=edge_ids, + edge_filter=edge_filter, + ): + flap_availability( + data, + down_probability=down_probability, + up_probability=effective_up_probability, + down_availability=down_availability, + up_availability=up_availability, + availability_key=availability_key, + unavailable_at_or_below=unavailable_at_or_below, + random=graph.rnd, + ) + + graph.logger.trace("Applied edge_availability_flap policy.") + + return policy diff --git a/eclypse/policies/failure/kill_edges.py b/eclypse/policies/failure/kill_edges.py new file mode 100644 index 0000000..f611346 --- /dev/null +++ b/eclypse/policies/failure/kill_edges.py @@ -0,0 +1,56 @@ +"""Edge failure policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import iter_selected_edges +from eclypse.policies._helpers import validate_probability +from eclypse.policies.failure._helpers import set_availability_with_probability +from eclypse.utils.constants import MIN_AVAILABILITY + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import EdgeFilter + from eclypse.utils.types import UpdatePolicy + + +def kill_edges( + probability: float, + *, + availability_key: str = "availability", + failed_availability: float = MIN_AVAILABILITY, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Mark selected edges as unavailable according to ``probability``. + + Args: + probability (float): Per-edge probability of applying the failure. + availability_key (str): Edge asset used to store availability. + failed_availability (float): Value written when an edge fails. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that marks selected edges as failed. + """ + validate_probability("probability", probability) + + def policy(graph: AssetGraph): + for _, _, data in iter_selected_edges( + graph, + edge_ids=edge_ids, + edge_filter=edge_filter, + ): + set_availability_with_probability( + data, + probability=probability, + availability_key=availability_key, + target_availability=failed_availability, + random=graph.rnd, + ) + + graph.logger.trace("Applied kill_edges policy.") + + return policy diff --git a/eclypse/policies/failure/kill_nodes.py b/eclypse/policies/failure/kill_nodes.py index d876a9e..fa91ad2 100644 --- a/eclypse/policies/failure/kill_nodes.py +++ b/eclypse/policies/failure/kill_nodes.py @@ -8,7 +8,8 @@ ensure_numeric_value, iter_selected_nodes, ) -from eclypse.policies.failure._helpers import validate_probability +from eclypse.policies._helpers import validate_probability +from eclypse.policies.failure._helpers import set_availability_with_probability from eclypse.utils.constants import MIN_AVAILABILITY if TYPE_CHECKING: @@ -57,12 +58,14 @@ def policy(graph: AssetGraph): ) if graph.rnd.random() < probability: data[availability_key] = down_availability - elif ( - revive_probability is not None - and availability <= down_availability - and graph.rnd.random() < revive_probability - ): - data[availability_key] = revived_availability + elif revive_probability is not None and availability <= down_availability: + set_availability_with_probability( + data, + probability=revive_probability, + availability_key=availability_key, + target_availability=revived_availability, + random=graph.rnd, + ) graph.logger.trace("Applied kill_nodes policy.") diff --git a/eclypse/policies/failure/latency_spike.py b/eclypse/policies/failure/latency_spike.py index d1b06f7..4403316 100644 --- a/eclypse/policies/failure/latency_spike.py +++ b/eclypse/policies/failure/latency_spike.py @@ -10,7 +10,7 @@ ensure_numeric_value, iter_selected_edges, ) -from eclypse.policies.failure._helpers import validate_probability +from eclypse.policies._helpers import validate_probability from eclypse.utils.constants import MIN_LATENCY if TYPE_CHECKING: diff --git a/eclypse/policies/failure/network_partition.py b/eclypse/policies/failure/network_partition.py new file mode 100644 index 0000000..2d74241 --- /dev/null +++ b/eclypse/policies/failure/network_partition.py @@ -0,0 +1,59 @@ +"""Network partition policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.utils.constants import MIN_AVAILABILITY + +_MIN_PARTITIONS = 2 + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +def network_partition( + groups: list[list[str]], + *, + availability_key: str = "availability", + unavailable_value: float = MIN_AVAILABILITY, + remove_edges: bool = False, +) -> UpdatePolicy: + """Partition node groups by disabling or removing cross-group edges. + + Args: + groups (list[list[str]]): Node identifiers grouped by partition. + availability_key (str): Edge asset used when cross-group edges are disabled. + unavailable_value (float): Value written to disabled cross-group edges. + remove_edges (bool): Whether to remove cross-group edges instead of mutating them. + + Returns: + Policy that isolates the configured partitions. + """ + if len(groups) < _MIN_PARTITIONS: + raise ValueError("groups must contain at least two partitions.") + node_to_group = { + node_id: group_idx + for group_idx, group in enumerate(groups) + for node_id in group + } + + def policy(graph: AssetGraph): + for source, target, data in list(graph.edges.data()): + source_group = node_to_group.get(source) + target_group = node_to_group.get(target) + if ( + source_group is None + or target_group is None + or source_group == target_group + ): + continue + if remove_edges: + graph.remove_edge(source, target) + else: + data[availability_key] = unavailable_value + + graph.logger.trace("Applied network_partition policy.") + + return policy diff --git a/eclypse/policies/failure/resource_exhaustion.py b/eclypse/policies/failure/resource_exhaustion.py new file mode 100644 index 0000000..bbadaef --- /dev/null +++ b/eclypse/policies/failure/resource_exhaustion.py @@ -0,0 +1,75 @@ +"""Resource exhaustion policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + apply_numeric_transform, + clamp, +) +from eclypse.policies._helpers import validate_probability + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def resource_exhaustion( + probability: float = 1.0, + *, + factor: float = 0.5, + minimum: float = 0.0, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Reduce selected capacity-like assets according to ``factor``. + + Args: + probability (float): Per-asset probability of applying the reduction. + factor (float): Multiplicative reduction factor. + minimum (float): Lower bound after reduction. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that reduces selected numeric assets. + """ + validate_probability("probability", probability) + if factor < 0: + raise ValueError("factor must be non-negative.") + if node_assets is None and edge_assets is None: + raise ValueError("At least one of node_assets or edge_assets must be provided.") + + def policy(graph: AssetGraph): + def transform(_key: str, current: float) -> float: + if graph.rnd.random() >= probability: + return current + return clamp(current * factor, lower=minimum) + + apply_numeric_transform( + graph, + node_assets=node_assets, + edge_assets=edge_assets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + transform=transform, + ) + + graph.logger.trace("Applied resource_exhaustion policy.") + + return policy diff --git a/eclypse/policies/failure/revive_edges.py b/eclypse/policies/failure/revive_edges.py new file mode 100644 index 0000000..adeb449 --- /dev/null +++ b/eclypse/policies/failure/revive_edges.py @@ -0,0 +1,56 @@ +"""Edge recovery policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import iter_selected_edges +from eclypse.policies._helpers import validate_probability +from eclypse.policies.failure._helpers import set_availability_with_probability +from eclypse.utils.constants import MAX_AVAILABILITY + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import EdgeFilter + from eclypse.utils.types import UpdatePolicy + + +def revive_edges( + probability: float, + *, + availability_key: str = "availability", + revived_availability: float = MAX_AVAILABILITY, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Mark selected edges as available according to ``probability``. + + Args: + probability (float): Per-edge probability of applying the recovery. + availability_key (str): Edge asset used to store availability. + revived_availability (float): Value written when an edge recovers. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that marks selected edges as recovered. + """ + validate_probability("probability", probability) + + def policy(graph: AssetGraph): + for _, _, data in iter_selected_edges( + graph, + edge_ids=edge_ids, + edge_filter=edge_filter, + ): + set_availability_with_probability( + data, + probability=probability, + availability_key=availability_key, + target_availability=revived_availability, + random=graph.rnd, + ) + + graph.logger.trace("Applied revive_edges policy.") + + return policy diff --git a/eclypse/policies/failure/revive_nodes.py b/eclypse/policies/failure/revive_nodes.py index 46cc159..bd03227 100644 --- a/eclypse/policies/failure/revive_nodes.py +++ b/eclypse/policies/failure/revive_nodes.py @@ -8,7 +8,8 @@ ensure_numeric_value, iter_selected_nodes, ) -from eclypse.policies.failure._helpers import validate_probability +from eclypse.policies._helpers import validate_probability +from eclypse.policies.failure._helpers import set_availability_with_probability from eclypse.utils.constants import MIN_AVAILABILITY if TYPE_CHECKING: @@ -49,8 +50,14 @@ def policy(graph: AssetGraph): node_filter=node_filter, ): current = ensure_numeric_value(availability_key, data[availability_key]) - if current <= unavailable_at_or_below and graph.rnd.random() < probability: - data[availability_key] = availability + if current <= unavailable_at_or_below: + set_availability_with_probability( + data, + probability=probability, + availability_key=availability_key, + target_availability=availability, + random=graph.rnd, + ) graph.logger.trace("Applied revive_nodes policy.") diff --git a/eclypse/policies/noise/__init__.py b/eclypse/policies/noise/__init__.py index a5908d8..ac58255 100644 --- a/eclypse/policies/noise/__init__.py +++ b/eclypse/policies/noise/__init__.py @@ -3,12 +3,24 @@ from __future__ import annotations +from .additive_jitter import additive_jitter from .bounded_random_walk import bounded_random_walk +from .correlated_noise import correlated_noise +from .dropout import dropout +from .gaussian_jitter import gaussian_jitter from .impulse import impulse from .momentum_walk import momentum_walk +from .multiplicative_jitter import multiplicative_jitter +from .seasonal_noise import seasonal_noise __all__ = [ + "additive_jitter", "bounded_random_walk", + "correlated_noise", + "dropout", + "gaussian_jitter", "impulse", "momentum_walk", + "multiplicative_jitter", + "seasonal_noise", ] diff --git a/eclypse/policies/noise/_helpers.py b/eclypse/policies/noise/_helpers.py index 4a234f8..dec14b4 100644 --- a/eclypse/policies/noise/_helpers.py +++ b/eclypse/policies/noise/_helpers.py @@ -2,29 +2,30 @@ from __future__ import annotations -from typing import ( - TYPE_CHECKING, - Any, -) +from typing import Any from eclypse.policies._filters import ( clamp, coerce_numeric_like, ensure_numeric_value, - iter_selected_keys, ) from eclypse.utils.constants import MIN_FLOAT -if TYPE_CHECKING: - from random import Random - def validate_steps( *, node_steps: dict[str, float] | None, edge_steps: dict[str, float] | None, ) -> None: - """Validate additive walk step sizes.""" + """Validate additive walk step sizes. + + Args: + node_steps (dict[str, float] | None): Mapping from node asset name to maximum step size. + edge_steps (dict[str, float] | None): Mapping from edge asset name to maximum step size. + + Returns: + None. + """ if not node_steps and not edge_steps: raise ValueError("At least one of node_steps or edge_steps must be provided.") @@ -44,7 +45,18 @@ def apply_additive_walk( *, delta_sampler: Any, ) -> None: - """Apply additive updates sampled independently per configured asset.""" + """Apply additive updates sampled independently per configured asset. + + Args: + values (dict[str, object]): Asset mapping to mutate. + steps (dict[str, float]): Mapping from asset name to maximum step size. + bounds (dict[str, tuple[float | None, float | None]] | None): + Optional mapping from asset name to ``(lower, upper)`` bounds. + delta_sampler (Any): Callable receiving ``(asset_key, step)``. + + Returns: + None. + """ for key, step in steps.items(): if key not in values: continue @@ -58,40 +70,7 @@ def apply_additive_walk( ) -def validate_probability(name: str, probability: float) -> None: - """Validate a probability parameter.""" - if probability < 0 or probability > 1: - raise ValueError(f"{name} must be between 0 and 1.") - - -def validate_factor_range(name: str, factor_range: tuple[float, float]) -> None: - """Validate a multiplicative factor range.""" - lower, upper = factor_range - if lower < 0: - raise ValueError(f"{name} must use non-negative factors.") - if lower > upper: - raise ValueError(f"{name} must be ordered as (low, high).") - - -def apply_impulses( - values: dict[str, object], - assets: str | list[str] | None, - *, - probability: float, - factor_range: tuple[float, float], - minimum: float, - random: Random, -) -> None: - """Apply rare multiplicative shocks to selected numeric assets.""" - lower_factor, upper_factor = factor_range - - for key in iter_selected_keys(values, assets): - if random.random() >= probability: - continue - - current = ensure_numeric_value(key, values[key]) - factor = random.uniform(lower_factor, upper_factor) - values[key] = coerce_numeric_like( - values[key], - clamp(current * factor, lower=minimum), - ) +__all__ = [ + "apply_additive_walk", + "validate_steps", +] diff --git a/eclypse/policies/noise/additive_jitter.py b/eclypse/policies/noise/additive_jitter.py new file mode 100644 index 0000000..7ec3efe --- /dev/null +++ b/eclypse/policies/noise/additive_jitter.py @@ -0,0 +1,86 @@ +"""Additive jitter noise policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + apply_numeric_transform, + clamp, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def additive_jitter( + *, + node_ranges: dict[str, tuple[float, float]] | None = None, + edge_ranges: dict[str, tuple[float, float]] | None = None, + lower: float | None = None, + upper: float | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Add uniformly sampled deltas to selected assets. + + Args: + node_ranges (dict[str, tuple[float, float]] | None): + Mapping from node asset name to ``(low, high)`` delta range. + edge_ranges (dict[str, tuple[float, float]] | None): + Mapping from edge asset name to ``(low, high)`` delta range. + lower (float | None): Optional lower bound after adding noise. + upper (float | None): Optional upper bound after adding noise. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that adds independent uniform jitter to selected assets. + """ + _validate_ranges(node_ranges, edge_ranges) + + def policy(graph: AssetGraph): + def node_transform(key: str, current: float) -> float: + low, high = (node_ranges or {})[key] + return clamp(current + graph.rnd.uniform(low, high), lower, upper) + + def edge_transform(key: str, current: float) -> float: + low, high = (edge_ranges or {})[key] + return clamp(current + graph.rnd.uniform(low, high), lower, upper) + + apply_numeric_transform( + graph, + node_assets=list(node_ranges or {}), + node_ids=node_ids, + node_filter=node_filter, + transform=node_transform, + ) + apply_numeric_transform( + graph, + edge_assets=list(edge_ranges or {}), + edge_ids=edge_ids, + edge_filter=edge_filter, + transform=edge_transform, + ) + + graph.logger.trace("Applied additive_jitter policy.") + + return policy + + +def _validate_ranges(*range_sets): + if all(not range_set for range_set in range_sets): + raise ValueError("At least one range mapping must be provided.") + for range_set in range_sets: + for low, high in (range_set or {}).values(): + if low > high: + raise ValueError("jitter ranges must be ordered as (low, high).") diff --git a/eclypse/policies/noise/correlated_noise.py b/eclypse/policies/noise/correlated_noise.py new file mode 100644 index 0000000..d0f960d --- /dev/null +++ b/eclypse/policies/noise/correlated_noise.py @@ -0,0 +1,70 @@ +"""Correlated additive noise policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + apply_numeric_transform, + clamp, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def correlated_noise( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + delta_range: tuple[float, float] = (-1.0, 1.0), + lower: float | None = None, + upper: float | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply one shared additive delta to all selected assets. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + delta_range (tuple[float, float]): Inclusive range used to sample the shared delta. + lower (float | None): Optional lower bound after adding the delta. + upper (float | None): Optional upper bound after adding the delta. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that adds a shared random delta to selected assets. + """ + low, high = delta_range + if low > high: + raise ValueError("delta_range must be ordered as (low, high).") + if node_assets is None and edge_assets is None: + raise ValueError("At least one of node_assets or edge_assets must be provided.") + + def policy(graph: AssetGraph): + delta = graph.rnd.uniform(low, high) + apply_numeric_transform( + graph, + node_assets=node_assets, + edge_assets=edge_assets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + transform=lambda _key, current: clamp(current + delta, lower, upper), + ) + + graph.logger.trace("Applied correlated_noise policy.") + + return policy diff --git a/eclypse/policies/noise/dropout.py b/eclypse/policies/noise/dropout.py new file mode 100644 index 0000000..07f2862 --- /dev/null +++ b/eclypse/policies/noise/dropout.py @@ -0,0 +1,77 @@ +"""Asset dropout noise policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + iter_selected_edges, + iter_selected_keys, + iter_selected_nodes, +) +from eclypse.policies._helpers import validate_probability + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def dropout( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + probability: float = 0.05, + value: float = 0.0, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Randomly replace selected asset values with ``value``. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + probability (float): Per-asset probability of replacing the value. + value (float): Replacement value. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that randomly replaces selected asset values. + """ + validate_probability("probability", probability) + if node_assets is None and edge_assets is None: + raise ValueError("At least one of node_assets or edge_assets must be provided.") + + def policy(graph: AssetGraph): + for _, data in iter_selected_nodes( + graph, + node_ids=node_ids, + node_filter=node_filter, + ): + _drop(data, node_assets, probability, value, graph) + for _, _, data in iter_selected_edges( + graph, + edge_ids=edge_ids, + edge_filter=edge_filter, + ): + _drop(data, edge_assets, probability, value, graph) + + graph.logger.trace("Applied dropout policy.") + + return policy + + +def _drop(data, assets, probability, value, graph): + if assets is None: + return + for key in iter_selected_keys(data, assets): + if graph.rnd.random() < probability: + data[key] = value diff --git a/eclypse/policies/noise/gaussian_jitter.py b/eclypse/policies/noise/gaussian_jitter.py new file mode 100644 index 0000000..0342fdd --- /dev/null +++ b/eclypse/policies/noise/gaussian_jitter.py @@ -0,0 +1,86 @@ +"""Gaussian additive jitter noise policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + apply_numeric_transform, + clamp, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def gaussian_jitter( + *, + node_parameters: dict[str, tuple[float, float]] | None = None, + edge_parameters: dict[str, tuple[float, float]] | None = None, + lower: float | None = None, + upper: float | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Add Gaussian sampled deltas to selected assets. + + Args: + node_parameters (dict[str, tuple[float, float]] | None): + Mapping from node asset name to ``(mean, std)``. + edge_parameters (dict[str, tuple[float, float]] | None): + Mapping from edge asset name to ``(mean, std)``. + lower (float | None): Optional lower bound after adding noise. + upper (float | None): Optional upper bound after adding noise. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that adds independent Gaussian jitter to selected assets. + """ + _validate_parameters(node_parameters, edge_parameters) + + def policy(graph: AssetGraph): + def node_transform(key: str, current: float) -> float: + mean, std = (node_parameters or {})[key] + return clamp(current + graph.rnd.gauss(mean, std), lower, upper) + + def edge_transform(key: str, current: float) -> float: + mean, std = (edge_parameters or {})[key] + return clamp(current + graph.rnd.gauss(mean, std), lower, upper) + + apply_numeric_transform( + graph, + node_assets=list(node_parameters or {}), + node_ids=node_ids, + node_filter=node_filter, + transform=node_transform, + ) + apply_numeric_transform( + graph, + edge_assets=list(edge_parameters or {}), + edge_ids=edge_ids, + edge_filter=edge_filter, + transform=edge_transform, + ) + + graph.logger.trace("Applied gaussian_jitter policy.") + + return policy + + +def _validate_parameters(*parameter_sets): + if all(not parameter_set for parameter_set in parameter_sets): + raise ValueError("At least one parameter mapping must be provided.") + for parameter_set in parameter_sets: + for _, std in (parameter_set or {}).values(): + if std < 0: + raise ValueError("standard deviation must be non-negative.") diff --git a/eclypse/policies/noise/impulse.py b/eclypse/policies/noise/impulse.py index 809bf4a..d659dcb 100644 --- a/eclypse/policies/noise/impulse.py +++ b/eclypse/policies/noise/impulse.py @@ -5,17 +5,19 @@ from typing import TYPE_CHECKING from eclypse.policies._filters import ( + clamp, + coerce_numeric_like, + ensure_numeric_value, iter_selected_edges, + iter_selected_keys, iter_selected_nodes, ) -from eclypse.policies.noise._helpers import ( - apply_impulses, - validate_factor_range, - validate_probability, -) +from eclypse.policies._helpers import validate_probability from eclypse.utils.constants import MIN_FLOAT if TYPE_CHECKING: + from random import Random + from eclypse.graph.asset_graph import AssetGraph from eclypse.policies._filters import ( EdgeFilter, @@ -61,12 +63,12 @@ def impulse( raise ValueError("At least one of node_assets or edge_assets must be provided.") validate_probability("probability", probability) - validate_factor_range("node_factor_range", node_factor_range) + _validate_factor_range("node_factor_range", node_factor_range) effective_edge_factor_range = ( node_factor_range if edge_factor_range is None else edge_factor_range ) - validate_factor_range("edge_factor_range", effective_edge_factor_range) + _validate_factor_range("edge_factor_range", effective_edge_factor_range) def policy(graph: AssetGraph): for _, data in iter_selected_nodes( @@ -74,7 +76,7 @@ def policy(graph: AssetGraph): node_ids=node_ids, node_filter=node_filter, ): - apply_impulses( + _apply_impulses( data, node_assets, probability=probability, @@ -88,7 +90,7 @@ def policy(graph: AssetGraph): edge_ids=edge_ids, edge_filter=edge_filter, ): - apply_impulses( + _apply_impulses( data, edge_assets, probability=probability, @@ -100,3 +102,34 @@ def policy(graph: AssetGraph): graph.logger.trace("Applied impulse policy.") return policy + + +def _validate_factor_range(name: str, factor_range: tuple[float, float]) -> None: + lower, upper = factor_range + if lower < 0: + raise ValueError(f"{name} must use non-negative factors.") + if lower > upper: + raise ValueError(f"{name} must be ordered as (low, high).") + + +def _apply_impulses( + values: dict[str, object], + assets: str | list[str] | None, + *, + probability: float, + factor_range: tuple[float, float], + minimum: float, + random: Random, +) -> None: + lower_factor, upper_factor = factor_range + + for key in iter_selected_keys(values, assets): + if random.random() >= probability: + continue + + current = ensure_numeric_value(key, values[key]) + factor = random.uniform(lower_factor, upper_factor) + values[key] = coerce_numeric_like( + values[key], + clamp(current * factor, lower=minimum), + ) diff --git a/eclypse/policies/noise/multiplicative_jitter.py b/eclypse/policies/noise/multiplicative_jitter.py new file mode 100644 index 0000000..657f56a --- /dev/null +++ b/eclypse/policies/noise/multiplicative_jitter.py @@ -0,0 +1,59 @@ +"""Multiplicative jitter noise policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.noise.impulse import impulse +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def multiplicative_jitter( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_factor_range: tuple[float, float] = (0.95, 1.05), + edge_factor_range: tuple[float, float] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply multiplicative jitter on every call. + + Args: + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_factor_range (tuple[float, float]): Multiplicative range for selected node assets. + edge_factor_range (tuple[float, float] | None): + Multiplicative range for selected edge assets. When + omitted, ``node_factor_range`` is reused. + minimum (float): Lower bound after jitter. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that applies multiplicative jitter to selected assets. + """ + return impulse( + node_assets=node_assets, + edge_assets=edge_assets, + probability=1.0, + node_factor_range=node_factor_range, + edge_factor_range=edge_factor_range, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/noise/seasonal_noise.py b/eclypse/policies/noise/seasonal_noise.py new file mode 100644 index 0000000..eb7afe3 --- /dev/null +++ b/eclypse/policies/noise/seasonal_noise.py @@ -0,0 +1,116 @@ +"""Seasonal additive noise policy.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + apply_numeric_transform, + clamp, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class SeasonalNoisePolicy: + """Apply sinusoidal additive noise to selected assets.""" + + amplitude: float + period: int + node_assets: str | list[str] | None = None + edge_assets: str | list[str] | None = None + phase: float = 0.0 + lower: float | None = None + upper: float | None = None + node_ids: list[str] | None = None + node_filter: NodeFilter | None = None + edge_ids: list[tuple[str, str]] | None = None + edge_filter: EdgeFilter | None = None + step: int = 0 + + def __post_init__(self): + """Validate the seasonal noise configuration.""" + if self.period <= 0: + raise ValueError("period must be strictly positive.") + if self.node_assets is None and self.edge_assets is None: + raise ValueError( + "At least one of node_assets or edge_assets must be provided." + ) + + def __call__(self, graph: AssetGraph): + """Apply one seasonal noise step.""" + delta = self.amplitude * math.sin( + ((2 * math.pi * self.step) / self.period) + self.phase + ) + apply_numeric_transform( + graph, + node_assets=self.node_assets, + edge_assets=self.edge_assets, + node_ids=self.node_ids, + node_filter=self.node_filter, + edge_ids=self.edge_ids, + edge_filter=self.edge_filter, + transform=lambda _key, current: clamp( + current + delta, + self.lower, + self.upper, + ), + ) + self.step += 1 + graph.logger.trace("Applied seasonal_noise policy.") + + +def seasonal_noise( + *, + amplitude: float, + period: int, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + phase: float = 0.0, + lower: float | None = None, + upper: float | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply sinusoidal additive noise to selected assets. + + Args: + amplitude (float): Peak additive delta. + period (int): Number of calls in one sinusoidal cycle. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + phase (float): Phase offset in radians. + lower (float | None): Optional lower bound after adding noise. + upper (float | None): Optional upper bound after adding noise. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Stateful policy that applies seasonal additive noise. + """ + return SeasonalNoisePolicy( + amplitude=amplitude, + period=period, + node_assets=node_assets, + edge_assets=edge_assets, + phase=phase, + lower=lower, + upper=upper, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/replay/__init__.py b/eclypse/policies/replay/__init__.py index 59f4178..f42a73f 100644 --- a/eclypse/policies/replay/__init__.py +++ b/eclypse/policies/replay/__init__.py @@ -3,23 +3,37 @@ from __future__ import annotations from .from_dataframe import from_dataframe +from .from_csv import from_csv from .from_parquet import from_parquet from .from_records import from_records +from .interpolated_replay import interpolated_replay from .replay_edges import ( ReplayEdgesPolicy, replay_edges, ) +from .replay_events import ( + ReplayEventsPolicy, + replay_events, +) +from .replay_graph import replay_graph from .replay_nodes import ( ReplayNodesPolicy, replay_nodes, ) +from .replay_with_mapping import replay_with_mapping __all__ = [ "ReplayEdgesPolicy", + "ReplayEventsPolicy", "ReplayNodesPolicy", + "from_csv", "from_dataframe", "from_parquet", "from_records", + "interpolated_replay", "replay_edges", + "replay_events", + "replay_graph", "replay_nodes", + "replay_with_mapping", ] diff --git a/eclypse/policies/replay/_helpers.py b/eclypse/policies/replay/_helpers.py index 3277974..1264e4f 100644 --- a/eclypse/policies/replay/_helpers.py +++ b/eclypse/policies/replay/_helpers.py @@ -4,18 +4,21 @@ from collections import defaultdict from typing import ( - TYPE_CHECKING, Any, ) -if TYPE_CHECKING: - from eclypse.utils.types import MissingPolicyBehaviour - def normalise_records( record_source: Any, ) -> list[dict[str, Any]]: - """Convert dataframe-like or iterable sources into plain dictionaries.""" + """Convert dataframe-like or iterable sources into plain dictionaries. + + Args: + record_source (Any): DataFrame-like object or iterable of mapping records. + + Returns: + List of plain dictionary records. + """ if hasattr(record_source, "to_dict"): try: records = record_source.to_dict(orient="records") @@ -40,7 +43,16 @@ def infer_value_columns( reserved_columns: list[str], value_columns: list[str] | tuple[str, ...] | None, ) -> list[str]: - """Determine which record columns should be applied to the graph.""" + """Determine which record columns should be applied to the graph. + + Args: + records (list[dict[str, Any]]): Replay records. + reserved_columns (list[str]): Columns used for identity or timing. + value_columns (list[str] | tuple[str, ...] | None): Optional explicit value columns. + + Returns: + Columns copied from records to graph assets. + """ if value_columns is not None: return list(value_columns) if not records: @@ -49,18 +61,20 @@ def infer_value_columns( return [column for column in records[0] if column not in reserved] -def validate_missing_behaviour(missing: MissingPolicyBehaviour): - """Validate the behaviour used for missing graph items.""" - if missing not in {"ignore", "error"}: - raise ValueError('missing must be either "ignore" or "error".') - - def group_records_by_step( records: list[dict[str, Any]], *, time_column: str, ) -> dict[int, list[dict[str, Any]]]: - """Group records by simulation step.""" + """Group records by simulation step. + + Args: + records (list[dict[str, Any]]): Replay records. + time_column (str): Column containing replay steps. + + Returns: + Mapping from step to records at that step. + """ records_by_step: dict[int, list[dict[str, Any]]] = defaultdict(list) for record in records: records_by_step[int(record[time_column])].append(record) @@ -71,9 +85,51 @@ def initial_step( records_by_step: dict[int, list[dict[str, Any]]], start_step: int | None, ) -> int: - """Resolve the step from which the replay should start.""" + """Resolve the step from which the replay should start. + + Args: + records_by_step (dict[int, list[dict[str, Any]]]): Replay records grouped by step. + start_step (int | None): Optional explicit start step. + + Returns: + Initial replay step. + """ if start_step is not None: return start_step if records_by_step: return min(records_by_step) return 0 + + +def resolve_replay_step( + records_by_step: dict[int, list[dict[str, Any]]], + current_step: int, + *, + cyclic: bool, +) -> int: + """Resolve the source replay step for a possibly cyclic policy. + + Args: + records_by_step (dict[int, list[dict[str, Any]]]): Replay records grouped by source step. + current_step (int): Policy call step to resolve. + cyclic (bool): Whether to wrap within available replay steps. + + Returns: + Source replay step to read from. + """ + if not cyclic or not records_by_step: + return current_step + + first_step = min(records_by_step) + last_step = max(records_by_step) + cycle_length = (last_step - first_step) + 1 + return first_step + ((current_step - first_step) % cycle_length) + + +__all__ = [ + "group_records_by_step", + "infer_value_columns", + "initial_step", + "normalise_records", + "resolve_replay_step", +] diff --git a/eclypse/policies/replay/from_csv.py b/eclypse/policies/replay/from_csv.py new file mode 100644 index 0000000..a80fb9a --- /dev/null +++ b/eclypse/policies/replay/from_csv.py @@ -0,0 +1,79 @@ +"""Replay policy builders from CSV files.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.replay.from_dataframe import from_dataframe + +if TYPE_CHECKING: + from pathlib import Path + + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import ( + MissingPolicyBehaviour, + ReplayTarget, + UpdatePolicy, + ) + + +def from_csv( + path: str | Path, + *, + target: ReplayTarget, + node_id_column: str = "node_id", + source_column: str = "source", + target_column: str = "target", + time_column: str = "time", + value_columns: list[str] | tuple[str, ...] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, + missing: MissingPolicyBehaviour = "ignore", + start_step: int | None = None, + cyclic: bool = False, +) -> UpdatePolicy: + """Build a replay policy from a CSV file using pandas. + + Args: + path (str | Path): CSV file path. + target (ReplayTarget): Replay target, either ``"nodes"`` or ``"edges"``. + node_id_column (str): Column containing node identifiers. + source_column (str): Column containing edge source identifiers. + target_column (str): Column containing edge target identifiers. + time_column (str): Column containing replay steps. + value_columns (list[str] | tuple[str, ...] | None): + Optional explicit columns to copy from records. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + missing (MissingPolicyBehaviour): Behaviour when a replay record targets a missing item. + start_step (int | None): Optional starting replay step. + cyclic (bool): Whether to wrap past the final available replay step. + + Returns: + Stateful replay policy. + """ + import pandas as pd + + return from_dataframe( + pd.read_csv(path), + target=target, + node_id_column=node_id_column, + source_column=source_column, + target_column=target_column, + time_column=time_column, + value_columns=value_columns, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + missing=missing, + start_step=start_step, + cyclic=cyclic, + ) diff --git a/eclypse/policies/replay/from_dataframe.py b/eclypse/policies/replay/from_dataframe.py index ec13664..56978fd 100644 --- a/eclypse/policies/replay/from_dataframe.py +++ b/eclypse/policies/replay/from_dataframe.py @@ -34,8 +34,30 @@ def from_dataframe( edge_filter: EdgeFilter | None = None, missing: MissingPolicyBehaviour = "ignore", start_step: int | None = None, + cyclic: bool = False, ) -> UpdatePolicy: - """Build a replay policy from a dataframe-like object.""" + """Build a replay policy from a dataframe-like object. + + Args: + dataframe (Any): DataFrame-like object convertible to replay records. + target (ReplayTarget): Replay target, either ``"nodes"`` or ``"edges"``. + node_id_column (str): Column containing node identifiers. + source_column (str): Column containing edge source identifiers. + target_column (str): Column containing edge target identifiers. + time_column (str): Column containing replay steps. + value_columns (list[str] | tuple[str, ...] | None): + Optional explicit columns to copy from records. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + missing (MissingPolicyBehaviour): Behaviour when a replay record targets a missing item. + start_step (int | None): Optional starting replay step. + cyclic (bool): Whether to wrap past the final available replay step. + + Returns: + Stateful replay policy. + """ return from_records( normalise_records(dataframe), target=target, @@ -50,4 +72,5 @@ def from_dataframe( edge_filter=edge_filter, missing=missing, start_step=start_step, + cyclic=cyclic, ) diff --git a/eclypse/policies/replay/from_parquet.py b/eclypse/policies/replay/from_parquet.py index 049d113..eabc260 100644 --- a/eclypse/policies/replay/from_parquet.py +++ b/eclypse/policies/replay/from_parquet.py @@ -33,8 +33,30 @@ def from_parquet( edge_filter: EdgeFilter | None = None, missing: MissingPolicyBehaviour = "ignore", start_step: int | None = None, + cyclic: bool = False, ) -> UpdatePolicy: - """Build a replay policy from a parquet file using pandas when available.""" + """Build a replay policy from a parquet file using pandas when available. + + Args: + path (str): Parquet file path. + target (ReplayTarget): Replay target, either ``"nodes"`` or ``"edges"``. + node_id_column (str): Column containing node identifiers. + source_column (str): Column containing edge source identifiers. + target_column (str): Column containing edge target identifiers. + time_column (str): Column containing replay steps. + value_columns (list[str] | tuple[str, ...] | None): + Optional explicit columns to copy from records. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + missing (MissingPolicyBehaviour): Behaviour when a replay record targets a missing item. + start_step (int | None): Optional starting replay step. + cyclic (bool): Whether to wrap past the final available replay step. + + Returns: + Stateful replay policy. + """ try: import pandas as pd except ImportError as exc: # pragma: no cover - optional dependency @@ -56,4 +78,5 @@ def from_parquet( edge_filter=edge_filter, missing=missing, start_step=start_step, + cyclic=cyclic, ) diff --git a/eclypse/policies/replay/from_records.py b/eclypse/policies/replay/from_records.py index 7eeb8b5..304e734 100644 --- a/eclypse/policies/replay/from_records.py +++ b/eclypse/policies/replay/from_records.py @@ -34,8 +34,30 @@ def from_records( edge_filter: EdgeFilter | None = None, missing: MissingPolicyBehaviour = "ignore", start_step: int | None = None, + cyclic: bool = False, ) -> UpdatePolicy: - """Build a replay policy from plain Python records.""" + """Build a replay policy from plain Python records. + + Args: + record_source (Any): Iterable of mapping records to replay. + target (ReplayTarget): Replay target, either ``"nodes"`` or ``"edges"``. + node_id_column (str): Column containing node identifiers. + source_column (str): Column containing edge source identifiers. + target_column (str): Column containing edge target identifiers. + time_column (str): Column containing replay steps. + value_columns (list[str] | tuple[str, ...] | None): + Optional explicit columns to copy from records. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + missing (MissingPolicyBehaviour): Behaviour when a replay record targets a missing item. + start_step (int | None): Optional starting replay step. + cyclic (bool): Whether to wrap past the final available replay step. + + Returns: + Stateful replay policy. + """ if target == "nodes": return replay_nodes( record_source, @@ -46,6 +68,7 @@ def from_records( node_filter=node_filter, missing=missing, start_step=start_step, + cyclic=cyclic, ) if target == "edges": return replay_edges( @@ -58,5 +81,6 @@ def from_records( edge_filter=edge_filter, missing=missing, start_step=start_step, + cyclic=cyclic, ) raise ValueError('target must be either "nodes" or "edges".') diff --git a/eclypse/policies/replay/interpolated_replay.py b/eclypse/policies/replay/interpolated_replay.py new file mode 100644 index 0000000..b7a10e5 --- /dev/null +++ b/eclypse/policies/replay/interpolated_replay.py @@ -0,0 +1,109 @@ +"""Interpolated replay policies.""" + +from __future__ import annotations + +from itertools import pairwise +from typing import TYPE_CHECKING + +from eclypse.policies.replay._helpers import ( + infer_value_columns, + normalise_records, +) +from eclypse.policies.replay.from_records import from_records + +if TYPE_CHECKING: + from eclypse.utils.types import ( + ReplayTarget, + UpdatePolicy, + ) + + +def interpolated_replay( + record_source, + *, + target: ReplayTarget, + node_id_column: str = "node_id", + source_column: str = "source", + target_column: str = "target", + time_column: str = "time", + value_columns: list[str] | tuple[str, ...] | None = None, + **kwargs, +) -> UpdatePolicy: + """Replay records after filling integer steps by linear interpolation. + + Args: + record_source (Any): Iterable of sparse replay records. + target (ReplayTarget): Replay target, either ``"nodes"`` or ``"edges"``. + node_id_column (str): Column containing node identifiers. + source_column (str): Column containing edge source identifiers. + target_column (str): Column containing edge target identifiers. + time_column (str): Column containing replay steps. + value_columns (list[str] | tuple[str, ...] | None): + Optional explicit numeric columns to interpolate. + kwargs (Any): Additional keyword arguments forwarded to ``from_records``. + + Returns: + Stateful replay policy using interpolated records. + """ + records = normalise_records(record_source) + identity_columns = ( + [node_id_column] if target == "nodes" else [source_column, target_column] + ) + columns = infer_value_columns( + records, + reserved_columns=[*identity_columns, time_column], + value_columns=value_columns, + ) + interpolated = _interpolate_records( + records, + identity_columns=identity_columns, + time_column=time_column, + value_columns=columns, + ) + return from_records( + interpolated, + target=target, + node_id_column=node_id_column, + source_column=source_column, + target_column=target_column, + time_column=time_column, + value_columns=columns, + **kwargs, + ) + + +def _interpolate_records( + records: list[dict], + *, + identity_columns: list[str], + time_column: str, + value_columns: list[str], +) -> list[dict]: + grouped: dict[tuple, list[dict]] = {} + for record in records: + identity = tuple(record[column] for column in identity_columns) + grouped.setdefault(identity, []).append(record) + + result: list[dict] = [] + for identity, group in grouped.items(): + ordered = sorted(group, key=lambda record: int(record[time_column])) + for left, right in pairwise(ordered): + left_step = int(left[time_column]) + right_step = int(right[time_column]) + result.append(dict(left)) + for step in range(left_step + 1, right_step): + progress = (step - left_step) / (right_step - left_step) + interpolated = { + column: value + for column, value in zip(identity_columns, identity, strict=True) + } + interpolated[time_column] = step + for column in value_columns: + if column in left and column in right: + interpolated[column] = left[column] + ( + (right[column] - left[column]) * progress + ) + result.append(interpolated) + if ordered: + result.append(dict(ordered[-1])) + return sorted(result, key=lambda record: int(record[time_column])) diff --git a/eclypse/policies/replay/replay_edges.py b/eclypse/policies/replay/replay_edges.py index 2363e38..32d8c8d 100644 --- a/eclypse/policies/replay/replay_edges.py +++ b/eclypse/policies/replay/replay_edges.py @@ -8,12 +8,13 @@ Any, ) +from eclypse.policies._helpers import validate_missing_behaviour from eclypse.policies.replay._helpers import ( group_records_by_step, infer_value_columns, initial_step, normalise_records, - validate_missing_behaviour, + resolve_replay_step, ) if TYPE_CHECKING: @@ -36,11 +37,17 @@ class ReplayEdgesPolicy: selected_edge_ids: set[tuple[str, str]] | None = None edge_filter: EdgeFilter | None = None missing: MissingPolicyBehaviour = "ignore" + cyclic: bool = False current_step: int = 0 def __call__(self, graph: AssetGraph): """Apply the replay records for the current step to matching edges.""" - for record in self.records_by_step.get(self.current_step, []): + replay_step = resolve_replay_step( + self.records_by_step, + self.current_step, + cyclic=self.cyclic, + ) + for record in self.records_by_step.get(replay_step, []): _update_edge_from_record( graph, record, @@ -52,7 +59,7 @@ def __call__(self, graph: AssetGraph): missing=self.missing, ) - graph.logger.trace(f"Applied replay_edges policy for step {self.current_step}.") + graph.logger.trace(f"Applied replay_edges policy for step {replay_step}.") self.current_step += 1 @@ -67,8 +74,26 @@ def replay_edges( edge_filter: EdgeFilter | None = None, missing: MissingPolicyBehaviour = "ignore", start_step: int | None = None, + cyclic: bool = False, ) -> UpdatePolicy: - """Replay edge attributes from time-indexed records.""" + """Replay edge attributes from time-indexed records. + + Args: + record_source (Any): Iterable of mapping records to replay. + source_column (str): Column containing edge source identifiers. + target_column (str): Column containing edge target identifiers. + time_column (str): Column containing replay steps. + value_columns (list[str] | tuple[str, ...] | None): + Optional explicit columns to copy from records. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + missing (MissingPolicyBehaviour): Behaviour when a replay record targets a missing edge. + start_step (int | None): Optional starting replay step. + cyclic (bool): Whether to wrap past the final available replay step. + + Returns: + Stateful edge replay policy. + """ validate_missing_behaviour(missing) records = normalise_records(record_source) columns = infer_value_columns( @@ -88,6 +113,7 @@ def replay_edges( selected_edge_ids=selected_edge_ids, edge_filter=edge_filter, missing=missing, + cyclic=cyclic, current_step=current_step, ) diff --git a/eclypse/policies/replay/replay_events.py b/eclypse/policies/replay/replay_events.py new file mode 100644 index 0000000..faee75b --- /dev/null +++ b/eclypse/policies/replay/replay_events.py @@ -0,0 +1,72 @@ +"""Replay arbitrary update policies from time-indexed event records.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + Any, +) + +from eclypse.policies.replay._helpers import ( + group_records_by_step, + initial_step, + normalise_records, + resolve_replay_step, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class ReplayEventsPolicy: + """Replay update callables stored in records.""" + + records_by_step: dict[int, list[dict[str, Any]]] + policy_column: str = "policy" + current_step: int = 0 + cyclic: bool = False + + def __call__(self, graph: AssetGraph): + """Apply all event policies for the current step.""" + replay_step = resolve_replay_step( + self.records_by_step, + self.current_step, + cyclic=self.cyclic, + ) + for record in self.records_by_step.get(replay_step, []): + record[self.policy_column](graph) + self.current_step += 1 + graph.logger.trace(f"Applied replay_events policy for step {replay_step}.") + + +def replay_events( + record_source, + *, + time_column: str = "time", + policy_column: str = "policy", + start_step: int | None = None, + cyclic: bool = False, +) -> UpdatePolicy: + """Replay arbitrary update policies from time-indexed records. + + Args: + record_source (Any): Iterable of records containing update policies. + time_column (str): Column containing replay steps. + policy_column (str): Column containing policy callables. + start_step (int | None): Optional starting replay step. + cyclic (bool): Whether to wrap past the final available replay step. + + Returns: + Stateful event replay policy. + """ + records = normalise_records(record_source) + records_by_step = group_records_by_step(records, time_column=time_column) + return ReplayEventsPolicy( + records_by_step=records_by_step, + policy_column=policy_column, + current_step=initial_step(records_by_step, start_step), + cyclic=cyclic, + ) diff --git a/eclypse/policies/replay/replay_graph.py b/eclypse/policies/replay/replay_graph.py new file mode 100644 index 0000000..58054cb --- /dev/null +++ b/eclypse/policies/replay/replay_graph.py @@ -0,0 +1,101 @@ +"""Replay node and edge attributes together.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.replay.replay_edges import replay_edges +from eclypse.policies.replay.replay_nodes import replay_nodes + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import ( + MissingPolicyBehaviour, + UpdatePolicy, + ) + + +def replay_graph( + *, + node_records=None, + edge_records=None, + node_id_column: str = "node_id", + source_column: str = "source", + target_column: str = "target", + time_column: str = "time", + node_value_columns: list[str] | tuple[str, ...] | None = None, + edge_value_columns: list[str] | tuple[str, ...] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, + missing: MissingPolicyBehaviour = "ignore", + start_step: int | None = None, + cyclic: bool = False, +) -> UpdatePolicy: + """Replay node and edge records as one graph policy. + + Args: + node_records (Any): Optional node replay records. + edge_records (Any): Optional edge replay records. + node_id_column (str): Column containing node identifiers. + source_column (str): Column containing edge source identifiers. + target_column (str): Column containing edge target identifiers. + time_column (str): Column containing replay steps. + node_value_columns (list[str] | tuple[str, ...] | None): + Optional explicit node columns to copy. + edge_value_columns (list[str] | tuple[str, ...] | None): + Optional explicit edge columns to copy. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + missing (MissingPolicyBehaviour): Behaviour when a replay record targets a missing item. + start_step (int | None): Optional starting replay step. + cyclic (bool): Whether to wrap past the final available replay step. + + Returns: + Stateful graph replay policy. + """ + policies: list[UpdatePolicy] = [] + if node_records is not None: + policies.append( + replay_nodes( + node_records, + node_id_column=node_id_column, + time_column=time_column, + value_columns=node_value_columns, + node_ids=node_ids, + node_filter=node_filter, + missing=missing, + start_step=start_step, + cyclic=cyclic, + ) + ) + if edge_records is not None: + policies.append( + replay_edges( + edge_records, + source_column=source_column, + target_column=target_column, + time_column=time_column, + value_columns=edge_value_columns, + edge_ids=edge_ids, + edge_filter=edge_filter, + missing=missing, + start_step=start_step, + cyclic=cyclic, + ) + ) + if not policies: + raise ValueError("At least one of node_records or edge_records is required.") + + def policy(graph: AssetGraph): + for child_policy in policies: + child_policy(graph) + + return policy diff --git a/eclypse/policies/replay/replay_nodes.py b/eclypse/policies/replay/replay_nodes.py index 69efa35..62d3996 100644 --- a/eclypse/policies/replay/replay_nodes.py +++ b/eclypse/policies/replay/replay_nodes.py @@ -8,12 +8,13 @@ Any, ) +from eclypse.policies._helpers import validate_missing_behaviour from eclypse.policies.replay._helpers import ( group_records_by_step, infer_value_columns, initial_step, normalise_records, - validate_missing_behaviour, + resolve_replay_step, ) if TYPE_CHECKING: @@ -35,11 +36,17 @@ class ReplayNodesPolicy: selected_node_ids: set[str] | None = None node_filter: NodeFilter | None = None missing: MissingPolicyBehaviour = "ignore" + cyclic: bool = False current_step: int = 0 def __call__(self, graph: AssetGraph): """Apply the replay records for the current step to matching nodes.""" - for record in self.records_by_step.get(self.current_step, []): + replay_step = resolve_replay_step( + self.records_by_step, + self.current_step, + cyclic=self.cyclic, + ) + for record in self.records_by_step.get(replay_step, []): _update_node_from_record( graph, record, @@ -50,7 +57,7 @@ def __call__(self, graph: AssetGraph): missing=self.missing, ) - graph.logger.trace(f"Applied replay_nodes policy for step {self.current_step}.") + graph.logger.trace(f"Applied replay_nodes policy for step {replay_step}.") self.current_step += 1 @@ -64,8 +71,25 @@ def replay_nodes( node_filter: NodeFilter | None = None, missing: MissingPolicyBehaviour = "ignore", start_step: int | None = None, + cyclic: bool = False, ) -> UpdatePolicy: - """Replay node attributes from time-indexed records.""" + """Replay node attributes from time-indexed records. + + Args: + record_source (Any): Iterable of mapping records to replay. + node_id_column (str): Column containing node identifiers. + time_column (str): Column containing replay steps. + value_columns (list[str] | tuple[str, ...] | None): + Optional explicit columns to copy from records. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + missing (MissingPolicyBehaviour): Behaviour when a replay record targets a missing node. + start_step (int | None): Optional starting replay step. + cyclic (bool): Whether to wrap past the final available replay step. + + Returns: + Stateful node replay policy. + """ validate_missing_behaviour(missing) records = normalise_records(record_source) columns = infer_value_columns( @@ -84,6 +108,7 @@ def replay_nodes( selected_node_ids=selected_node_ids, node_filter=node_filter, missing=missing, + cyclic=cyclic, current_step=current_step, ) diff --git a/eclypse/policies/replay/replay_with_mapping.py b/eclypse/policies/replay/replay_with_mapping.py new file mode 100644 index 0000000..1c7fed0 --- /dev/null +++ b/eclypse/policies/replay/replay_with_mapping.py @@ -0,0 +1,50 @@ +"""Replay records after applying column and id mappings.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.replay._helpers import normalise_records +from eclypse.policies.replay.from_records import from_records + +if TYPE_CHECKING: + from eclypse.utils.types import ( + ReplayTarget, + UpdatePolicy, + ) + + +def replay_with_mapping( + record_source, + *, + target: ReplayTarget, + column_mapping: dict[str, str] | None = None, + id_mapping: dict[str, str] | None = None, + **kwargs, +) -> UpdatePolicy: + """Replay records after renaming columns and external graph ids. + + Args: + record_source (Any): Iterable of mapping records to replay. + target (ReplayTarget): Replay target, either ``"nodes"`` or ``"edges"``. + column_mapping (dict[str, str] | None): + Optional mapping from input column names to replay columns. + id_mapping (dict[str, str] | None): + Optional mapping from external graph ids to local graph ids. + kwargs (Any): Additional keyword arguments forwarded to ``from_records``. + + Returns: + Stateful replay policy. + """ + mapped_records = [] + for record in normalise_records(record_source): + mapped = { + (column_mapping or {}).get(column, column): value + for column, value in record.items() + } + for key in ("node_id", "source", "target"): + if key in mapped: + mapped[key] = (id_mapping or {}).get(mapped[key], mapped[key]) + mapped_records.append(mapped) + + return from_records(mapped_records, target=target, **kwargs) diff --git a/eclypse/policies/schedule/__init__.py b/eclypse/policies/schedule/__init__.py index 97296ff..50f8ead 100644 --- a/eclypse/policies/schedule/__init__.py +++ b/eclypse/policies/schedule/__init__.py @@ -6,26 +6,62 @@ AfterPolicy, after, ) +from .at import ( + AtPolicy, + at, +) from .between import ( BetweenPolicy, between, ) +from .cooldown import ( + CooldownPolicy, + cooldown, +) from .every import ( EveryPolicy, every, ) +from .jittered_every import ( + JitteredEveryPolicy, + jittered_every, +) from .once_at import ( OnceAtPolicy, once_at, ) +from .repeat import ( + RepeatPolicy, + repeat, +) +from .until import ( + UntilPolicy, + until, +) +from .with_probability import ( + WithProbabilityPolicy, + with_probability, +) __all__ = [ "AfterPolicy", + "AtPolicy", "BetweenPolicy", + "CooldownPolicy", "EveryPolicy", + "JitteredEveryPolicy", "OnceAtPolicy", + "RepeatPolicy", + "UntilPolicy", + "WithProbabilityPolicy", "after", + "at", "between", + "cooldown", "every", + "jittered_every", "once_at", + "repeat", + "until", + "with_probability", ] diff --git a/eclypse/policies/schedule/at.py b/eclypse/policies/schedule/at.py new file mode 100644 index 0000000..029c402 --- /dev/null +++ b/eclypse/policies/schedule/at.py @@ -0,0 +1,48 @@ +"""Run a policy at explicit steps.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class AtPolicy: + """Run a policy at selected steps.""" + + steps: set[int] + policy: UpdatePolicy + step: int = 0 + + def __post_init__(self): + """Validate the schedule configuration.""" + if not self.steps: + raise ValueError("steps must not be empty.") + if any(step < 0 for step in self.steps): + raise ValueError("steps must be non-negative.") + + def __call__(self, graph: AssetGraph): + """Apply the wrapped policy when the current step is selected.""" + if self.step in self.steps: + self.policy(graph) + graph.logger.trace(f"Triggered at policy at step {self.step}.") + self.step += 1 + + +def at(steps: int | list[int] | tuple[int, ...], policy: UpdatePolicy) -> UpdatePolicy: + """Run a policy at one or more explicit steps. + + Args: + steps (int | list[int] | tuple[int, ...]): + Step number or step numbers that trigger the policy. + policy (UpdatePolicy): Wrapped policy to call when the step matches. + + Returns: + Stateful schedule policy. + """ + selected_steps = {steps} if isinstance(steps, int) else set(steps) + return AtPolicy(steps=selected_steps, policy=policy) diff --git a/eclypse/policies/schedule/cooldown.py b/eclypse/policies/schedule/cooldown.py new file mode 100644 index 0000000..d369c77 --- /dev/null +++ b/eclypse/policies/schedule/cooldown.py @@ -0,0 +1,46 @@ +"""Run a policy with a minimum gap between applications.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class CooldownPolicy: + """Run a policy at most once every ``steps`` calls.""" + + steps: int + policy: UpdatePolicy + step: int = 0 + next_allowed_step: int = 0 + + def __post_init__(self): + """Validate the schedule configuration.""" + if self.steps < 0: + raise ValueError("steps must be non-negative.") + + def __call__(self, graph: AssetGraph): + """Apply the wrapped policy when the cooldown has elapsed.""" + if self.step >= self.next_allowed_step: + self.policy(graph) + self.next_allowed_step = self.step + self.steps + 1 + graph.logger.trace(f"Triggered cooldown policy at step {self.step}.") + self.step += 1 + + +def cooldown(steps: int, policy: UpdatePolicy) -> UpdatePolicy: + """Run a policy at most once every ``steps`` calls. + + Args: + steps (int): Minimum number of skipped calls after each application. + policy (UpdatePolicy): Wrapped policy to throttle. + + Returns: + Stateful schedule policy. + """ + return CooldownPolicy(steps=steps, policy=policy) diff --git a/eclypse/policies/schedule/jittered_every.py b/eclypse/policies/schedule/jittered_every.py new file mode 100644 index 0000000..423ed8d --- /dev/null +++ b/eclypse/policies/schedule/jittered_every.py @@ -0,0 +1,70 @@ +"""Run a periodic policy with bounded jitter.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class JitteredEveryPolicy: + """Run a policy periodically with integer step jitter.""" + + interval: int + policy: UpdatePolicy + jitter: int = 0 + start: int = 0 + step: int = 0 + next_step: int | None = None + + def __post_init__(self): + """Validate the schedule configuration.""" + if self.interval <= 0: + raise ValueError("interval must be strictly positive.") + if self.jitter < 0: + raise ValueError("jitter must be non-negative.") + if self.start < 0: + raise ValueError("start must be non-negative.") + + def __call__(self, graph: AssetGraph): + """Apply the wrapped policy when the jittered step is reached.""" + if self.next_step is None: + self.next_step = self.start + if self.step >= self.next_step: + self.policy(graph) + delta = self.interval + if self.jitter: + delta += graph.rnd.randint(-self.jitter, self.jitter) + self.next_step = self.step + max(1, delta) + graph.logger.trace(f"Triggered jittered_every policy at step {self.step}.") + self.step += 1 + + +def jittered_every( + interval: int, + policy: UpdatePolicy, + *, + jitter: int = 0, + start: int = 0, +) -> UpdatePolicy: + """Run a policy every ``interval`` steps with optional integer jitter. + + Args: + interval (int): Base interval between applications. + policy (UpdatePolicy): Wrapped policy to call. + jitter (int): Maximum integer offset added to each next interval. + start (int): First eligible step. + + Returns: + Stateful schedule policy. + """ + return JitteredEveryPolicy( + interval=interval, + policy=policy, + jitter=jitter, + start=start, + ) diff --git a/eclypse/policies/schedule/repeat.py b/eclypse/policies/schedule/repeat.py new file mode 100644 index 0000000..20d4b4f --- /dev/null +++ b/eclypse/policies/schedule/repeat.py @@ -0,0 +1,44 @@ +"""Run a policy a fixed number of times.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class RepeatPolicy: + """Run a policy for the first ``times`` calls.""" + + times: int + policy: UpdatePolicy + count: int = 0 + + def __post_init__(self): + """Validate the schedule configuration.""" + if self.times < 0: + raise ValueError("times must be non-negative.") + + def __call__(self, graph: AssetGraph): + """Apply the wrapped policy while repetitions remain.""" + if self.count < self.times: + self.policy(graph) + graph.logger.trace(f"Triggered repeat policy at count {self.count}.") + self.count += 1 + + +def repeat(times: int, policy: UpdatePolicy) -> UpdatePolicy: + """Run a policy for the first ``times`` calls. + + Args: + times (int): Number of calls that should trigger the wrapped policy. + policy (UpdatePolicy): Wrapped policy to call. + + Returns: + Stateful schedule policy. + """ + return RepeatPolicy(times=times, policy=policy) diff --git a/eclypse/policies/schedule/until.py b/eclypse/policies/schedule/until.py new file mode 100644 index 0000000..63d1bd4 --- /dev/null +++ b/eclypse/policies/schedule/until.py @@ -0,0 +1,44 @@ +"""Run a policy until an inclusive end step.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class UntilPolicy: + """Run a policy from step zero through ``end``.""" + + end: int + policy: UpdatePolicy + step: int = 0 + + def __post_init__(self): + """Validate the schedule configuration.""" + if self.end < 0: + raise ValueError("end must be non-negative.") + + def __call__(self, graph: AssetGraph): + """Apply the wrapped policy until the end step is passed.""" + if self.step <= self.end: + self.policy(graph) + graph.logger.trace(f"Triggered until policy at step {self.step}.") + self.step += 1 + + +def until(end: int, policy: UpdatePolicy) -> UpdatePolicy: + """Run a policy from step zero through ``end``. + + Args: + end (int): Inclusive final step that triggers the policy. + policy (UpdatePolicy): Wrapped policy to call. + + Returns: + Stateful schedule policy. + """ + return UntilPolicy(end=end, policy=policy) diff --git a/eclypse/policies/schedule/with_probability.py b/eclypse/policies/schedule/with_probability.py new file mode 100644 index 0000000..dbb1888 --- /dev/null +++ b/eclypse/policies/schedule/with_probability.py @@ -0,0 +1,43 @@ +"""Run a policy according to a probability.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from eclypse.policies._helpers import validate_probability + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class WithProbabilityPolicy: + """Run a policy when a graph RNG draw is below ``probability``.""" + + probability: float + policy: UpdatePolicy + + def __post_init__(self): + """Validate the schedule configuration.""" + validate_probability("probability", self.probability) + + def __call__(self, graph: AssetGraph): + """Apply the wrapped policy after a successful random draw.""" + if graph.rnd.random() < self.probability: + self.policy(graph) + graph.logger.trace("Triggered with_probability policy.") + + +def with_probability(probability: float, policy: UpdatePolicy) -> UpdatePolicy: + """Run a policy according to a probability. + + Args: + probability (float): Per-call probability of triggering the policy. + policy (UpdatePolicy): Wrapped policy to call. + + Returns: + Policy that applies the wrapped policy after successful random draws. + """ + return WithProbabilityPolicy(probability=probability, policy=policy) diff --git a/eclypse/policies/topology/__init__.py b/eclypse/policies/topology/__init__.py new file mode 100644 index 0000000..e6aee4a --- /dev/null +++ b/eclypse/policies/topology/__init__.py @@ -0,0 +1,15 @@ +"""Topology mutation policies.""" + +from .add_edge import add_edge +from .add_node import add_node +from .churn import churn +from .remove_node import remove_node +from .rewire import rewire + +__all__ = [ + "add_edge", + "add_node", + "churn", + "remove_node", + "rewire", +] diff --git a/eclypse/policies/topology/add_edge.py b/eclypse/policies/topology/add_edge.py new file mode 100644 index 0000000..98d6118 --- /dev/null +++ b/eclypse/policies/topology/add_edge.py @@ -0,0 +1,37 @@ +"""Edge insertion topology policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +def add_edge( + source: str, + target: str, + *, + symmetric: bool = False, + strict: bool = False, + **assets, +) -> UpdatePolicy: + """Add an edge if both endpoints exist. + + Args: + source (str): Source node identifier. + target (str): Target node identifier. + symmetric (bool): Whether to add the symmetric edge too. + strict (bool): Whether graph insertion should use strict duplicate checks. + assets (Any): Edge assets passed to the graph. + + Returns: + Policy that adds the edge when endpoints exist. + """ + + def policy(graph: AssetGraph): + if graph.has_node(source) and graph.has_node(target): + graph.add_edge(source, target, symmetric=symmetric, strict=strict, **assets) + + return policy diff --git a/eclypse/policies/topology/add_node.py b/eclypse/policies/topology/add_node.py new file mode 100644 index 0000000..b2a2a85 --- /dev/null +++ b/eclypse/policies/topology/add_node.py @@ -0,0 +1,28 @@ +"""Node insertion topology policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +def add_node(node_id: str, *, strict: bool = False, **assets) -> UpdatePolicy: + """Add a node if it is missing. + + Args: + node_id (str): Node identifier to add. + strict (bool): Whether graph insertion should use strict duplicate checks. + assets (Any): Node assets passed to the graph. + + Returns: + Policy that adds the node when absent. + """ + + def policy(graph: AssetGraph): + if not graph.has_node(node_id): + graph.add_node(node_id, strict=strict, **assets) + + return policy diff --git a/eclypse/policies/topology/churn.py b/eclypse/policies/topology/churn.py new file mode 100644 index 0000000..cf8fc67 --- /dev/null +++ b/eclypse/policies/topology/churn.py @@ -0,0 +1,41 @@ +"""Node churn topology policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._helpers import validate_probability + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +def churn( + *, + remove_probability: float = 0.0, + add_probability: float = 0.0, + candidate_nodes: dict[str, dict] | None = None, +) -> UpdatePolicy: + """Randomly remove existing nodes and add candidate nodes. + + Args: + remove_probability (float): Per-existing-node probability of removal. + add_probability (float): Per-candidate-node probability of addition. + candidate_nodes (dict[str, dict] | None): Optional mapping from node id to node assets. + + Returns: + Policy that applies node churn. + """ + validate_probability("remove_probability", remove_probability) + validate_probability("add_probability", add_probability) + + def policy(graph: AssetGraph): + for node_id in list(graph.nodes): + if graph.rnd.random() < remove_probability: + graph.remove_node(node_id) + for node_id, assets in (candidate_nodes or {}).items(): + if not graph.has_node(node_id) and graph.rnd.random() < add_probability: + graph.add_node(node_id, strict=False, **assets) + + return policy diff --git a/eclypse/policies/topology/remove_node.py b/eclypse/policies/topology/remove_node.py new file mode 100644 index 0000000..2691ffb --- /dev/null +++ b/eclypse/policies/topology/remove_node.py @@ -0,0 +1,40 @@ +"""Node removal topology policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._helpers import validate_missing_behaviour + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import ( + MissingPolicyBehaviour, + UpdatePolicy, + ) + + +def remove_node( + node_id: str, + *, + missing: MissingPolicyBehaviour = "ignore", +) -> UpdatePolicy: + """Remove a node. + + Args: + node_id (str): Node identifier to remove. + missing (MissingPolicyBehaviour): + Behaviour for absent nodes, either ``"ignore"`` or ``"error"``. + + Returns: + Policy that removes the configured node. + """ + validate_missing_behaviour(missing) + + def policy(graph: AssetGraph): + if graph.has_node(node_id): + graph.remove_node(node_id) + elif missing == "error": + raise KeyError(f'Node "{node_id}" not found in the graph.') + + return policy diff --git a/eclypse/policies/topology/rewire.py b/eclypse/policies/topology/rewire.py new file mode 100644 index 0000000..a1f9a2e --- /dev/null +++ b/eclypse/policies/topology/rewire.py @@ -0,0 +1,46 @@ +"""Edge rewiring topology policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._helpers import validate_probability + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + +MIN_REWIRE_NODES = 2 +"""Minimum node count needed to rewire an edge.""" + + +def rewire( + edge_ids: list[tuple[str, str]], *, probability: float = 1.0 +) -> UpdatePolicy: + """Rewire selected edges to random targets. + + Args: + edge_ids (list[tuple[str, str]]): Edge identifiers eligible for rewiring. + probability (float): Per-edge probability of rewiring. + + Returns: + Policy that rewires selected edges. + """ + validate_probability("probability", probability) + + def policy(graph: AssetGraph): + nodes = list(graph.nodes) + if len(nodes) < MIN_REWIRE_NODES: + return + for source, target in list(edge_ids): + if not graph.has_edge(source, target) or graph.rnd.random() >= probability: + continue + data = dict(graph.edges[source, target]) + candidates = [node for node in nodes if node not in {source, target}] + if not candidates: + continue + new_target = graph.rnd.choice(candidates) + graph.remove_edge(source, target) + graph.add_edge(source, new_target, strict=False, **data) + + return policy diff --git a/eclypse/policies/workload/__init__.py b/eclypse/policies/workload/__init__.py new file mode 100644 index 0000000..89bf31c --- /dev/null +++ b/eclypse/policies/workload/__init__.py @@ -0,0 +1,15 @@ +"""Workload-oriented update policies.""" + +from .arrival_process import arrival_process +from .diurnal_load import ( + DiurnalLoadPolicy, + diurnal_load, +) +from .traffic_matrix import traffic_matrix + +__all__ = [ + "DiurnalLoadPolicy", + "arrival_process", + "diurnal_load", + "traffic_matrix", +] diff --git a/eclypse/policies/workload/_helpers.py b/eclypse/policies/workload/_helpers.py new file mode 100644 index 0000000..a929310 --- /dev/null +++ b/eclypse/policies/workload/_helpers.py @@ -0,0 +1,39 @@ +"""Shared helpers for workload policies.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Any, +) + +from eclypse.policies._filters import apply_numeric_transform_to_values + +if TYPE_CHECKING: + from collections.abc import Callable + + +def apply_selected_asset_transform( + data: dict[str, Any], + assets: str | list[str] | None, + *, + transform: Callable[[str, float], float], +) -> None: + """Apply a numeric transform only when explicit assets are configured. + + Args: + data (dict[str, Any]): Asset mapping to mutate. + assets (str | list[str] | None): Asset selector. ``None`` skips the mutation. + transform (Callable[[str, float], float]): + Callable receiving ``(asset_key, current_value)``. + + Returns: + None. + """ + if assets is None: + return + + apply_numeric_transform_to_values(data, assets, transform=transform) + + +__all__ = ["apply_selected_asset_transform"] diff --git a/eclypse/policies/workload/arrival_process.py b/eclypse/policies/workload/arrival_process.py new file mode 100644 index 0000000..8758fe1 --- /dev/null +++ b/eclypse/policies/workload/arrival_process.py @@ -0,0 +1,85 @@ +"""Poisson arrival workload policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + iter_selected_edges, + iter_selected_nodes, +) +from eclypse.policies.distribution.poisson import _sample_poisson +from eclypse.policies.workload._helpers import apply_selected_asset_transform + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def arrival_process( + rate: float, + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Add Poisson arrivals to selected workload assets. + + Args: + rate (float): Poisson arrival rate. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + node_ids (list[str] | None): Optional explicit node identifiers to mutate. + node_filter (NodeFilter | None): Optional predicate receiving ``(node_id, data)``. + edge_ids (list[tuple[str, str]] | None): Optional explicit edge identifiers to mutate. + edge_filter (EdgeFilter | None): Optional predicate receiving ``(source, target, data)``. + + Returns: + Policy that increments selected workload assets. + """ + if rate < 0: + raise ValueError("rate must be non-negative.") + if node_assets is None and edge_assets is None: + raise ValueError("At least one of node_assets or edge_assets must be provided.") + + def policy(graph: AssetGraph): + for _, data in iter_selected_nodes( + graph, + node_ids=node_ids, + node_filter=node_filter, + ): + _add_arrivals(data, node_assets, rate, graph) + for _, _, data in iter_selected_edges( + graph, + edge_ids=edge_ids, + edge_filter=edge_filter, + ): + _add_arrivals(data, edge_assets, rate, graph) + + return policy + + +def _add_arrivals(data, assets, rate, graph): + """Apply sampled arrivals to one asset mapping. + + Args: + data (dict[str, object]): Asset mapping to mutate. + assets (str | list[str] | None): Optional asset selector. + rate (float): Poisson arrival rate. + graph (AssetGraph): Graph providing the random generator. + + Returns: + None. + """ + apply_selected_asset_transform( + data, + assets, + transform=lambda _key, current: current + _sample_poisson(graph.rnd, rate), + ) diff --git a/eclypse/policies/workload/diurnal_load.py b/eclypse/policies/workload/diurnal_load.py new file mode 100644 index 0000000..32194a9 --- /dev/null +++ b/eclypse/policies/workload/diurnal_load.py @@ -0,0 +1,106 @@ +"""Diurnal load workload policy.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from eclypse.policies.workload._helpers import apply_selected_asset_transform + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class DiurnalLoadPolicy: + """Apply sinusoidal multiplicative load over a period.""" + + amplitude: float + period: int + baseline: float = 1.0 + node_assets: str | list[str] | None = None + edge_assets: str | list[str] | None = None + step: int = 0 + + def __post_init__(self): + """Validate the diurnal load configuration. + + Args: + None. + + Returns: + None. + """ + if self.period <= 0: + raise ValueError("period must be strictly positive.") + if self.node_assets is None and self.edge_assets is None: + raise ValueError( + "At least one of node_assets or edge_assets must be provided." + ) + + def __call__(self, graph: AssetGraph): + """Apply one diurnal load step. + + Args: + graph (AssetGraph): Asset graph to mutate. + + Returns: + None. + """ + factor = self.baseline + ( + self.amplitude * math.sin((2 * math.pi * self.step) / self.period) + ) + for _, data in graph.nodes.data(): + _scale_assets(data, self.node_assets, factor) + for _, _, data in graph.edges.data(): + _scale_assets(data, self.edge_assets, factor) + self.step += 1 + + +def diurnal_load( + *, + amplitude: float, + period: int, + baseline: float = 1.0, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, +) -> UpdatePolicy: + """Apply sinusoidal multiplicative load over a period. + + Args: + amplitude (float): Peak sinusoidal multiplier offset. + period (int): Number of calls in one cycle. + baseline (float): Base multiplier around which the load oscillates. + node_assets (str | list[str] | None): Optional node asset key selector. + edge_assets (str | list[str] | None): Optional edge asset key selector. + + Returns: + Stateful policy that applies diurnal load. + """ + return DiurnalLoadPolicy( + amplitude=amplitude, + period=period, + baseline=baseline, + node_assets=node_assets, + edge_assets=edge_assets, + ) + + +def _scale_assets(data, assets, factor): + """Scale selected assets inside one asset mapping. + + Args: + data (dict[str, object]): Asset mapping to mutate. + assets (str | list[str] | None): Optional asset selector. + factor (float): Multiplicative factor to apply. + + Returns: + None. + """ + apply_selected_asset_transform( + data, + assets, + transform=lambda _key, current: current * factor, + ) diff --git a/eclypse/policies/workload/traffic_matrix.py b/eclypse/policies/workload/traffic_matrix.py new file mode 100644 index 0000000..48d6082 --- /dev/null +++ b/eclypse/policies/workload/traffic_matrix.py @@ -0,0 +1,39 @@ +"""Traffic matrix workload policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +def traffic_matrix( + matrix: dict[tuple[str, str], float], + *, + asset: str = "traffic", + additive: bool = False, +) -> UpdatePolicy: + """Apply edge traffic values from a source-target matrix. + + Args: + matrix (dict[tuple[str, str], float]): + Mapping from ``(source, target)`` edge id to traffic value. + asset (str): Edge asset written by the policy. + additive (bool): Whether to add to existing traffic instead of replacing it. + + Returns: + Policy that writes traffic values onto matching edges. + """ + + def policy(graph: AssetGraph): + for edge_id, value in matrix.items(): + if not graph.has_edge(*edge_id): + continue + if additive: + graph.edges[edge_id][asset] = graph.edges[edge_id].get(asset, 0) + value + else: + graph.edges[edge_id][asset] = value + + return policy diff --git a/eclypse/remote/_node/node.py b/eclypse/remote/_node/node.py index d3f2b13..85e9acd 100644 --- a/eclypse/remote/_node/node.py +++ b/eclypse/remote/_node/node.py @@ -31,13 +31,12 @@ from .ops_thread import RemoteOpsThread if TYPE_CHECKING: - from collections.abc import ( - Callable, - ) + from collections.abc import Callable from eclypse.remote.communication import Route from eclypse.remote.service import Service from eclypse.remote.utils import RemoteOps + from eclypse.utils._logging import Logger class RemoteNode: @@ -195,6 +194,6 @@ def engine_loop(self) -> asyncio.AbstractEventLoop: return self._engine_loop @property - def logger(self) -> Any: + def logger(self) -> Logger: """Returns the logger of the node.""" return self._logger.bind(id=self.id) diff --git a/eclypse/remote/bootstrap/bootstrap.py b/eclypse/remote/bootstrap/bootstrap.py index 80deae4..823a9ea 100644 --- a/eclypse/remote/bootstrap/bootstrap.py +++ b/eclypse/remote/bootstrap/bootstrap.py @@ -10,6 +10,7 @@ dataclass, field, ) +from importlib import import_module from typing import ( TYPE_CHECKING, Any, @@ -116,13 +117,9 @@ def _create_remote( def _get_default_remote_simulator_class() -> type[Any]: """Return the default remote simulator class.""" - from eclypse.simulation._simulator import RemoteSimulator - - return RemoteSimulator + return import_module("eclypse.simulation._simulator").RemoteSimulator def _get_default_remote_node_class() -> type[Any]: """Return the default remote node class.""" - from eclypse.remote._node import RemoteNode - - return RemoteNode + return import_module("eclypse.remote._node").RemoteNode diff --git a/eclypse/remote/communication/request.py b/eclypse/remote/communication/request.py index ae364fd..0d3881a 100644 --- a/eclypse/remote/communication/request.py +++ b/eclypse/remote/communication/request.py @@ -57,7 +57,7 @@ def __init__( Defaults to None. """ self._data = data - self._timestamp = timestamp if timestamp is not None else datetime.now() + self._timestamp = timestamp or datetime.now() self._recipient_ids: list[str] = recipient_ids self._routes: list[Future[Route]] = [ diff --git a/eclypse/remote/service/rest.py b/eclypse/remote/service/rest.py index c362bd2..ec91d28 100644 --- a/eclypse/remote/service/rest.py +++ b/eclypse/remote/service/rest.py @@ -17,13 +17,20 @@ class RESTService(Service): def __init__( self, service_id: str, + store_step: bool = False, ): """Initializes a Service object. Args: service_id (str): The name of the service. + store_step (bool, optional): Whether to store the results of + each step. Defaults to False. """ - super().__init__(service_id=service_id, communication_interface="rest") + super().__init__( + service_id=service_id, + communication_interface="rest", + store_step=store_step, + ) async def step(self): """The service's main loop. diff --git a/eclypse/remote/service/service.py b/eclypse/remote/service/service.py index 0b4f724..f404439 100644 --- a/eclypse/remote/service/service.py +++ b/eclypse/remote/service/service.py @@ -21,20 +21,28 @@ import asyncio import threading +from abc import ( + ABC, + abstractmethod, +) from collections import deque from typing import ( TYPE_CHECKING, Any, cast, - get_args, ) from eclypse.remote.communication.mpi import EclypseMPI from eclypse.remote.communication.request import RouteNotFoundError from eclypse.remote.communication.rest import EclypseREST -from eclypse.utils._logging import print_exception -from eclypse.utils.defaults import DEFAULT_STEP_QUEUE_SIZE -from eclypse.utils.types import CommunicationInterface +from eclypse.utils._logging import ( + logger, + print_exception, +) +from eclypse.utils.defaults import ( + DEFAULT_STEP_QUEUE_SIZE, + SUPPORTED_COMMUNICATION_INTERFACES, +) if TYPE_CHECKING: from collections.abc import ( @@ -44,13 +52,10 @@ from eclypse.remote._node import RemoteNode from eclypse.remote.communication import EclypseCommunicationInterface from eclypse.utils._logging import Logger + from eclypse.utils.types import CommunicationInterface -_SUPPORTED_COMMUNICATION_INTERFACES = get_args(CommunicationInterface) -"""Supported runtime communication interfaces for remote services.""" - - -class Service: +class Service(ABC): """Base class for services in ECLYPSE remote applications.""" def __init__( @@ -68,7 +73,7 @@ def __init__( store_step (bool, optional): Whether to store the results of each step. Defaults to False. """ - if communication_interface not in _SUPPORTED_COMMUNICATION_INTERFACES: + if communication_interface not in SUPPORTED_COMMUNICATION_INTERFACES: raise ValueError("Invalid communication interface.") self._service_id: str = service_id @@ -108,24 +113,23 @@ async def run(self): if step_result is not None and self._store_step: self._step_queue.append(step_result) + @abstractmethod async def step(self): """The service's main loop. - This method must be overridden by the user. + Subclasses must implement this method with their service logic. Returns: Any: The result of the step (if any). - - Raises: - NotImplementedError: If the method is not overridden. """ - raise NotImplementedError("Method `step` must be overridden.") def on_deploy(self): """Hook called when the service is deployed on a node.""" + return None def on_undeploy(self): """Hook called when the service is undeployed from a node.""" + return None def _init_thread(self): """Initializes the thread for the service.""" @@ -302,9 +306,17 @@ def _start_loop(service: Service): if str(e) == "Event loop stopped before Future completed.": pass else: - print_exception(e, f"{service.id}") + print_exception(e, f"{service.id}", _exception_logger(service)) except Exception as e: - print_exception(e, f"{service.id}") + print_exception(e, f"{service.id}", _exception_logger(service)) if service._comm is not None: service._comm.disconnect() service.event_loop.close() + + +def _exception_logger(service: Service) -> Logger: + """Return a service-bound logger without masking the original exception.""" + try: + return service.logger + except Exception: + return logger.bind(id=service.id) diff --git a/eclypse/remote/utils/ray_interface.py b/eclypse/remote/utils/ray_interface.py index c91d8a4..f4e1266 100644 --- a/eclypse/remote/utils/ray_interface.py +++ b/eclypse/remote/utils/ray_interface.py @@ -8,6 +8,7 @@ import os from contextlib import redirect_stderr +from importlib import import_module from typing import ( TYPE_CHECKING, Any, @@ -98,9 +99,7 @@ def backend(self): the required dependencies are missing. """ if self._backend is None: - import ray # pylint: disable=import-outside-toplevel - - self._backend = ray + self._backend = import_module("ray") return self._backend diff --git a/eclypse/report/backends/pandas_backend.py b/eclypse/report/backends/pandas_backend.py index fc4d38b..e441e8f 100644 --- a/eclypse/report/backends/pandas_backend.py +++ b/eclypse/report/backends/pandas_backend.py @@ -6,6 +6,7 @@ from __future__ import annotations +from importlib import import_module from typing import ( TYPE_CHECKING, Any, @@ -49,9 +50,7 @@ def __init__(self): Imports pandas lazily to keep it as an optional dependency. """ super().__init__(name="pandas") - import pandas as pd - - self._pd = pd + self._pd = import_module("pandas") def _read_csv(self, source) -> DataFrame: """Read a CSV report into a pandas DataFrame.""" diff --git a/eclypse/report/backends/polars_backend.py b/eclypse/report/backends/polars_backend.py index 8140efa..f3e16e0 100644 --- a/eclypse/report/backends/polars_backend.py +++ b/eclypse/report/backends/polars_backend.py @@ -6,6 +6,7 @@ from __future__ import annotations +from importlib import import_module from typing import ( TYPE_CHECKING, Any, @@ -34,9 +35,7 @@ def __init__(self): Imports polars lazily to keep it as an optional dependency. """ super().__init__(name="polars") - import polars as pl - - self._pl = pl + self._pl = import_module("polars") def _read_csv(self, source) -> DataFrame: """Read a CSV report into a polars DataFrame.""" diff --git a/eclypse/report/backends/polars_lazy_backend.py b/eclypse/report/backends/polars_lazy_backend.py index 55a6fa9..4f58358 100644 --- a/eclypse/report/backends/polars_lazy_backend.py +++ b/eclypse/report/backends/polars_lazy_backend.py @@ -8,6 +8,7 @@ from __future__ import annotations +from importlib import import_module from typing import ( TYPE_CHECKING, Any, @@ -41,9 +42,7 @@ def __init__(self): Imports polars lazily to keep it as an optional dependency. """ super().__init__(name="polars_lazy") - import polars as pl - - self._pl = pl + self._pl = import_module("polars") def _read_csv(self, source) -> LazyFrame: """Read a CSV report into a polars LazyFrame.""" diff --git a/eclypse/report/metrics/metric.py b/eclypse/report/metrics/metric.py index 142718a..e0f0a68 100644 --- a/eclypse/report/metrics/metric.py +++ b/eclypse/report/metrics/metric.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import dataclass from typing import ( TYPE_CHECKING, ) @@ -11,10 +12,8 @@ MAX_FLOAT, ) from eclypse.utils.defaults import DEFAULT_REPORT_TYPE -from eclypse.workflow.event import ( - EventRole, - event, -) +from eclypse.workflow.event import EventRole +from eclypse.workflow.event.decorator import _event if TYPE_CHECKING: from collections.abc import ( @@ -23,11 +22,50 @@ from eclypse.utils.types import ( ActivatesOnType, + EventType, TriggerCondition, ) from eclypse.workflow.trigger import Trigger +@dataclass(frozen=True) +class _MetricOptions: + """Shared options accepted by metric decorators.""" + + name: str | None + activates_on: ActivatesOnType + trigger_every_ms: float | None + max_triggers: int | None + triggers: Trigger | list[Trigger] | None + trigger_condition: TriggerCondition | None + report: str | list[str] | None + remote: bool + verbose: bool + + +def _metric( + fn_or_class: Callable | None, + *, + event_type: EventType, + options: _MetricOptions, +) -> Callable: + """Create a metric event decorator for a report event type.""" + return _event( + fn_or_class, + name=options.name, + event_type=event_type, + role=EventRole.METRIC, + activates_on=options.activates_on, + trigger_every_ms=options.trigger_every_ms, + max_triggers=options.max_triggers, + triggers=options.triggers, + trigger_condition=options.trigger_condition, + report=options.report, + remote=options.remote, + verbose=options.verbose, + ) + + def simulation( fn_or_class: Callable | None = None, *, @@ -74,19 +112,20 @@ def simulation( Returns: Callable: The decorated function or class. """ - return event( + return _metric( fn_or_class, - name=name, event_type="simulation", - role=EventRole.METRIC, - activates_on=activates_on, - trigger_every_ms=trigger_every_ms, - max_triggers=max_triggers, - triggers=triggers, - trigger_condition=trigger_condition, - report=report, - remote=remote, - verbose=verbose, + options=_MetricOptions( + name, + activates_on, + trigger_every_ms, + max_triggers, + triggers, + trigger_condition, + report, + remote, + verbose, + ), ) @@ -134,19 +173,20 @@ def application( Returns: Callable: The decorated function. """ - return event( + return _metric( fn_or_class, - name=name, event_type="application", - role=EventRole.METRIC, - activates_on=activates_on, - trigger_every_ms=trigger_every_ms, - max_triggers=max_triggers, - triggers=triggers, - trigger_condition=trigger_condition, - report=report, - remote=remote, - verbose=verbose, + options=_MetricOptions( + name, + activates_on, + trigger_every_ms, + max_triggers, + triggers, + trigger_condition, + report, + remote, + verbose, + ), ) @@ -194,19 +234,20 @@ def service( Returns: Callable: The decorated function. """ - return event( + return _metric( fn_or_class, - name=name, event_type="service", - role=EventRole.METRIC, - activates_on=activates_on, - trigger_every_ms=trigger_every_ms, - max_triggers=max_triggers, - triggers=triggers, - trigger_condition=trigger_condition, - report=report, - remote=remote, - verbose=verbose, + options=_MetricOptions( + name, + activates_on, + trigger_every_ms, + max_triggers, + triggers, + trigger_condition, + report, + remote, + verbose, + ), ) @@ -254,19 +295,20 @@ def interaction( Returns: Callable: The decorated function. """ - return event( + return _metric( fn_or_class, - name=name, event_type="interaction", - role=EventRole.METRIC, - activates_on=activates_on, - trigger_every_ms=trigger_every_ms, - max_triggers=max_triggers, - triggers=triggers, - trigger_condition=trigger_condition, - report=report, - remote=remote, - verbose=verbose, + options=_MetricOptions( + name, + activates_on, + trigger_every_ms, + max_triggers, + triggers, + trigger_condition, + report, + remote, + verbose, + ), ) @@ -314,19 +356,20 @@ def infrastructure( Returns: Callable: The decorated function. """ - return event( + return _metric( fn_or_class, - name=name, event_type="infrastructure", - role=EventRole.METRIC, - activates_on=activates_on, - trigger_every_ms=trigger_every_ms, - max_triggers=max_triggers, - triggers=triggers, - trigger_condition=trigger_condition, - report=report, - remote=remote, - verbose=verbose, + options=_MetricOptions( + name, + activates_on, + trigger_every_ms, + max_triggers, + triggers, + trigger_condition, + report, + remote, + verbose, + ), ) @@ -372,18 +415,20 @@ def node( Returns: Callable: The decorated function. """ - return event( + return _metric( fn_or_class, - name=name, event_type="node", - role=EventRole.METRIC, - activates_on=activates_on, - trigger_every_ms=trigger_every_ms, - max_triggers=max_triggers, - triggers=triggers, - trigger_condition=trigger_condition, - report=report, - verbose=verbose, + options=_MetricOptions( + name, + activates_on, + trigger_every_ms, + max_triggers, + triggers, + trigger_condition, + report, + False, + verbose, + ), ) @@ -431,17 +476,18 @@ def link( Returns: Callable: The decorated function. """ - return event( + return _metric( fn_or_class, - name=name, event_type="link", - role=EventRole.METRIC, - activates_on=activates_on, - trigger_every_ms=trigger_every_ms, - max_triggers=max_triggers, - triggers=triggers, - trigger_condition=trigger_condition, - report=report, - remote=remote, - verbose=verbose, + options=_MetricOptions( + name, + activates_on, + trigger_every_ms, + max_triggers, + triggers, + trigger_condition, + report, + remote, + verbose, + ), ) diff --git a/eclypse/report/report.py b/eclypse/report/report.py index 42042b9..dedf7f0 100644 --- a/eclypse/report/report.py +++ b/eclypse/report/report.py @@ -284,6 +284,52 @@ def query(self, report_type: EventType) -> ReportQuery: """Create a composable query for the given report type.""" return ReportQuery(self, report_type) + def describe(self) -> str: + """Return a compact human-readable summary of available reports. + + The summary includes total rows, unique simulation steps, unique metric + callback IDs, and a per-report breakdown. Missing report files are skipped. + + Returns: + A summary string such as ``"12 rows x 3 steps x 5 metrics"``. + """ + total_rows = 0 + steps: set[Any] = set() + metrics: set[Any] = set() + applications: set[Any] = set() + breakdown: list[str] = [] + + for report_type in REPORT_TYPES: + try: + self._read_frame(report_type) + except FileNotFoundError: + continue + + frame = self.stats[report_type] + if frame is None: + continue + materialized = _materialize_frame(frame) + row_count = _frame_row_count(materialized) + total_rows += row_count + + report_steps = set(_column_values(materialized, "n_event")) + report_metrics = set(_column_values(materialized, "callback_id")) + steps.update(report_steps) + metrics.update(report_metrics) + applications.update(_column_values(materialized, "application_id")) + + breakdown.append( + f"{report_type}: {row_count} rows, {len(report_metrics)} metrics" + ) + + summary = ( + f"{total_rows} rows x {len(steps)} steps x {len(metrics)} metrics" + f" | {len(applications)} applications" + ) + if breakdown: + return f"{summary} | " + "; ".join(breakdown) + return summary + def get_dataframes( self, report_types: list[EventType] | None = None, @@ -444,3 +490,53 @@ def _resolve_report_format( return cast("ReportFormat", config_format) return cast("ReportFormat", DEFAULT_REPORT_TYPE) + + +def _materialize_frame(frame: Any) -> Any: + """Materialise lazy frames for summary inspection.""" + collect = getattr(frame, "collect", None) + if callable(collect): + return collect() + return frame + + +def _frame_row_count(frame: Any) -> int: + """Return a frame's row count across supported backends.""" + if hasattr(frame, "height"): + return int(frame.height) + try: + return len(frame) + except TypeError: + return 0 + + +def _column_values(frame: Any, column: str) -> list[Any]: + """Return non-null values for a column across supported backends.""" + if isinstance(frame, list): + return [ + row[column] for row in frame if column in row and row[column] is not None + ] + + columns = getattr(frame, "columns", None) + if columns is None or column not in columns: + return [] + + get_column = getattr(frame, "get_column", None) + if callable(get_column): + series = get_column(column) + drop_nulls = getattr(series, "drop_nulls", None) + if callable(drop_nulls): + series = drop_nulls() + return list(series.to_list()) + + series = frame[column] + dropna = getattr(series, "dropna", None) + if callable(dropna): + series = dropna() + to_list = getattr(series, "to_list", None) + if callable(to_list): + return list(to_list()) + tolist = getattr(series, "tolist", None) + if callable(tolist): + return list(tolist()) + return list(series) diff --git a/eclypse/report/reporters/parquet.py b/eclypse/report/reporters/parquet.py index 94b3eb8..bdda3a7 100644 --- a/eclypse/report/reporters/parquet.py +++ b/eclypse/report/reporters/parquet.py @@ -4,6 +4,7 @@ import asyncio from datetime import datetime as dt +from importlib import import_module from pathlib import Path from typing import ( TYPE_CHECKING, @@ -35,9 +36,7 @@ def __init__(self, report_path: str | Path): async def init(self): """Initialize the reporter and import polars lazily.""" await super().init() - import polars as pl # pylint: disable=import-outside-toplevel - - self._pl = pl + self._pl = import_module("polars") def report( self, diff --git a/eclypse/report/reporters/tensorboard.py b/eclypse/report/reporters/tensorboard.py index 5debbe8..eeaa00f 100644 --- a/eclypse/report/reporters/tensorboard.py +++ b/eclypse/report/reporters/tensorboard.py @@ -9,6 +9,7 @@ from __future__ import annotations +from importlib import import_module from typing import ( TYPE_CHECKING, Any, @@ -39,10 +40,7 @@ def __init__(self, report_path: str | Path): async def init(self): """Initialize the TensorBoard reporter.""" - from tensorboardX import ( # pylint: disable=import-outside-toplevel - SummaryWriter, - ) - + SummaryWriter = import_module("tensorboardX").SummaryWriter self._writer = SummaryWriter(log_dir=self.report_path) async def close(self): diff --git a/eclypse/simulation/_simulator/local.py b/eclypse/simulation/_simulator/local.py index d402d55..cf4a1ca 100644 --- a/eclypse/simulation/_simulator/local.py +++ b/eclypse/simulation/_simulator/local.py @@ -47,7 +47,7 @@ Placement, PlacementView, ) - from eclypse.placement.strategies.strategy import PlacementStrategy + from eclypse.placement.strategies import PlacementStrategy from eclypse.simulation.config import SimulationConfig from eclypse.utils._logging import Logger from eclypse.workflow.event import EclypseEvent @@ -85,14 +85,17 @@ def __init__( ) self._infrastructure = infrastructure - self._manager = PlacementManager(infrastructure=self._infrastructure) + self._manager = PlacementManager( + infrastructure=self._infrastructure, + default_strategy=self._config.default_strategy, + ) self._events: dict[str, EclypseEvent] = { event.name: event for event in self._config.events } for event in self._events.values(): event.attach_simulator(self) - event.trigger_bucket.init() + event.trigger_bucket.prepare() # Simulation state self._event_loop: asyncio.AbstractEventLoop = asyncio.new_event_loop() @@ -217,7 +220,7 @@ async def run(self): except (asyncio.QueueEmpty, TimeoutError): pass except Exception as e: - print_exception(e, self.__class__.__name__) + print_exception(e, self.__class__.__name__, self.logger) if self.status != SimulationState.STOPPING: await self.enqueue_event(STOP_EVENT) finally: diff --git a/eclypse/simulation/config.py b/eclypse/simulation/config.py index 75b5aa1..1c63656 100644 --- a/eclypse/simulation/config.py +++ b/eclypse/simulation/config.py @@ -47,12 +47,12 @@ ) if TYPE_CHECKING: - from collections.abc import ( - Callable, - ) + from collections.abc import Callable + from eclypse.placement.strategies import PlacementStrategy from eclypse.report import FrameBackend from eclypse.report.reporter import Reporter + from eclypse.utils._logging import Logger from eclypse.utils.types import ( LogLevel, ReportBackend, @@ -65,8 +65,13 @@ class SimulationConfig: """Configuration object for a simulation runtime.""" - step_every_ms: Literal["manual", "auto"] | float | None = "manual" - """Cadence of the driving event in milliseconds, or ``"manual"``/``"auto"``.""" + step_every_ms: Literal["manual", "auto"] | float | None = "auto" + """Cadence of the driving event. + + ``"auto"`` continuously advances local simulations and resolves to manual mode + for remote simulations. Use ``None`` or ``"manual"`` for explicit manual + stepping, or pass a number for a millisecond cadence. + """ timeout: float | None = None """Maximum wall-clock duration of the simulation, in seconds.""" @@ -108,13 +113,20 @@ class SimulationConfig: remote: bool | RemoteBootstrap = False """Whether to run in remote emulation mode, or the bootstrap to use for it.""" + default_strategy: PlacementStrategy | None = None + """Default placement strategy used when ``Simulation.register`` gets none.""" + _runtime_prepared: bool = field(init=False, default=False, repr=False) def __post_init__(self): """Normalize permissive user input into a runtime-ready configuration.""" - self.step_every_ms = self._resolve_step_every_ms(self.step_every_ms) self.seed = self.seed if self.seed is not None else randint(0, int(1e9)) self.path = self._resolve_path(self.path) + self.remote = self._resolve_remote(self.remote) + self.step_every_ms = self._resolve_step_every_ms( + self.step_every_ms, + remote=self.remote is not None, + ) self.report_format = cast( "ReportFormat", ( @@ -131,7 +143,6 @@ def __post_init__(self): else DEFAULT_REPORT_BACKEND ), ) - self.remote = self._resolve_remote(self.remote) self.events = self._build_events(self.events, self.include_default_metrics) self._apply_default_report_format(self.events) self.reporters = self._resolve_reporters(self.reporters, self.events) @@ -171,7 +182,7 @@ def _resolve_reporters( "dict[str, type[Reporter]]", get_default_reporters(report_types), ) - resolved_reporters.update(reporters if reporters is not None else {}) + resolved_reporters.update(reporters or {}) return resolved_reporters def _ensure_optional_dependencies(self): @@ -179,11 +190,11 @@ def _ensure_optional_dependencies(self): raise RuntimeError("Reporters must be resolved before dependency checks.") if TENSORBOARD_REPORT_DIR in self.reporters: - _require_module("tensorboardX", extras_name="tboard") + _require_module("tensorboardX") if PARQUET_REPORT_DIR in self.reporters: _require_module("polars") if self.remote is not None: - _require_module("ray", extras_name="remote") + _require_module("ray") if self.report_backend == "pandas": _require_module("pandas") if self.report_backend in ("polars", "polars_lazy"): @@ -192,11 +203,13 @@ def _ensure_optional_dependencies(self): @staticmethod def _resolve_step_every_ms( step_every_ms: Literal["manual", "auto"] | float | None, + *, + remote: bool = False, ) -> float | None: if isinstance(step_every_ms, str) and step_every_ms == "manual": return None if isinstance(step_every_ms, str) and step_every_ms == "auto": - return 0.0 + return None if remote else 0.0 if isinstance(step_every_ms, (float, int)) or step_every_ms is None: return step_every_ms raise ValueError("step_every_ms must be a float, 'manual', 'auto' or None.") @@ -205,7 +218,11 @@ def _resolve_step_every_ms( def _resolve_path(path: str | Path | None) -> Path: base_path = get_default_sim_path() if path is None else Path(path) if base_path.exists(): - return Path(f"{base_path}-{strftime('%Y%m%d_%H%M%S')}") + resolved = Path(f"{base_path}-{strftime('%Y%m%d_%H%M%S')}") + logger.bind(id="SimulationConfig").info( + f"Target path exists; writing to {resolved} instead" + ) + return resolved return base_path @staticmethod @@ -269,7 +286,7 @@ def callbacks(self) -> list[EclypseEvent]: ] @property - def logger(self) -> Any: + def logger(self) -> Logger: """Logger bound to the config component.""" return logger.bind(id="SimulationConfig") @@ -316,21 +333,22 @@ def to_dict(self) -> dict[str, Any]: else self.report_backend ), "remote": bool(self.remote), + "default_strategy": ( + self.default_strategy.__class__.__name__ + if self.default_strategy is not None + else None + ), } -def _require_module(module_name: str, extras_name: str | None = None): +def _require_module(module_name: str): """Require a module and raise an ImportError if it is not found.""" try: __import__(module_name) except ImportError as e: - install_hint = ( - f"pip install eclypse[{extras_name}]" - if extras_name is not None - else f"pip install {module_name}" - ) raise ImportError( - f"{module_name} is not installed. Please install it with '{install_hint}'." + f"{module_name} is not installed. Please install it with " + f"'pip install {module_name}'." ) from e diff --git a/eclypse/simulation/simulation.py b/eclypse/simulation/simulation.py index eaefc1b..fd97253 100644 --- a/eclypse/simulation/simulation.py +++ b/eclypse/simulation/simulation.py @@ -5,7 +5,6 @@ import json from typing import ( TYPE_CHECKING, - Any, cast, ) @@ -29,11 +28,12 @@ from eclypse.graph.application import Application from eclypse.graph.infrastructure import Infrastructure - from eclypse.placement.strategies.strategy import PlacementStrategy + from eclypse.placement.strategies import PlacementStrategy from eclypse.remote.bootstrap.bootstrap import RemoteBootstrap from eclypse.report import FrameBackend from eclypse.simulation._simulator.local import SimulationState from eclypse.simulation._simulator.remote import RemoteSimulator + from eclypse.utils._logging import Logger class Simulation: @@ -46,9 +46,7 @@ def __init__( ): """Create a simulation bound to an infrastructure and configuration.""" self.infrastructure = infrastructure - self._sim_config = ( - simulation_config if simulation_config is not None else SimulationConfig() - ) + self._sim_config = simulation_config or SimulationConfig() self._sim_config.prepare_runtime() self.remote: RemoteBootstrap | None = cast( @@ -76,6 +74,19 @@ def prepare_runtime(self): """Prepare the process environment required by the simulation runtime.""" self._sim_config.prepare_runtime() + def __enter__(self) -> Simulation: + """Return the simulation so it can be managed with a ``with`` block.""" + return self + + def __exit__(self, *_exc_info): + """Stop the simulation when leaving a context-managed block.""" + try: + self.stop() + except Exception as error: + self.logger.exception(f"Failed to stop simulation during cleanup: {error}") + raise + return False + def start(self): """Start the simulation.""" self.prepare_runtime() @@ -90,14 +101,36 @@ def start(self): _local_remote_event_call(self.simulator, self.remote, START_EVENT) self._finished_logged = False + self._log_configuration() + self.logger.log("ECLYPSE", "Simulation started.") + + def _log_configuration(self): + """Log the run configuration that gives context to subsequent events.""" + report_backend = self._sim_config.report_backend + if report_backend is not None and not isinstance(report_backend, str): + report_backend = report_backend.name + self.logger.log( "ECLYPSE", - "Simulation started | " + "Simulation configuration | " + format_log_kv( infrastructure=self.infrastructure.id, - # apps=[app.id for app in self.applications.values()], - # path=self._sim_config.path, - # remote=self.remote is not None, + path=self._sim_config.path, + remote=self.remote is not None, + step_every_ms=self._sim_config.step_every_ms, + timeout=self._sim_config.timeout, + max_steps=self._sim_config.max_steps, + seed=self._sim_config.seed, + report_format=self._sim_config.report_format, + report_backend=report_backend, + log_level=self._sim_config.log_level, + log_to_file=self._sim_config.log_to_file, + include_default_metrics=self._sim_config.include_default_metrics, + default_strategy=( + self._sim_config.default_strategy.__class__.__name__ + if self._sim_config.default_strategy is not None + else None + ), ), ) @@ -138,33 +171,56 @@ def wait(self, timeout: float | None = None): raise interrupted = True - self.logger.log( - "ECLYPSE", - "Simulation stop requested. Press Ctrl+C again to stop the simulation.", + self.logger.warning( + ( + "Simulation stop requested. Press Ctrl+C again to " + "stop the simulation." + ), ) self.stop(blocking=False) timeout = None + def run(self, steps: int | None = None, seconds: float | None = None): + """Start the simulation and wait for it to complete. + + Args: + steps (int | None): If provided, manually trigger this many simulation + steps before stopping. + seconds (float | None): If provided, wait for at most this many seconds + before requesting a stop. + """ + if steps is not None and seconds is not None: + raise ValueError("Only one of 'steps' and 'seconds' can be provided.") + if steps is not None and steps < 0: + raise ValueError("'steps' must be greater than or equal to 0.") + if seconds is not None and seconds < 0: + raise ValueError("'seconds' must be greater than or equal to 0.") + + self.start() + if steps is not None: + for _ in range(steps): + self.step() + self.stop() + return + + self.wait(timeout=seconds) + if seconds is not None and self.status.name != "IDLE": + self.stop() + def register( self, application: Application, placement_strategy: PlacementStrategy | None = None, ): """Include an application in the simulation.""" - if placement_strategy is None: - if not self.infrastructure.has_strategy: - raise ValueError( - "Must provide a global placement strategy for the infrastructure " - + f"or a placement strategy for the application {application.id}" - ) - elif self.infrastructure.has_strategy: - self.logger.warning( - "Ignoring the provided placement strategy, using the global one." - + " Unset the global strategy to use the provided one." + if placement_strategy is None and self._sim_config.default_strategy is None: + raise ValueError( + "Must provide a default placement strategy in SimulationConfig " + + f"or a placement strategy for application {application.id}" ) if self.remote: - if application.has_logic: + if application.has_service_implementations: ray_backend.get( self.simulator.register.remote( # type: ignore[attr-defined] application, @@ -189,7 +245,7 @@ def applications(self) -> dict[str, Application]: return self.simulator.applications @property - def logger(self) -> Any: + def logger(self) -> Logger: """Logger bound to the simulation component.""" return self._logger.bind(id="Simulation") diff --git a/eclypse/utils/__init__.py b/eclypse/utils/__init__.py index 88e530a..d33e659 100644 --- a/eclypse/utils/__init__.py +++ b/eclypse/utils/__init__.py @@ -2,3 +2,7 @@ It comprises logging, constants and default values used in ECLYPSE. """ + +from ._logging import format_log_kv + +__all__ = ["format_log_kv"] diff --git a/eclypse/utils/_logging.py b/eclypse/utils/_logging.py index dd18cdd..7291328 100644 --- a/eclypse/utils/_logging.py +++ b/eclypse/utils/_logging.py @@ -4,7 +4,10 @@ import os import traceback -from sys import stdout +from sys import ( + stderr, + stdout, +) from typing import ( TYPE_CHECKING, Any, @@ -25,9 +28,9 @@ def config_logger(): """Configure the loguru logger. - It adds a custom level ECLYPSE for the logs related to the Eclypse library. The logs - are printed to stdout and saved to a file if the LOG_FILE environment variable is - set. + It adds custom ECLYPSE levels for library logs and async exception reports. + Regular logs are printed to stdout, exception reports are printed to stderr, + and all logs are saved to a file if the LOG_FILE environment variable is set. """ head = "{time:HH:mm:ss.SSS} | {level} | " fmt = head + "{extra[id]} - {message}" @@ -36,6 +39,10 @@ def config_logger(): eclypse_fmt = head + "{extra[id]} - {message}" if "ECLYPSE" not in logger.__dict__["_core"].__dict__["levels"]: logger.level("ECLYPSE", no=15, color="", icon="🌘") + if "ECLYPSE_EXCEPTION" not in logger.__dict__["_core"].__dict__["levels"]: + logger.level("ECLYPSE_EXCEPTION", no=45, color="", icon="!") + + exception_fmt = head + "{extra[id]} - {message}" level = os.getenv(LOG_LEVEL, "ECLYPSE") file = os.getenv(LOG_FILE) @@ -57,6 +64,14 @@ def config_logger(): "level": level, "enqueue": True, }, + { + "sink": stderr, + "filter": _is_eclypse_exception, + "format": exception_fmt, + "colorize": True, + "level": level, + "enqueue": True, + }, ] if file: handlers.append({"sink": file, "format": fmt, "enqueue": True, "level": level}) @@ -67,24 +82,32 @@ def _is_eclypse(record: dict[str, Any]): return record["level"].name == "ECLYPSE" +def _is_eclypse_exception(record: dict[str, Any]): + return record["level"].name == "ECLYPSE_EXCEPTION" + + def _is_not_eclypse(record: dict[str, Any]): - return record["level"].name != "ECLYPSE" + return record["level"].name not in {"ECLYPSE", "ECLYPSE_EXCEPTION"} -def print_exception(e: Exception, raised_by: str): - """Print the exception traceback and message. +def print_exception(e: Exception, raised_by: str, exception_logger: Logger): + """Log an exception traceback and message. - This is an internal function used to catch and print exception from asyncio tasks. + This is an internal helper used to surface exceptions from asyncio tasks. Args: e (Exception): The exception raised. raised_by (str): The name of the function that raised the exception. + exception_logger (Logger): Logger bound to the component that caught it. """ tb_lines = traceback.format_tb(e.__traceback__) tb_string = "".join(tb_lines) - print("Traceback (most recent call last):") - print(tb_string) - print(f"{e.__class__.__name__} in {raised_by}: {e}") + exception_logger.log( + "ECLYPSE_EXCEPTION", + "Traceback (most recent call last):\n" + + tb_string + + f"{e.__class__.__name__} in {raised_by}: {e}", + ) def format_log_kv(separator: str = " | ", **values: Any) -> str: @@ -154,6 +177,10 @@ def log_assets_violations( __all__ = [ "Logger", "format_log_kv", - "log_placement_violations", + "log_assets_violations", "print_exception", ] + +# Configure the default ECLYPSE logging format at import time so components +# created before simulation runtime setup still emit consistently formatted logs. +config_logger() diff --git a/eclypse/utils/constants.py b/eclypse/utils/constants.py index 677411d..d19c373 100644 --- a/eclypse/utils/constants.py +++ b/eclypse/utils/constants.py @@ -7,10 +7,10 @@ # Numeric domains MIN_FLOAT = 0.0 -"""Smallest domain value accepted by numeric assets.""" +"""Smallest reasonable value for bounded numeric defaults.""" MAX_FLOAT = 1e9 -"""Largest domain value used for bounded numeric defaults.""" +"""Largest reasonable value for bounded numeric defaults.""" FLOAT_EPSILON = sys.float_info.min """Smallest positive representable float.""" diff --git a/eclypse/utils/defaults.py b/eclypse/utils/defaults.py index 98b9fd2..2697680 100644 --- a/eclypse/utils/defaults.py +++ b/eclypse/utils/defaults.py @@ -3,8 +3,10 @@ from __future__ import annotations from pathlib import Path +from typing import get_args from eclypse.utils.constants import MAX_FLOAT +from eclypse.utils.types import CommunicationInterface # Reporting @@ -60,6 +62,9 @@ } """Default Ray environment variables applied to simulation runtimes.""" +SUPPORTED_COMMUNICATION_INTERFACES = get_args(CommunicationInterface) +"""Supported runtime communication interfaces for services and builders.""" + # Paths @@ -83,6 +88,7 @@ def get_default_sim_path() -> Path: "PARQUET_REPORT_DIR", "SIMULATION_CONFIG_FILENAME", "SIMULATION_LOG_FILENAME", + "SUPPORTED_COMMUNICATION_INTERFACES", "TENSORBOARD_REPORT_DIR", "get_default_sim_path", ] diff --git a/eclypse/utils/types.py b/eclypse/utils/types.py index 7878e12..3880b60 100644 --- a/eclypse/utils/types.py +++ b/eclypse/utils/types.py @@ -96,14 +96,24 @@ class ValueAdjustmentOverride(TypedDict, total=False): Distribution: TypeAlias = Literal[ "beta", + "bernoulli", + "constant", + "discrete", + "empirical", + "exponential", "gamma", "lognormal", "normal", + "pareto", + "poisson", + "triangular", + "truncated_normal", "uniform", + "weibull", ] """Type alias for the supported built-in distribution policies.""" -ReplayTarget: TypeAlias = Literal["nodes", "edges"] +ReplayTarget: TypeAlias = Literal["nodes", "edges", "graph"] """Type alias for the supported replay targets.""" MissingPolicyBehaviour: TypeAlias = Literal["ignore", "error"] diff --git a/eclypse/workflow/__init__.py b/eclypse/workflow/__init__.py index 6331ef4..cb916ff 100644 --- a/eclypse/workflow/__init__.py +++ b/eclypse/workflow/__init__.py @@ -1 +1,39 @@ -"""Package for workflow management, including events and triggers.""" +"""Workflow primitives for defining events, triggers, and callbacks.""" + +from .event import ( + EclypseEvent, + EventRole, + after, + every, + get_default_events, + once_at, +) +from .trigger import ( + CascadeTrigger, + PeriodicCascadeTrigger, + PeriodicTrigger, + RandomCascadeTrigger, + RandomTrigger, + ScheduledCascadeTrigger, + ScheduledTrigger, + Trigger, + TriggerBucket, +) + +__all__ = [ + "CascadeTrigger", + "EclypseEvent", + "EventRole", + "PeriodicCascadeTrigger", + "PeriodicTrigger", + "RandomCascadeTrigger", + "RandomTrigger", + "ScheduledCascadeTrigger", + "ScheduledTrigger", + "Trigger", + "TriggerBucket", + "after", + "every", + "get_default_events", + "once_at", +] diff --git a/eclypse/workflow/event/__init__.py b/eclypse/workflow/event/__init__.py index b5a0706..23e406e 100644 --- a/eclypse/workflow/event/__init__.py +++ b/eclypse/workflow/event/__init__.py @@ -1,16 +1,22 @@ """Package for managing events in the Eclypse framework. -It provides a decorator to define events for the simulation. +It provides scheduled decorators to define events for the simulation. """ from .event import EclypseEvent -from .decorator import event +from .decorator import ( + after, + every, + once_at, +) from .defaults import get_default_events from .role import EventRole __all__ = [ "EclypseEvent", "EventRole", - "event", + "after", + "every", "get_default_events", + "once_at", ] diff --git a/eclypse/workflow/event/decorator.py b/eclypse/workflow/event/decorator.py index f8d91a0..2860667 100644 --- a/eclypse/workflow/event/decorator.py +++ b/eclypse/workflow/event/decorator.py @@ -1,4 +1,4 @@ -"""Module containing the event decorator. +"""Module containing convenience event decorators. An event is a function that is triggered by other events or by the simulation itself. """ @@ -7,11 +7,16 @@ import inspect import re +from datetime import timedelta from typing import ( TYPE_CHECKING, ) from eclypse.utils.constants import MAX_FLOAT +from eclypse.workflow.trigger import ( + PeriodicTrigger, + ScheduledTrigger, +) from .event import ( EclypseEvent, @@ -32,12 +37,13 @@ from eclypse.workflow.trigger.trigger import Trigger -def event( +def _event( fn_or_class: Callable | None = None, *, name: str | None = None, event_type: EventType | None = None, activates_on: ActivatesOnType | None = None, + schedule_trigger: Trigger | None = None, trigger_every_ms: float | None = None, max_triggers: int | None = int(MAX_FLOAT), triggers: Trigger | list[Trigger] | None = None, @@ -47,7 +53,7 @@ def event( remote: bool = False, verbose: bool = False, ) -> Callable: - """A decorator to define an event in the simulation. + """Build an event wrapper from a callable. Args: fn_or_class (Callable | None, optional): The function or class to decorate @@ -58,6 +64,7 @@ def event( The type of the event. Defaults to None. activates_on (ActivatesOnType | None, optional): The conditions that will trigger the event. Defaults to None. + schedule_trigger: Trigger added by the public scheduling decorator. trigger_every_ms (float | None, optional): The time in milliseconds between each trigger of the event. Defaults to None. max_triggers (int | None, optional): The maximum number of times the event @@ -92,6 +99,8 @@ def decorator(decoratee: Callable) -> Callable: _triggers = ( triggers if isinstance(triggers, list) else [triggers] if triggers else [] ) + if schedule_trigger: + _triggers.insert(0, schedule_trigger) curr_opt = { "name": _name, @@ -128,6 +137,154 @@ def __new__(cls, *args, **kwargs): return decorator +def every( + fn_or_class: Callable | None = None, + *, + ms: float, + name: str | None = None, + event_type: EventType | None = None, + activates_on: ActivatesOnType | None = None, + max_triggers: int | None = int(MAX_FLOAT), + triggers: Trigger | list[Trigger] | None = None, + trigger_condition: TriggerCondition | None = "any", + role: EventRole = EventRole.EVENT, + report: str | list[str] | None = None, + remote: bool = False, + verbose: bool = False, +) -> Callable: + """Define an event that fires periodically. + + Args: + fn_or_class: The function or class to decorate. + ms: The period between triggers in milliseconds. + name: Optional event name. Defaults to the decorated object name. + event_type: Optional report event type. + activates_on: Cascade activation rules. + max_triggers: Maximum number of firings. + triggers: Additional triggers to combine with the periodic trigger. + trigger_condition: Whether any or all triggers must fire. + role: Workflow role assigned to the event. + report: Report formats generated by the event. + remote: Whether the event runs remotely. + verbose: Whether verbose event logging is enabled. + + Returns: + The decorated event wrapper. + """ + return _event( + fn_or_class, + name=name, + event_type=event_type, + activates_on=activates_on, + schedule_trigger=PeriodicTrigger(ms), + max_triggers=max_triggers, + triggers=triggers, + trigger_condition=trigger_condition, + role=role, + report=report, + remote=remote, + verbose=verbose, + ) + + +def after( + fn_or_class: Callable | None = None, + *, + sim_seconds: float, + name: str | None = None, + event_type: EventType | None = None, + activates_on: ActivatesOnType | None = None, + max_triggers: int | None = 1, + triggers: Trigger | list[Trigger] | None = None, + trigger_condition: TriggerCondition | None = "any", + role: EventRole = EventRole.EVENT, + report: str | list[str] | None = None, + remote: bool = False, + verbose: bool = False, +) -> Callable: + """Define an event that fires after a simulation-time delay. + + Args: + fn_or_class: The function or class to decorate. + sim_seconds: Delay in simulation seconds before the event can fire. + name: Optional event name. Defaults to the decorated object name. + event_type: Optional report event type. + activates_on: Cascade activation rules. + max_triggers: Maximum number of firings. Defaults to once. + triggers: Additional triggers to combine with the scheduled trigger. + trigger_condition: Whether any or all triggers must fire. + role: Workflow role assigned to the event. + report: Report formats generated by the event. + remote: Whether the event runs remotely. + verbose: Whether verbose event logging is enabled. + + Returns: + The decorated event wrapper. + """ + return _event( + fn_or_class, + name=name, + event_type=event_type, + activates_on=activates_on, + schedule_trigger=ScheduledTrigger(timedelta(seconds=sim_seconds)), + max_triggers=max_triggers, + triggers=triggers, + trigger_condition=trigger_condition, + role=role, + report=report, + remote=remote, + verbose=verbose, + ) + + +def once_at( + fn_or_class: Callable | None = None, + *, + sim_seconds: float, + name: str | None = None, + event_type: EventType | None = None, + activates_on: ActivatesOnType | None = None, + triggers: Trigger | list[Trigger] | None = None, + trigger_condition: TriggerCondition | None = "any", + role: EventRole = EventRole.EVENT, + report: str | list[str] | None = None, + remote: bool = False, + verbose: bool = False, +) -> Callable: + """Define an event that fires once at a simulation-time offset. + + Args: + fn_or_class: The function or class to decorate. + sim_seconds: Simulation-time offset in seconds. + name: Optional event name. Defaults to the decorated object name. + event_type: Optional report event type. + activates_on: Cascade activation rules. + triggers: Additional triggers to combine with the scheduled trigger. + trigger_condition: Whether any or all triggers must fire. + role: Workflow role assigned to the event. + report: Report formats generated by the event. + remote: Whether the event runs remotely. + verbose: Whether verbose event logging is enabled. + + Returns: + The decorated event wrapper. + """ + return after( + fn_or_class, + sim_seconds=sim_seconds, + name=name, + event_type=event_type, + activates_on=activates_on, + max_triggers=1, + triggers=triggers, + trigger_condition=trigger_condition, + role=role, + report=report, + remote=remote, + verbose=verbose, + ) + + def _camel_to_snake(name: str) -> str: """Convert a CamelCase string to a snake_case string. diff --git a/eclypse/workflow/event/event.py b/eclypse/workflow/event/event.py index f13a8a0..29226cc 100644 --- a/eclypse/workflow/event/event.py +++ b/eclypse/workflow/event/event.py @@ -43,6 +43,7 @@ PlacementView, ) from eclypse.simulation._simulator.local import Simulator + from eclypse.utils._logging import Logger from eclypse.utils.types import ( EventType, TriggerCondition, @@ -349,7 +350,7 @@ def remote(self) -> bool: return self._remote @property - def logger(self) -> Any: + def logger(self) -> Logger: """Get a logger for the graph, binding the graph id in the logs. Returns: diff --git a/eclypse/workflow/trigger/__init__.py b/eclypse/workflow/trigger/__init__.py index dfc3b18..d464e19 100644 --- a/eclypse/workflow/trigger/__init__.py +++ b/eclypse/workflow/trigger/__init__.py @@ -10,6 +10,7 @@ PeriodicTrigger, ScheduledTrigger, ) +from .bucket import TriggerBucket from .cascade import ( CascadeTrigger, @@ -27,4 +28,5 @@ "ScheduledCascadeTrigger", "ScheduledTrigger", "Trigger", + "TriggerBucket", ] diff --git a/eclypse/workflow/trigger/bucket.py b/eclypse/workflow/trigger/bucket.py index 0c1d208..5434c1a 100644 --- a/eclypse/workflow/trigger/bucket.py +++ b/eclypse/workflow/trigger/bucket.py @@ -53,14 +53,14 @@ def __init__( self._n_triggers: int = 0 self._n_executions: int = 0 - def init(self): + def prepare(self): """Prepare the trigger for use. This method can be overridden in subclasses to perform any necessary initialization before the trigger is used. """ for trigger in self.triggers: - trigger.init() + trigger.prepare() def trigger(self, trigger_event: EclypseEvent | None = None) -> bool: """Check if the trigger should fire. diff --git a/eclypse/workflow/trigger/cascade.py b/eclypse/workflow/trigger/cascade.py index 74ab662..7a927c5 100644 --- a/eclypse/workflow/trigger/cascade.py +++ b/eclypse/workflow/trigger/cascade.py @@ -146,7 +146,7 @@ def __init__( self.seed = seed self.rnd = None - def init(self): + def prepare(self): """Initialize the random number generator.""" self.seed = int(os.getenv(RND_SEED)) if self.seed is None else self.seed self.rnd = random.Random(self.seed) @@ -154,7 +154,9 @@ def init(self): def trigger(self, trigger_event: EclypseEvent | None = None) -> bool: """Check if the trigger should fire based on its condition.""" if self.rnd is None: - raise RuntimeError("Trigger not initialised. Call init() before trigger().") + raise RuntimeError( + "Trigger not initialised. Call prepare() before trigger()." + ) return super().trigger(trigger_event) and self.rnd.random() < self.probability def __repr__(self) -> str: diff --git a/eclypse/workflow/trigger/trigger.py b/eclypse/workflow/trigger/trigger.py index 6b7d3f6..10ecf13 100644 --- a/eclypse/workflow/trigger/trigger.py +++ b/eclypse/workflow/trigger/trigger.py @@ -46,7 +46,7 @@ def trigger(self, _: EclypseEvent | None = None) -> bool: bool: True if the trigger should fire, False otherwise. """ - def init(self): + def prepare(self): """Prepare the trigger for use. This method can be overridden in subclasses to perform any necessary @@ -130,7 +130,7 @@ def __init__( self._init_time: datetime | None = None self._scheduled_times: list[datetime] = [] - def init(self): + def prepare(self): """Prepare the trigger by setting the initial time.""" self._init_time = datetime.now() self._scheduled_timedelta = sorted(self._scheduled_timedelta) @@ -141,7 +141,12 @@ def init(self): def trigger(self, _: EclypseEvent | None = None) -> bool: """Return True if the current call count matches a scheduled time.""" if self._init_time is None: - raise RuntimeError("Trigger not initialised. Call init() before trigger().") + raise RuntimeError( + "Trigger not initialised. Call prepare() before trigger()." + ) + + if not self._scheduled_times: + return False current_time = datetime.now() if current_time >= self._scheduled_times[0]: @@ -171,7 +176,7 @@ def __init__(self, probability: float = 0.5, seed: int | None = None): self.seed = seed self.rnd = None - def init(self): + def prepare(self): """Initialize the random number generator.""" self.seed = int(os.getenv(RND_SEED)) if self.seed is None else self.seed self.rnd = random.Random(self.seed) @@ -179,7 +184,9 @@ def init(self): def trigger(self, _: EclypseEvent | None = None) -> bool: """Check if the trigger should fire based on its probability.""" if self.rnd is None: - raise RuntimeError("Trigger not initialised. Call init() before trigger().") + raise RuntimeError( + "Trigger not initialised. Call prepare() before trigger()." + ) return self.rnd.random() < self.probability def __repr__(self) -> str: diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..17a846b --- /dev/null +++ b/examples/README.md @@ -0,0 +1,13 @@ +# ECLYPSE Examples + +Run examples from the repository root with Poetry scripts. + +| Name | Demonstrates | How to run | Docs | +| --- | --- | --- | --- | +| Echo | A small application comparing service communication patterns. | `poetry run echo` | `docs/source/overview/examples/echo.rst` | +| Off the shelf | Built-in infrastructure, application, placement, and metrics. | `poetry run off-the-shelf` | `docs/source/overview/examples/off_the_shelf.rst` | +| Sock Shop MPI | Remote emulation of the Sock Shop application using MPI communication. | `poetry run sock-shop-mpi` | `docs/source/overview/examples/sock_shop.rst` | +| Sock Shop REST | Remote emulation of the Sock Shop application using REST communication. | `poetry run sock-shop-rest` | `docs/source/overview/examples/sock_shop.rst` | +| Grid analysis | Ray Tune parameter sweeps over topologies, policies, and strategies. | `poetry run grid-analysis` | `docs/source/overview/examples/grid_analysis.rst` | +| Image prediction | Remote image-prediction services with custom metrics and degradation. | `poetry run image-prediction` | `docs/source/overview/examples/image_prediction.rst` | +| User distribution | User-distribution-driven infrastructure changes and custom metrics. | `poetry run user-distribution` | `docs/source/overview/examples/user_distribution.rst` | diff --git a/examples/echo/__init__.py b/examples/echo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/echo/application.py b/examples/echo/application.py index c697d89..b35ad6c 100755 --- a/examples/echo/application.py +++ b/examples/echo/application.py @@ -1,4 +1,4 @@ -from echo import EchoService +from .echo import EchoService from eclypse.graph import Application diff --git a/examples/echo/infrastructure.py b/examples/echo/infrastructure.py index ddfae0f..03e6ab9 100755 --- a/examples/echo/infrastructure.py +++ b/examples/echo/infrastructure.py @@ -1,4 +1,4 @@ -from update_policy import random_update +from .update_policy import random_update from eclypse.graph import Infrastructure diff --git a/examples/echo/main.py b/examples/echo/main.py index 9caecf3..2a695f3 100755 --- a/examples/echo/main.py +++ b/examples/echo/main.py @@ -1,5 +1,5 @@ -from application import echo_app as app -from infrastructure import get_infrastructure +from .application import echo_app as app +from .infrastructure import get_infrastructure from eclypse.placement.strategies import RandomStrategy from eclypse.simulation import ( @@ -8,8 +8,9 @@ ) from eclypse.utils.defaults import get_default_sim_path -if __name__ == "__main__": +def main() -> None: + """Run the Echo example.""" seed = 2 sim_config = SimulationConfig( seed=seed, @@ -28,6 +29,9 @@ ) sim.register(app, RandomStrategy(seed=seed)) - sim.start() - sim.wait() + sim.run() print(sim.report.application()) + + +if __name__ == "__main__": + main() diff --git a/examples/echo/notebook.ipynb b/examples/echo/notebook.ipynb index eba73d4..ab782bf 100644 --- a/examples/echo/notebook.ipynb +++ b/examples/echo/notebook.ipynb @@ -400,14 +400,14 @@ "\n", "strategy = RandomStrategy(seed=SEED)\n", "\n", - "simulation.register(echo_app, placement_strategy=strategy)" + "simulation.register(echo_app, strategy)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Finally we can run the simulation by starting it with the `start` method, then waiting for it to finish with the `wait` method:" + "Finally we can run the simulation with the `run` helper:" ] }, { @@ -1099,8 +1099,7 @@ } ], "source": [ - "simulation.start()\n", - "simulation.wait()" + "simulation.run()" ] }, { diff --git a/examples/grid_analysis/infrastructure.py b/examples/grid_analysis/infrastructure.py index a963a0a..ddd3fb7 100644 --- a/examples/grid_analysis/infrastructure.py +++ b/examples/grid_analysis/infrastructure.py @@ -7,15 +7,15 @@ Union, ) -from policies import ( +from .policies import ( degrade_policy, kill_policy, ) from eclypse.builders.infrastructure import ( - hierarchical, - random, - star, + get_hierarchical, + get_random, + get_star, ) from eclypse.graph import Infrastructure from eclypse.graph.assets import Concave @@ -41,20 +41,20 @@ def get_infrastructure(config) -> Infrastructure: "include_default_assets": True, } if config["topology"][0] == "star": - infr = star( + infr = get_star( infrastructure_id="star", n_clients=config["nodes"], **common_config, ) elif config["topology"][0] == "random": - infr = random( + infr = get_random( infrastructure_id="random", n=config["nodes"], p=config["topology"][1], **common_config, ) elif config["topology"][0] == "hierarchical": - infr = hierarchical( + infr = get_hierarchical( infrastructure_id="hierarchical", n=config["nodes"], **common_config, diff --git a/examples/grid_analysis/main.py b/examples/grid_analysis/main.py index ad3fb90..fcd8249 100644 --- a/examples/grid_analysis/main.py +++ b/examples/grid_analysis/main.py @@ -2,14 +2,15 @@ from time import time import ray -from applications import get_apps -from infrastructure import get_infrastructure -from metrics import get_metrics from ray import ( train, tune, ) -from strategy import EnergyMinimizationStrategy + +from .applications import get_apps +from .infrastructure import get_infrastructure +from .metrics import get_metrics +from .strategy import EnergyMinimizationStrategy from eclypse.placement.strategies import ( BestFitStrategy, @@ -48,8 +49,7 @@ def eclypse_grid(config): for app in apps: sim.register(app, get_strategy(config)) - sim.start() - sim.wait() + sim.run() print("End of simulation") @@ -121,17 +121,8 @@ def get_strategy(config): ), } -if __name__ == "__main__": - config_example = { - "max_steps": 10, - "load": 0, - "nodes": 50, - "seed": 42, - "policy": ("kill", 0.1), - "strategy": "min-energy", - "topology": ("hierarchical",), - } - +def main() -> None: + """Run the Grid Analysis Ray Tune example.""" ray.init(address="auto") start_time = time() @@ -144,3 +135,7 @@ def get_strategy(config): ) tuner.fit() print("Elapsed time: ", time() - start_time) + + +if __name__ == "__main__": + main() diff --git a/examples/image_prediction/__init__.py b/examples/image_prediction/__init__.py new file mode 100644 index 0000000..3591d14 --- /dev/null +++ b/examples/image_prediction/__init__.py @@ -0,0 +1 @@ +"""Image prediction example package.""" diff --git a/examples/image_prediction/application.py b/examples/image_prediction/application.py index e247111..93f586e 100644 --- a/examples/image_prediction/application.py +++ b/examples/image_prediction/application.py @@ -1,4 +1,4 @@ -from services import ( +from .services import ( EndService, PredictorService, TrainerService, diff --git a/examples/image_prediction/main.py b/examples/image_prediction/main.py index e682a91..fde75a6 100644 --- a/examples/image_prediction/main.py +++ b/examples/image_prediction/main.py @@ -1,27 +1,23 @@ -from application import image_app as app -from metrics import get_metrics -from services.utils import ( +from .application import image_app as app +from .metrics import get_metrics +from .services.utils import ( BASE_PATH, STEP_EVERY_MS, STEPS, ) -from update_policy import DegradePolicy +from .update_policy import DegradePolicy -from eclypse.builders.infrastructure import star +from eclypse.builders.infrastructure import get_star from eclypse.placement.strategies import RandomStrategy -from eclypse.remote.bootstrap import ( - RayOptionsFactory, - RemoteBootstrap, -) from eclypse.simulation import ( Simulation, SimulationConfig, ) -if __name__ == "__main__": +def main() -> None: + """Run the Image Prediction example.""" seed = 2 - with_gpus = RemoteBootstrap(ray_options_factory=RayOptionsFactory(num_gpus=0.1)) sim_config = SimulationConfig( seed=seed, @@ -31,11 +27,12 @@ events=get_metrics(), log_to_file=True, path=BASE_PATH, - remote=True, # use "with_gpus" instead of "True" if you have available GPUs + # Pass a RemoteBootstrap instance here to request specific Ray resources. + remote=True, ) sim = Simulation( - star( + get_star( infrastructure_id="IPInfr", n_clients=5, seed=seed, @@ -49,5 +46,8 @@ strategy = RandomStrategy(spread=True) sim.register(app, strategy) - sim.start() - sim.wait() + sim.run() + + +if __name__ == "__main__": + main() diff --git a/examples/off_the_shelf/__init__.py b/examples/off_the_shelf/__init__.py new file mode 100644 index 0000000..5602c10 --- /dev/null +++ b/examples/off_the_shelf/__init__.py @@ -0,0 +1 @@ +"""Off-the-shelf example package.""" diff --git a/examples/off_the_shelf/application.py b/examples/off_the_shelf/application.py index 9f75afb..8161ec3 100644 --- a/examples/off_the_shelf/application.py +++ b/examples/off_the_shelf/application.py @@ -3,13 +3,13 @@ from __future__ import annotations from eclypse import policies -from eclypse.builders.application import get_sock_shop +from eclypse.builders.application import get_hotel_reservation def get_application(seed: int = 7): - """Create a Sock Shop application using built-in policies only.""" - return get_sock_shop( - application_id="SockShopBuiltins", + """Create a hotel reservation application using built-in policies only.""" + return get_hotel_reservation( + application_id="HotelReservationBuiltins", include_default_assets=True, seed=seed, update_policies=[ diff --git a/examples/off_the_shelf/infrastructure.py b/examples/off_the_shelf/infrastructure.py index 7343abc..9f26b70 100644 --- a/examples/off_the_shelf/infrastructure.py +++ b/examples/off_the_shelf/infrastructure.py @@ -3,12 +3,12 @@ from __future__ import annotations from eclypse import policies -from eclypse.builders.infrastructure import hierarchical +from eclypse.builders.infrastructure import get_hierarchical def get_infrastructure(seed: int = 7): """Create a generated infrastructure with built-in update policies.""" - return hierarchical( + return get_hierarchical( n=64, infrastructure_id="BuiltinsInfrastructure", symmetric=True, diff --git a/examples/off_the_shelf/main.py b/examples/off_the_shelf/main.py index 074e9d7..a0442dd 100644 --- a/examples/off_the_shelf/main.py +++ b/examples/off_the_shelf/main.py @@ -2,8 +2,10 @@ from __future__ import annotations -from application import get_application -from infrastructure import get_infrastructure +import os + +from .application import get_application +from .infrastructure import get_infrastructure from eclypse.placement.strategies import BestFitStrategy from eclypse.simulation import ( @@ -12,10 +14,11 @@ ) from eclypse.utils.defaults import get_default_sim_path -if __name__ == "__main__": +def main() -> None: + """Run the off-the-shelf example.""" SEED = 42 - MAX_STEPS = 50 + MAX_STEPS = int(os.environ.get("ECLYPSE_EXAMPLE_MAX_STEPS", "50")) simulation = Simulation( get_infrastructure(seed=SEED), simulation_config=SimulationConfig( @@ -30,6 +33,9 @@ ) simulation.register(get_application(seed=SEED), BestFitStrategy()) - simulation.start() - simulation.wait() + simulation.run() print(simulation.report.application()) + + +if __name__ == "__main__": + main() diff --git a/examples/sock_shop/__init__.py b/examples/sock_shop/__init__.py new file mode 100644 index 0000000..3b1a0c4 --- /dev/null +++ b/examples/sock_shop/__init__.py @@ -0,0 +1 @@ +"""Sock Shop example package.""" diff --git a/examples/sock_shop/mpi/main.py b/examples/sock_shop/mpi.py similarity index 75% rename from examples/sock_shop/mpi/main.py rename to examples/sock_shop/mpi.py index 1ad00e0..ff7307e 100644 --- a/examples/sock_shop/mpi/main.py +++ b/examples/sock_shop/mpi.py @@ -1,18 +1,20 @@ -from update_policy import random_update +from .update_policy import get_update_policies from eclypse.builders.application import get_sock_shop -from eclypse.builders.infrastructure import hierarchical +from eclypse.builders.infrastructure import get_hierarchical from eclypse.placement.strategies import RandomStrategy from eclypse.simulation import Simulation from eclypse.simulation.config import SimulationConfig from eclypse.utils.defaults import get_default_sim_path -if __name__ == "__main__": + +def main() -> None: + """Run the Sock Shop MPI example.""" seed = 22 - infrastructure = hierarchical( + infrastructure = get_hierarchical( n=30, node_partitioning=[0.6, 0.1, 0.15, 0.15], - update_policies=random_update, + update_policies=get_update_policies(), include_default_assets=True, symmetric=True, seed=seed, @@ -23,8 +25,8 @@ step_every_ms=500, max_steps=100, path=get_default_sim_path() / "SockShopMPI", - remote=True, include_default_metrics=True, + remote=True, ) sim = Simulation(infrastructure, simulation_config=sim_config) @@ -32,5 +34,8 @@ app = get_sock_shop(communication_interface="mpi", include_default_assets=True) sim.register(app, RandomStrategy(seed=seed)) - sim.start() - sim.wait() + sim.run() + + +if __name__ == "__main__": + main() diff --git a/examples/sock_shop/mpi/update_policy.py b/examples/sock_shop/mpi/update_policy.py deleted file mode 100644 index 1e3e482..0000000 --- a/examples/sock_shop/mpi/update_policy.py +++ /dev/null @@ -1,31 +0,0 @@ -import random as rnd - -from eclypse.graph import AssetGraph - - -def random_update(graph: AssetGraph): - for _, resources in graph.nodes.data(): - if rnd.random() < 0.02: - resources["availability"] = 0 - elif rnd.random() < 0.5 and resources["availability"] == 0: - resources["availability"] = 1 - else: - # Randomly update resources with different ranges - resources["cpu"] = round(max(0, resources["cpu"] * rnd.uniform(0.95, 1.05))) - resources["gpu"] = round(max(0, resources["gpu"] * rnd.uniform(0.9, 1.1))) - resources["ram"] = round(max(0, resources["ram"] * rnd.uniform(0.8, 1.2))) - resources["storage"] = round( - max(0, resources["storage"] * rnd.uniform(0.9, 1.1)) - ) - resources["availability"] = min( - 1, max(0, resources["availability"] * rnd.uniform(0.995, 1.005)) - ) - - for _, _, resources in graph.edges.data(): - # Randomly update resources with different ranges - resources["latency"] = round( - max(0, resources["latency"] * rnd.uniform(0.9, 1.1)) - ) - resources["bandwidth"] = round( - max(0, resources["bandwidth"] * rnd.uniform(0.95, 1.05)) - ) diff --git a/examples/sock_shop/rest/main.py b/examples/sock_shop/rest.py similarity index 69% rename from examples/sock_shop/rest/main.py rename to examples/sock_shop/rest.py index 7f7d691..844563a 100644 --- a/examples/sock_shop/rest/main.py +++ b/examples/sock_shop/rest.py @@ -1,19 +1,20 @@ -from update_policy import random_update +from .update_policy import get_update_policies from eclypse.builders.application import get_sock_shop -from eclypse.builders.infrastructure import hierarchical +from eclypse.builders.infrastructure import get_hierarchical from eclypse.placement.strategies import RandomStrategy -from eclypse.placement.strategies.random import RandomStrategy from eclypse.simulation import Simulation from eclypse.simulation.config import SimulationConfig from eclypse.utils.defaults import get_default_sim_path -if __name__ == "__main__": + +def main() -> None: + """Run the Sock Shop REST example.""" seed = 22 - infrastructure = hierarchical( + infrastructure = get_hierarchical( n=30, node_partitioning=[0.6, 0.2, 0.1, 0.1], - update_policies=random_update, + update_policies=get_update_policies(), include_default_assets=True, symmetric=True, seed=seed, @@ -28,13 +29,13 @@ remote=True, ) - sim = Simulation( - infrastructure, - simulation_config=sim_config, - ) + sim = Simulation(infrastructure, simulation_config=sim_config) app = get_sock_shop(communication_interface="rest", include_default_assets=True) sim.register(app, RandomStrategy(seed=seed)) - sim.start() - sim.wait() + sim.run() + + +if __name__ == "__main__": + main() diff --git a/examples/sock_shop/rest/update_policy.py b/examples/sock_shop/rest/update_policy.py deleted file mode 100644 index 1e3e482..0000000 --- a/examples/sock_shop/rest/update_policy.py +++ /dev/null @@ -1,31 +0,0 @@ -import random as rnd - -from eclypse.graph import AssetGraph - - -def random_update(graph: AssetGraph): - for _, resources in graph.nodes.data(): - if rnd.random() < 0.02: - resources["availability"] = 0 - elif rnd.random() < 0.5 and resources["availability"] == 0: - resources["availability"] = 1 - else: - # Randomly update resources with different ranges - resources["cpu"] = round(max(0, resources["cpu"] * rnd.uniform(0.95, 1.05))) - resources["gpu"] = round(max(0, resources["gpu"] * rnd.uniform(0.9, 1.1))) - resources["ram"] = round(max(0, resources["ram"] * rnd.uniform(0.8, 1.2))) - resources["storage"] = round( - max(0, resources["storage"] * rnd.uniform(0.9, 1.1)) - ) - resources["availability"] = min( - 1, max(0, resources["availability"] * rnd.uniform(0.995, 1.005)) - ) - - for _, _, resources in graph.edges.data(): - # Randomly update resources with different ranges - resources["latency"] = round( - max(0, resources["latency"] * rnd.uniform(0.9, 1.1)) - ) - resources["bandwidth"] = round( - max(0, resources["bandwidth"] * rnd.uniform(0.95, 1.05)) - ) diff --git a/examples/sock_shop/update_policy.py b/examples/sock_shop/update_policy.py new file mode 100644 index 0000000..e68ce14 --- /dev/null +++ b/examples/sock_shop/update_policy.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from eclypse import policies + + +def get_update_policies(): + """Build the shared Sock Shop infrastructure update policies. + + Returns: + list[callable]: Built-in update policies for availability flapping and + multiplicative node and edge drift. + """ + return [ + policies.failure.availability_flap( + down_probability=0.02, + up_probability=0.5, + ), + policies.distribution.uniform( + node_asset_distributions={ + "cpu": (0.95, 1.05), + "gpu": (0.9, 1.1), + "ram": (0.8, 1.2), + "storage": (0.9, 1.1), + }, + edge_asset_distributions={ + "latency": (0.9, 1.1), + "bandwidth": (0.95, 1.05), + }, + ), + ] diff --git a/examples/user_distribution/__init__.py b/examples/user_distribution/__init__.py new file mode 100644 index 0000000..6e55d48 --- /dev/null +++ b/examples/user_distribution/__init__.py @@ -0,0 +1 @@ +"""User distribution simulation example.""" diff --git a/examples/user_distribution/infrastructure.py b/examples/user_distribution/infrastructure.py index 2e78ca2..ebd44c8 100644 --- a/examples/user_distribution/infrastructure.py +++ b/examples/user_distribution/infrastructure.py @@ -1,19 +1,20 @@ import networkx as nx -from metric import user_count_asset -from update_policy import ( + +from .metric import user_count_asset +from .update_policy import ( LatencyUpdatePolicy, UserDistributionPolicy, kill_policy, ) -from eclypse.builders.infrastructure import hierarchical +from eclypse.builders.infrastructure import get_hierarchical def get_infrastructure(seed: int): kill_probability = 0.1 - i = hierarchical( + i = get_hierarchical( node_assets={"user_count": user_count_asset()}, - infrastructure_id="hierarchical", + infrastructure_id="get_hierarchical", n=187, update_policies=[ kill_policy(kill_probability=kill_probability), diff --git a/examples/user_distribution/main.py b/examples/user_distribution/main.py index b546c8c..0ac044d 100644 --- a/examples/user_distribution/main.py +++ b/examples/user_distribution/main.py @@ -1,7 +1,7 @@ from time import time -from infrastructure import get_infrastructure -from metric import get_metrics +from .infrastructure import get_infrastructure +from .metric import get_metrics from eclypse.builders.application import get_sock_shop from eclypse.placement.strategies import BestFitStrategy @@ -14,23 +14,29 @@ SEED = 42 STEPS = 4167 -app = get_sock_shop(seed=SEED) -strategy = BestFitStrategy() -sim_config = SimulationConfig( - step_every_ms="auto", - seed=SEED, - max_steps=STEPS, - path=get_default_sim_path() / "user-distribution", - events=get_metrics(), - log_to_file=True, -) -infrastructure = get_infrastructure(SEED) +def main() -> None: + """Run the user distribution example.""" + app = get_sock_shop(seed=SEED) + strategy = BestFitStrategy() + + sim_config = SimulationConfig( + step_every_ms="auto", + seed=SEED, + max_steps=STEPS, + path=get_default_sim_path() / "user-distribution", + events=get_metrics(), + log_to_file=True, + ) + infrastructure = get_infrastructure(SEED) + + sim = Simulation(infrastructure, simulation_config=sim_config) + sim.register(app, strategy) + + start_time = time() + sim.run() + print("Elapsed time: ", time() - start_time) -sim = Simulation(infrastructure, simulation_config=sim_config) -sim.register(app, strategy) -start_time = time() -sim.start() -sim.wait() -print("Elapsed time: ", time() - start_time) +if __name__ == "__main__": + main() diff --git a/poetry.lock b/poetry.lock index 98e37fd..cfc305d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -35,10 +35,9 @@ files = [ name = "aiohappyeyeballs" version = "2.6.1" description = "Happy Eyeballs for asyncio" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8"}, {file = "aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558"}, @@ -48,10 +47,9 @@ files = [ name = "aiohttp" version = "3.13.5" description = "Async http client/server framework (asyncio)" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:02222e7e233295f40e011c1b00e3b0bd451f22cf853a0304c3595633ee47da4b"}, {file = "aiohttp-3.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bace460460ed20614fa6bc8cb09966c0b8517b8c58ad8046828c6078d25333b5"}, @@ -191,10 +189,9 @@ speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "a name = "aiohttp-cors" version = "0.8.1" description = "CORS support for aiohttp" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "aiohttp_cors-0.8.1-py3-none-any.whl", hash = "sha256:3180cf304c5c712d626b9162b195b1db7ddf976a2a25172b35bb2448b890a80d"}, {file = "aiohttp_cors-0.8.1.tar.gz", hash = "sha256:ccacf9cb84b64939ea15f859a146af1f662a6b1d68175754a07315e305fb1403"}, @@ -207,10 +204,9 @@ aiohttp = ">=3.9" name = "aiosignal" version = "1.4.0" description = "aiosignal: a list of registered asynchronous callbacks" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, @@ -236,10 +232,9 @@ files = [ name = "annotated-types" version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" -optional = true +optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -283,10 +278,9 @@ test = ["coverage", "mypy", "pexpect", "ruff", "wheel"] name = "attrs" version = "26.1.0" description = "Classes Without Boilerplate" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, @@ -347,69 +341,17 @@ charset-normalizer = ["charset-normalizer"] html5lib = ["html5lib"] lxml = ["lxml"] -[[package]] -name = "black" -version = "26.3.1" -description = "The uncompromising code formatter." -optional = false -python-versions = ">=3.10" -groups = ["dev"] -files = [ - {file = "black-26.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:86a8b5035fce64f5dcd1b794cf8ec4d31fe458cf6ce3986a30deb434df82a1d2"}, - {file = "black-26.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5602bdb96d52d2d0672f24f6ffe5218795736dd34807fd0fd55ccd6bf206168b"}, - {file = "black-26.3.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c54a4a82e291a1fee5137371ab488866b7c86a3305af4026bdd4dc78642e1ac"}, - {file = "black-26.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:6e131579c243c98f35bce64a7e08e87fb2d610544754675d4a0e73a070a5aa3a"}, - {file = "black-26.3.1-cp310-cp310-win_arm64.whl", hash = "sha256:5ed0ca58586c8d9a487352a96b15272b7fa55d139fc8496b519e78023a8dab0a"}, - {file = "black-26.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28ef38aee69e4b12fda8dba75e21f9b4f979b490c8ac0baa7cb505369ac9e1ff"}, - {file = "black-26.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf9bf162ed91a26f1adba8efda0b573bc6924ec1408a52cc6f82cb73ec2b142c"}, - {file = "black-26.3.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:474c27574d6d7037c1bc875a81d9be0a9a4f9ee95e62800dab3cfaadbf75acd5"}, - {file = "black-26.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:5e9d0d86df21f2e1677cc4bd090cd0e446278bcbbe49bf3659c308c3e402843e"}, - {file = "black-26.3.1-cp311-cp311-win_arm64.whl", hash = "sha256:9a5e9f45e5d5e1c5b5c29b3bd4265dcc90e8b92cf4534520896ed77f791f4da5"}, - {file = "black-26.3.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e6f89631eb88a7302d416594a32faeee9fb8fb848290da9d0a5f2903519fc1"}, - {file = "black-26.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:41cd2012d35b47d589cb8a16faf8a32ef7a336f56356babd9fcf70939ad1897f"}, - {file = "black-26.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f76ff19ec5297dd8e66eb64deda23631e642c9393ab592826fd4bdc97a4bce7"}, - {file = "black-26.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:ddb113db38838eb9f043623ba274cfaf7d51d5b0c22ecb30afe58b1bb8322983"}, - {file = "black-26.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:dfdd51fc3e64ea4f35873d1b3fb25326773d55d2329ff8449139ebaad7357efb"}, - {file = "black-26.3.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:855822d90f884905362f602880ed8b5df1b7e3ee7d0db2502d4388a954cc8c54"}, - {file = "black-26.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8a33d657f3276328ce00e4d37fe70361e1ec7614da5d7b6e78de5426cb56332f"}, - {file = "black-26.3.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f1cd08e99d2f9317292a311dfe578fd2a24b15dbce97792f9c4d752275c1fa56"}, - {file = "black-26.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:c7e72339f841b5a237ff14f7d3880ddd0fc7f98a1199e8c4327f9a4f478c1839"}, - {file = "black-26.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:afc622538b430aa4c8c853f7f63bc582b3b8030fd8c80b70fb5fa5b834e575c2"}, - {file = "black-26.3.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2d6bfaf7fd0993b420bed691f20f9492d53ce9a2bcccea4b797d34e947318a78"}, - {file = "black-26.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f89f2ab047c76a9c03f78d0d66ca519e389519902fa27e7a91117ef7611c0568"}, - {file = "black-26.3.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b07fc0dab849d24a80a29cfab8d8a19187d1c4685d8a5e6385a5ce323c1f015f"}, - {file = "black-26.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:0126ae5b7c09957da2bdbd91a9ba1207453feada9e9fe51992848658c6c8e01c"}, - {file = "black-26.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:92c0ec1f2cc149551a2b7b47efc32c866406b6891b0ee4625e95967c8f4acfb1"}, - {file = "black-26.3.1-py3-none-any.whl", hash = "sha256:2bd5aa94fc267d38bb21a70d7410a89f1a1d318841855f698746f8e7f51acd1b"}, - {file = "black-26.3.1.tar.gz", hash = "sha256:2c50f5063a9641c7eed7795014ba37b0f5fa227f3d408b968936e24bc0566b07"}, -] - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -packaging = ">=22.0" -pathspec = ">=1.0.0" -platformdirs = ">=2" -pytokens = ">=0.4.0,<0.5.0" - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.10)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2) ; sys_platform != \"win32\"", "winloop (>=0.5.0) ; sys_platform == \"win32\""] - [[package]] name = "certifi" -version = "2026.2.25" +version = "2026.4.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" -groups = ["main", "deploy", "docs"] +groups = ["deploy", "docs", "test"] files = [ - {file = "certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa"}, - {file = "certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7"}, + {file = "certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a"}, + {file = "certifi-2026.4.22.tar.gz", hash = "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580"}, ] -markers = {main = "extra == \"remote\""} [[package]] name = "cffi" @@ -417,7 +359,7 @@ version = "2.0.0" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.9" -groups = ["main", "deploy"] +groups = ["deploy", "test"] files = [ {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, @@ -504,7 +446,7 @@ files = [ {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, ] -markers = {main = "platform_python_implementation != \"PyPy\" and extra == \"remote\"", deploy = "sys_platform == \"linux\" and platform_python_implementation != \"PyPy\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} +markers = {deploy = "sys_platform == \"linux\" and platform_python_implementation != \"PyPy\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"", test = "platform_python_implementation != \"PyPy\""} [package.dependencies] pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} @@ -527,7 +469,7 @@ version = "3.4.7" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" -groups = ["main", "deploy", "dev", "docs"] +groups = ["deploy", "dev", "docs", "test"] files = [ {file = "charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d"}, {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8"}, @@ -659,20 +601,18 @@ files = [ {file = "charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d"}, {file = "charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5"}, ] -markers = {main = "extra == \"remote\""} [[package]] name = "click" -version = "8.3.2" +version = "8.3.3" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" -groups = ["main", "dev", "docs"] +groups = ["docs", "test"] files = [ - {file = "click-8.3.2-py3-none-any.whl", hash = "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d"}, - {file = "click-8.3.2.tar.gz", hash = "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5"}, + {file = "click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613"}, + {file = "click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2"}, ] -markers = {main = "extra == \"remote\""} [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -688,16 +628,15 @@ files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "(platform_system == \"Windows\" or sys_platform == \"win32\") and (extra == \"remote\" or sys_platform == \"win32\")", test = "sys_platform == \"win32\""} +markers = {main = "sys_platform == \"win32\"", test = "platform_system == \"Windows\" or sys_platform == \"win32\""} [[package]] name = "colorful" version = "0.5.8" description = "Terminal string styling done right, in Python." -optional = true +optional = false python-versions = "*" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "colorful-0.5.8-py2.py3-none-any.whl", hash = "sha256:a9381fdda3337fbaba5771991020abc69676afa102646650b759927892875992"}, {file = "colorful-0.5.8.tar.gz", hash = "sha256:bb16502b198be2f1c42ba3c52c703d5f651d826076817185f0294c1a549a7445"}, @@ -708,14 +647,14 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "commitizen" -version = "4.13.9" +version = "4.13.10" description = "Python commitizen client tool" optional = false python-versions = "<4.0,>=3.10" groups = ["dev"] files = [ - {file = "commitizen-4.13.9-py3-none-any.whl", hash = "sha256:d2af3d6a83cacec9d5200e17768942c5de6266f93d932c955986c60c4285f2db"}, - {file = "commitizen-4.13.9.tar.gz", hash = "sha256:2b4567ed50555e10920e5bd804a6a4e2c42ec70bb74f14a83f2680fe9eaf9727"}, + {file = "commitizen-4.13.10-py3-none-any.whl", hash = "sha256:95a281317990ac613501fdfe65745cec1fa4042bc5d003a72d332a74926e3039"}, + {file = "commitizen-4.13.10.tar.gz", hash = "sha256:402b5bcd466be69ba79a3f380be6ba5b55ac658c7d2a93e82fc99668a6eb2673"}, ] [package.dependencies] @@ -725,7 +664,7 @@ colorama = ">=0.4.1,<1.0" decli = ">=0.6.0,<1.0" deprecated = ">=1.2.13,<2" jinja2 = ">=2.10.3" -packaging = ">=19" +packaging = ">=26" prompt-toolkit = "!=3.0.52" pyyaml = ">=3.8" questionary = ">=2.0,<3.0" @@ -857,7 +796,7 @@ version = "46.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.8" -groups = ["main", "deploy"] +groups = ["deploy", "test"] files = [ {file = "cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4"}, {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325"}, @@ -909,7 +848,7 @@ files = [ {file = "cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4"}, {file = "cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5"}, ] -markers = {main = "extra == \"remote\"", deploy = "sys_platform == \"linux\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} +markers = {deploy = "sys_platform == \"linux\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} [package.dependencies] cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} @@ -960,12 +899,11 @@ version = "0.4.0" description = "Distribution utilities" optional = false python-versions = "*" -groups = ["main", "dev"] +groups = ["dev", "test"] files = [ {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, ] -markers = {main = "extra == \"remote\""} [[package]] name = "docformatter" @@ -1015,25 +953,23 @@ testing = ["hatch", "pre-commit", "pytest", "tox"] [[package]] name = "filelock" -version = "3.25.2" +version = "3.29.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" -groups = ["main", "dev"] +groups = ["dev", "test"] files = [ - {file = "filelock-3.25.2-py3-none-any.whl", hash = "sha256:ca8afb0da15f229774c9ad1b455ed96e85a81373065fb10446672f64444ddf70"}, - {file = "filelock-3.25.2.tar.gz", hash = "sha256:b64ece2b38f4ca29dd3e810287aa8c48182bbecd1ae6e9ae126c9b35f1382694"}, + {file = "filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258"}, + {file = "filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90"}, ] -markers = {main = "extra == \"remote\""} [[package]] name = "frozenlist" version = "1.8.0" description = "A list-like structure which implements collections.abc.MutableSequence" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, @@ -1171,10 +1107,9 @@ files = [ name = "google-api-core" version = "2.30.3" description = "Google API client core library" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8"}, {file = "google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b"}, @@ -1198,10 +1133,9 @@ grpc = ["grpcio (>=1.33.2,<2.0.0)", "grpcio (>=1.49.1,<2.0.0) ; python_version > name = "google-auth" version = "2.49.2" description = "Google Authentication Library" -optional = true +optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "google_auth-2.49.2-py3-none-any.whl", hash = "sha256:c2720924dfc82dedb962c9f52cabb2ab16714fd0a6a707e40561d217574ed6d5"}, {file = "google_auth-2.49.2.tar.gz", hash = "sha256:c1ae38500e73065dcae57355adb6278cf8b5c8e391994ae9cbadbcb9631ab409"}, @@ -1227,10 +1161,9 @@ urllib3 = ["packaging", "urllib3"] name = "googleapis-common-protos" version = "1.74.0" description = "Common protobufs used in Google APIs" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5"}, {file = "googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1"}, @@ -1246,10 +1179,9 @@ grpc = ["grpcio (>=1.44.0,<2.0.0)"] name = "grpcio" version = "1.80.0" description = "HTTP/2-based RPC framework" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "grpcio-1.80.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:886457a7768e408cdce226ad1ca67d2958917d306523a0e21e1a2fdaa75c9c9c"}, {file = "grpcio-1.80.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:7b641fc3f1dc647bfd80bd713addc68f6d145956f64677e56d9ebafc0bd72388"}, @@ -1354,14 +1286,14 @@ test = ["coverage[toml]", "pretend", "pytest", "pytest-cov"] [[package]] name = "identify" -version = "2.6.18" +version = "2.6.19" description = "File identification library for Python" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "identify-2.6.18-py2.py3-none-any.whl", hash = "sha256:8db9d3c8ea9079db92cafb0ebf97abdc09d52e97f4dcf773a2e694048b7cd737"}, - {file = "identify-2.6.18.tar.gz", hash = "sha256:873ac56a5e3fd63e7438a7ecbc4d91aca692eb3fefa4534db2b7913f3fc352fd"}, + {file = "identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a"}, + {file = "identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842"}, ] [package.extras] @@ -1369,19 +1301,18 @@ license = ["ukkonen"] [[package]] name = "idna" -version = "3.11" +version = "3.13" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.8" -groups = ["main", "deploy", "docs"] +groups = ["deploy", "docs", "test"] files = [ - {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, - {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, + {file = "idna-3.13-py3-none-any.whl", hash = "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3"}, + {file = "idna-3.13.tar.gz", hash = "sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242"}, ] -markers = {main = "extra == \"remote\""} [package.extras] -all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] [[package]] name = "imagesize" @@ -1399,14 +1330,14 @@ files = [ name = "importlib-metadata" version = "8.7.1" description = "Read metadata from Python packages" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main", "deploy"] +groups = ["deploy", "test"] files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, ] -markers = {main = "extra == \"remote\"", deploy = "python_version == \"3.11\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} +markers = {deploy = "python_version == \"3.11\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} [package.dependencies] zipp = ">=3.20" @@ -1554,10 +1485,9 @@ i18n = ["Babel (>=2.7)"] name = "jsonschema" version = "4.26.0" description = "An implementation of JSON Schema validation for Python" -optional = true +optional = false python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce"}, {file = "jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326"}, @@ -1577,10 +1507,9 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "jsonschema-specifications" version = "2025.9.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe"}, {file = "jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d"}, @@ -1912,10 +1841,9 @@ files = [ name = "msgpack" version = "1.1.2" description = "MessagePack serializer" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "msgpack-1.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2"}, {file = "msgpack-1.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87"}, @@ -1985,10 +1913,9 @@ files = [ name = "multidict" version = "6.7.1" description = "multidict implementation" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, @@ -2140,63 +2067,63 @@ files = [ [[package]] name = "mypy" -version = "1.20.0" +version = "1.20.2" description = "Optional static typing for Python" optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "mypy-1.20.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d99f515f95fd03a90875fdb2cca12ff074aa04490db4d190905851bdf8a549a8"}, - {file = "mypy-1.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bd0212976dc57a5bfeede7c219e7cd66568a32c05c9129686dd487c059c1b88a"}, - {file = "mypy-1.20.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f8426d4d75d68714abc17a4292d922f6ba2cfb984b72c2278c437f6dae797865"}, - {file = "mypy-1.20.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02cca0761c75b42a20a2757ae58713276605eb29a08dd8a6e092aa347c4115ca"}, - {file = "mypy-1.20.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b3a49064504be59e59da664c5e149edc1f26c67c4f8e8456f6ba6aba55033018"}, - {file = "mypy-1.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:ebea00201737ad4391142808ed16e875add5c17f676e0912b387739f84991e13"}, - {file = "mypy-1.20.0-cp310-cp310-win_arm64.whl", hash = "sha256:e80cf77847d0d3e6e3111b7b25db32a7f8762fd4b9a3a72ce53fe16a2863b281"}, - {file = "mypy-1.20.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4525e7010b1b38334516181c5b81e16180b8e149e6684cee5a727c78186b4e3b"}, - {file = "mypy-1.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a17c5d0bdcca61ce24a35beb828a2d0d323d3fcf387d7512206888c900193367"}, - {file = "mypy-1.20.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f75ff57defcd0f1d6e006d721ccdec6c88d4f6a7816eb92f1c4890d979d9ee62"}, - {file = "mypy-1.20.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b503ab55a836136b619b5fc21c8803d810c5b87551af8600b72eecafb0059cb0"}, - {file = "mypy-1.20.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1973868d2adbb4584a3835780b27436f06d1dc606af5be09f187aaa25be1070f"}, - {file = "mypy-1.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:2fcedb16d456106e545b2bfd7ef9d24e70b38ec252d2a629823a4d07ebcdb69e"}, - {file = "mypy-1.20.0-cp311-cp311-win_arm64.whl", hash = "sha256:379edf079ce44ac8d2805bcf9b3dd7340d4f97aad3a5e0ebabbf9d125b84b442"}, - {file = "mypy-1.20.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:002b613ae19f4ac7d18b7e168ffe1cb9013b37c57f7411984abbd3b817b0a214"}, - {file = "mypy-1.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9336b5e6712f4adaf5afc3203a99a40b379049104349d747eb3e5a3aa23ac2e"}, - {file = "mypy-1.20.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f13b3e41bce9d257eded794c0f12878af3129d80aacd8a3ee0dee51f3a978651"}, - {file = "mypy-1.20.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9804c3ad27f78e54e58b32e7cb532d128b43dbfb9f3f9f06262b821a0f6bd3f5"}, - {file = "mypy-1.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:697f102c5c1d526bdd761a69f17c6070f9892eebcb94b1a5963d679288c09e78"}, - {file = "mypy-1.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ecd63f75fdd30327e4ad8b5704bd6d91fc6c1b2e029f8ee14705e1207212489"}, - {file = "mypy-1.20.0-cp312-cp312-win_arm64.whl", hash = "sha256:f194db59657c58593a3c47c6dfd7bad4ef4ac12dbc94d01b3a95521f78177e33"}, - {file = "mypy-1.20.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b20c8b0fd5877abdf402e79a3af987053de07e6fb208c18df6659f708b535134"}, - {file = "mypy-1.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:367e5c993ba34d5054d11937d0485ad6dfc60ba760fa326c01090fc256adf15c"}, - {file = "mypy-1.20.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f799d9db89fc00446f03281f84a221e50018fc40113a3ba9864b132895619ebe"}, - {file = "mypy-1.20.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555658c611099455b2da507582ea20d2043dfdfe7f5ad0add472b1c6238b433f"}, - {file = "mypy-1.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:efe8d70949c3023698c3fca1e94527e7e790a361ab8116f90d11221421cd8726"}, - {file = "mypy-1.20.0-cp313-cp313-win_amd64.whl", hash = "sha256:f49590891d2c2f8a9de15614e32e459a794bcba84693c2394291a2038bbaaa69"}, - {file = "mypy-1.20.0-cp313-cp313-win_arm64.whl", hash = "sha256:76a70bf840495729be47510856b978f1b0ec7d08f257ca38c9d932720bf6b43e"}, - {file = "mypy-1.20.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0f42dfaab7ec1baff3b383ad7af562ab0de573c5f6edb44b2dab016082b89948"}, - {file = "mypy-1.20.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:31b5dbb55293c1bd27c0fc813a0d2bb5ceef9d65ac5afa2e58f829dab7921fd5"}, - {file = "mypy-1.20.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49d11c6f573a5a08f77fad13faff2139f6d0730ebed2cfa9b3d2702671dd7188"}, - {file = "mypy-1.20.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d3243c406773185144527f83be0e0aefc7bf4601b0b2b956665608bf7c98a83"}, - {file = "mypy-1.20.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a79c1eba7ac4209f2d850f0edd0a2f8bba88cbfdfefe6fb76a19e9d4fe5e71a2"}, - {file = "mypy-1.20.0-cp314-cp314-win_amd64.whl", hash = "sha256:00e047c74d3ec6e71a2eb88e9ea551a2edb90c21f993aefa9e0d2a898e0bb732"}, - {file = "mypy-1.20.0-cp314-cp314-win_arm64.whl", hash = "sha256:931a7630bba591593dcf6e97224a21ff80fb357e7982628d25e3c618e7f598ef"}, - {file = "mypy-1.20.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:26c8b52627b6552f47ff11adb4e1509605f094e29815323e487fc0053ebe93d1"}, - {file = "mypy-1.20.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:39362cdb4ba5f916e7976fccecaab1ba3a83e35f60fa68b64e9a70e221bb2436"}, - {file = "mypy-1.20.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34506397dbf40c15dc567635d18a21d33827e9ab29014fb83d292a8f4f8953b6"}, - {file = "mypy-1.20.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:555493c44a4f5a1b58d611a43333e71a9981c6dbe26270377b6f8174126a0526"}, - {file = "mypy-1.20.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2721f0ce49cb74a38f00c50da67cb7d36317b5eda38877a49614dc018e91c787"}, - {file = "mypy-1.20.0-cp314-cp314t-win_amd64.whl", hash = "sha256:47781555a7aa5fedcc2d16bcd72e0dc83eb272c10dd657f9fb3f9cc08e2e6abb"}, - {file = "mypy-1.20.0-cp314-cp314t-win_arm64.whl", hash = "sha256:c70380fe5d64010f79fb863b9081c7004dd65225d2277333c219d93a10dad4dd"}, - {file = "mypy-1.20.0-py3-none-any.whl", hash = "sha256:a6e0641147cbfa7e4e94efdb95c2dab1aff8cfc159ded13e07f308ddccc8c48e"}, - {file = "mypy-1.20.0.tar.gz", hash = "sha256:eb96c84efcc33f0b5e0e04beacf00129dd963b67226b01c00b9dfc8affb464c3"}, + {file = "mypy-1.20.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cf5a4db6dca263010e2c7bff081c89383c72d187ba2cf4c44759aac970e2f0c4"}, + {file = "mypy-1.20.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b0e817b518bff7facd7f85ea05b643ad8bdcce684cf29784987b0a7c8e1f997"}, + {file = "mypy-1.20.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97d7b9a485b40f8ca425460e89bf1da2814625b2da627c0dcc6aa46c92631d14"}, + {file = "mypy-1.20.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e1c12f6d2db3d78b909b5f77513c11eb7f2dd2782b96a3ab6dffc7d44575c99"}, + {file = "mypy-1.20.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89dce27e142d25ffbc154c1819383b69f2e9234dc4ed4766f42e0e8cb264ab5c"}, + {file = "mypy-1.20.2-cp310-cp310-win_amd64.whl", hash = "sha256:f376e37f9bf2a946872fc5fd1199c99310748e3c26c7a26683f13f8bdb756cbd"}, + {file = "mypy-1.20.2-cp310-cp310-win_arm64.whl", hash = "sha256:6e2b469efd811707bc530fd1effef0f5d6eebcb7fe376affae69025da4b979a2"}, + {file = "mypy-1.20.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4077797a273e56e8843d001e9dfe4ba10e33323d6ade647ff260e5cd97d9758c"}, + {file = "mypy-1.20.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cdecf62abcc4292500d7858aeae87a1f8f1150f4c4dd08fb0b336ee79b2a6df3"}, + {file = "mypy-1.20.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c566c3a88b6ece59b3d70f65bedef17304f48eb52ff040a6a18214e1917b3254"}, + {file = "mypy-1.20.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0deb80d062b2479f2c87ae568f89845afc71d11bc41b04179e58165fd9f31e98"}, + {file = "mypy-1.20.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bba9ad231e92a3e424b3e56b65aa17704993425bba97e302c832f9466bb85bac"}, + {file = "mypy-1.20.2-cp311-cp311-win_amd64.whl", hash = "sha256:baf593f2765fa3a6b1ef95807dbaa3d25b594f6a52adcc506a6b9cb115e1be67"}, + {file = "mypy-1.20.2-cp311-cp311-win_arm64.whl", hash = "sha256:20175a1c0f49863946ec20b7f63255768058ac4f07d2b9ded6a6b46cfb5a9100"}, + {file = "mypy-1.20.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4dbfcf869f6b0517f70cf0030ba6ea1d6645e132337a7d5204a18d8d5636c02b"}, + {file = "mypy-1.20.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b6481b228d072315b053210b01ac320e1be243dc17f9e5887ef167f23f5fae4"}, + {file = "mypy-1.20.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34397cdced6b90b836e38182076049fdb41424322e0b0728c946b0939ebdf9f6"}, + {file = "mypy-1.20.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5da6976f20cae27059ea8d0c86e7cef3de720e04c4bb9ee18e3690fdb792066"}, + {file = "mypy-1.20.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:56908d7e08318d39f85b1f0c6cfd47b0cac1a130da677630dac0de3e0623e102"}, + {file = "mypy-1.20.2-cp312-cp312-win_amd64.whl", hash = "sha256:d52ad8d78522da1d308789df651ee5379088e77c76cb1994858d40a426b343b9"}, + {file = "mypy-1.20.2-cp312-cp312-win_arm64.whl", hash = "sha256:785b08db19c9f214dc37d65f7c165d19a30fcecb48abfa30f31b01b5acaabb58"}, + {file = "mypy-1.20.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:edfbfca868cdd6bd8d974a60f8a3682f5565d3f5c99b327640cedd24c4264026"}, + {file = "mypy-1.20.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e2877a02380adfcdbc69071a0f74d6e9dbbf593c0dc9d174e1f223ffd5281943"}, + {file = "mypy-1.20.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7488448de6007cd5177c6cea0517ac33b4c0f5ee9b5e9f2be51ce75511a85517"}, + {file = "mypy-1.20.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb9c2fa06887e21d6a3a868762acb82aec34e2c6fd0174064f27c93ede68ad15"}, + {file = "mypy-1.20.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d56a78b646f2e3daa865bc70cd5ec5a46c50045801ca8ff17a0c43abc97e3ee"}, + {file = "mypy-1.20.2-cp313-cp313-win_amd64.whl", hash = "sha256:2a4102b03bb7481d9a91a6da8d174740c9c8c4401024684b9ca3b7cc5e49852f"}, + {file = "mypy-1.20.2-cp313-cp313-win_arm64.whl", hash = "sha256:a95a9248b0c6fd933a442c03c3b113c3b61320086b88e2c444676d3fd1ca3330"}, + {file = "mypy-1.20.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:419413398fe250aae057fd2fe50166b61077083c9b82754c341cf4fd73038f30"}, + {file = "mypy-1.20.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e73c07f23009962885c197ccb9b41356a30cc0e5a1d0c2ea8fd8fb1362d7f924"}, + {file = "mypy-1.20.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c64e5973df366b747646fc98da921f9d6eba9716d57d1db94a83c026a08e0fb"}, + {file = "mypy-1.20.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a65aa591af023864fd08a97da9974e919452cfe19cb146c8a5dc692626445dc"}, + {file = "mypy-1.20.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fef51b01e638974a6e69885687e9bd40c8d1e09a6cd291cca0619625cf1f558"}, + {file = "mypy-1.20.2-cp314-cp314-win_amd64.whl", hash = "sha256:913485a03f1bcf5d279409a9d2b9ed565c151f61c09f29991e5faa14033da4c8"}, + {file = "mypy-1.20.2-cp314-cp314-win_arm64.whl", hash = "sha256:c3bae4f855d965b5453784300c12ffc63a548304ac7f99e55d4dc7c898673aa3"}, + {file = "mypy-1.20.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2de3dcea53babc1c3237a19002bc3d228ce1833278f093b8d619e06e7cc79609"}, + {file = "mypy-1.20.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:52b176444e2e5054dfcbcb8c75b0b719865c96247b37407184bbfca5c353f2c2"}, + {file = "mypy-1.20.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:688c3312e5dadb573a2c69c82af3a298d43ecf9e6d264e0f95df960b5f6ac19c"}, + {file = "mypy-1.20.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29752dbbf8cc53f89f6ac096d363314333045c257c9c75cbd189ca2de0455744"}, + {file = "mypy-1.20.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:803203d2b6ea644982c644895c2f78b28d0e208bba7b27d9b921e0ec5eb207c6"}, + {file = "mypy-1.20.2-cp314-cp314t-win_amd64.whl", hash = "sha256:9bcb8aa397ff0093c824182fd76a935a9ba7ad097fcbef80ae89bf6c1731d8ec"}, + {file = "mypy-1.20.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e061b58443f1736f8a37c48978d7ab581636d6ab03e3d4f99e3fa90463bb9382"}, + {file = "mypy-1.20.2-py3-none-any.whl", hash = "sha256:a94c5a76ab46c5e6257c7972b6c8cff0574201ca7dc05647e33e795d78680563"}, + {file = "mypy-1.20.2.tar.gz", hash = "sha256:e8222c26daaafd9e8626dec58ae36029f82585890589576f769a650dd20fd665"}, ] [package.dependencies] librt = {version = ">=0.8.0", markers = "platform_python_implementation != \"PyPy\""} mypy_extensions = ">=1.0.0" pathspec = ">=1.0.0" -typing_extensions = ">=4.6.0" +typing_extensions = {version = ">=4.6.0", markers = "python_version < \"3.15\""} [package.extras] dmypy = ["psutil (>=4.0)"] @@ -2398,16 +2325,14 @@ files = [ {file = "numpy-2.4.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f2cf083b324a467e1ab358c105f6cad5ea950f50524668a80c486ff1db24e119"}, {file = "numpy-2.4.4.tar.gz", hash = "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0"}, ] -markers = {main = "extra == \"tboard\""} [[package]] name = "opencensus" version = "0.11.4" description = "A stats collection and distributed tracing framework" -optional = true +optional = false python-versions = "*" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "opencensus-0.11.4-py2.py3-none-any.whl", hash = "sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864"}, {file = "opencensus-0.11.4.tar.gz", hash = "sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2"}, @@ -2422,10 +2347,9 @@ six = ">=1.16,<2.0" name = "opencensus-context" version = "0.1.3" description = "OpenCensus Runtime Context" -optional = true +optional = false python-versions = "*" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "opencensus-context-0.1.3.tar.gz", hash = "sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c"}, {file = "opencensus_context-0.1.3-py2.py3-none-any.whl", hash = "sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039"}, @@ -2433,15 +2357,14 @@ files = [ [[package]] name = "opentelemetry-api" -version = "1.41.0" +version = "1.41.1" description = "OpenTelemetry Python API" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ - {file = "opentelemetry_api-1.41.0-py3-none-any.whl", hash = "sha256:0e77c806e6a89c9e4f8d372034622f3e1418a11bdbe1c80a50b3d3397ad0fa4f"}, - {file = "opentelemetry_api-1.41.0.tar.gz", hash = "sha256:9421d911326ec12dee8bc933f7839090cad7a3f13fcfb0f9e82f8174dc003c09"}, + {file = "opentelemetry_api-1.41.1-py3-none-any.whl", hash = "sha256:a22df900e75c76dc08440710e51f52f1aa6b451b429298896023e60db5b3139f"}, + {file = "opentelemetry_api-1.41.1.tar.gz", hash = "sha256:0ad1814d73b875f84494387dae86ce0b12c68556331ce6ce8fe789197c949621"}, ] [package.dependencies] @@ -2450,33 +2373,31 @@ typing-extensions = ">=4.5.0" [[package]] name = "opentelemetry-exporter-prometheus" -version = "0.62b0" +version = "0.62b1" description = "Prometheus Metric Exporter for OpenTelemetry" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ - {file = "opentelemetry_exporter_prometheus-0.62b0-py3-none-any.whl", hash = "sha256:cd7e8acae3be5f425ffa2e0864eea474fa7a40706f786de7a2d23846573d8f75"}, - {file = "opentelemetry_exporter_prometheus-0.62b0.tar.gz", hash = "sha256:4d1106566a9b3e8dff028e69e9f2dc90723e6b431c900ff8c72982fcf11dbae5"}, + {file = "opentelemetry_exporter_prometheus-0.62b1-py3-none-any.whl", hash = "sha256:7a0b8a6402e107e1f93e38f074a668797e1103936b189561959531a67ffeba55"}, + {file = "opentelemetry_exporter_prometheus-0.62b1.tar.gz", hash = "sha256:7ecbac9aa76e7abb44082ab0ff2983e0a573e4091c4653f7db483b02bae03506"}, ] [package.dependencies] opentelemetry-api = ">=1.12,<2.0" -opentelemetry-sdk = ">=1.41.0,<1.42.0" +opentelemetry-sdk = ">=1.41.1,<1.42.0" prometheus-client = ">=0.5.0,<1.0.0" [[package]] name = "opentelemetry-proto" -version = "1.41.0" +version = "1.41.1" description = "OpenTelemetry Python Proto" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ - {file = "opentelemetry_proto-1.41.0-py3-none-any.whl", hash = "sha256:b970ab537309f9eed296be482c3e7cca05d8aca8165346e929f658dbe153b247"}, - {file = "opentelemetry_proto-1.41.0.tar.gz", hash = "sha256:95d2e576f9fb1800473a3e4cfcca054295d06bdb869fda4dc9f4f779dc68f7b6"}, + {file = "opentelemetry_proto-1.41.1-py3-none-any.whl", hash = "sha256:0496713b804d127a4147e32849fbaf5683fac8ee98550e8e7679cd706c289720"}, + {file = "opentelemetry_proto-1.41.1.tar.gz", hash = "sha256:4b9d2eb631237ea43b80e16c073af438554e32bc7e9e3f8ca4a9582f900020e5"}, ] [package.dependencies] @@ -2484,20 +2405,19 @@ protobuf = ">=5.0,<7.0" [[package]] name = "opentelemetry-sdk" -version = "1.41.0" +version = "1.41.1" description = "OpenTelemetry Python SDK" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ - {file = "opentelemetry_sdk-1.41.0-py3-none-any.whl", hash = "sha256:a596f5687964a3e0d7f8edfdcf5b79cbca9c93c7025ebf5fb00f398a9443b0bd"}, - {file = "opentelemetry_sdk-1.41.0.tar.gz", hash = "sha256:7bddf3961131b318fc2d158947971a8e37e38b1cd23470cfb72b624e7cc108bd"}, + {file = "opentelemetry_sdk-1.41.1-py3-none-any.whl", hash = "sha256:edee379c126c1bce952b0c812b48fe8ff35b30df0eecf17e98afa4d598b7d85d"}, + {file = "opentelemetry_sdk-1.41.1.tar.gz", hash = "sha256:724b615e1215b5aeacda0abb8a6a8922c9a1853068948bd0bd225a56d0c792e6"}, ] [package.dependencies] -opentelemetry-api = "1.41.0" -opentelemetry-semantic-conventions = "0.62b0" +opentelemetry-api = "1.41.1" +opentelemetry-semantic-conventions = "0.62b1" typing-extensions = ">=4.5.0" [package.extras] @@ -2505,33 +2425,31 @@ file-configuration = ["jsonschema (>=4.0)", "pyyaml (>=6.0)"] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.62b0" +version = "0.62b1" description = "OpenTelemetry Semantic Conventions" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ - {file = "opentelemetry_semantic_conventions-0.62b0-py3-none-any.whl", hash = "sha256:0ddac1ce59eaf1a827d9987ab60d9315fb27aea23304144242d1fcad9e16b489"}, - {file = "opentelemetry_semantic_conventions-0.62b0.tar.gz", hash = "sha256:cbfb3c8fc259575cf68a6e1b94083cc35adc4a6b06e8cf431efa0d62606c0097"}, + {file = "opentelemetry_semantic_conventions-0.62b1-py3-none-any.whl", hash = "sha256:cf506938103d331fbb78eded0d9788095f7fd59016f2bda813c3324e5a74a93c"}, + {file = "opentelemetry_semantic_conventions-0.62b1.tar.gz", hash = "sha256:c5cc6e04a7f8c7cdd30be2ed81499fa4e75bfbd52c9cb70d40af1f9cd3619802"}, ] [package.dependencies] -opentelemetry-api = "1.41.0" +opentelemetry-api = "1.41.1" typing-extensions = ">=4.5.0" [[package]] name = "packaging" -version = "26.0" +version = "26.1" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" -groups = ["main", "deploy", "dev", "docs", "test"] +groups = ["deploy", "dev", "docs", "test"] files = [ - {file = "packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529"}, - {file = "packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4"}, + {file = "packaging-26.1-py3-none-any.whl", hash = "sha256:5d9c0669c6285e491e0ced2eee587eaf67b670d94a19e94e3984a481aba6802f"}, + {file = "packaging-26.1.tar.gz", hash = "sha256:f042152b681c4bfac5cae2742a55e103d27ab2ec0f3d88037136b6bfe7c9c5de"}, ] -markers = {main = "extra == \"remote\" or extra == \"tboard\""} [[package]] name = "pandas" @@ -2539,7 +2457,7 @@ version = "3.0.2" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.11" -groups = ["test"] +groups = ["main"] files = [ {file = "pandas-3.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a727a73cbdba2f7458dc82449e2315899d5140b449015d822f515749a46cbbe0"}, {file = "pandas-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dbbd4aa20ca51e63b53bbde6a0fa4254b1aaabb74d2f542df7a7959feb1d760c"}, @@ -2624,21 +2542,20 @@ xml = ["lxml (>=5.3.0)"] [[package]] name = "pathspec" -version = "1.0.4" +version = "1.1.0" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723"}, - {file = "pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645"}, + {file = "pathspec-1.1.0-py3-none-any.whl", hash = "sha256:574b128f7456bd899045ccd142dd446af7e6cfd0072d63ad73fbc55fbb4aaa42"}, + {file = "pathspec-1.1.0.tar.gz", hash = "sha256:f5d7c555da02fd8dde3e4a2354b6aba817a89112fa8f333f7917a2a4834dd080"}, ] [package.extras] hyperscan = ["hyperscan (>=0.7)"] optional = ["typing-extensions (>=4)"] re2 = ["google-re2 (>=1.1)"] -tests = ["pytest (>=9)", "typing-extensions (>=4.15)"] [[package]] name = "platformdirs" @@ -2646,12 +2563,11 @@ version = "4.9.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" -groups = ["main", "dev"] +groups = ["dev", "test"] files = [ {file = "platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917"}, {file = "platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a"}, ] -markers = {main = "extra == \"remote\""} [[package]] name = "pluggy" @@ -2671,18 +2587,18 @@ testing = ["coverage", "pytest", "pytest-benchmark"] [[package]] name = "polars" -version = "1.39.3" +version = "1.40.1" description = "Blazingly fast DataFrame library" optional = false python-versions = ">=3.10" groups = ["test"] files = [ - {file = "polars-1.39.3-py3-none-any.whl", hash = "sha256:c2b955ccc0a08a2bc9259785decf3d5c007b489b523bf2390cf21cec2bb82a56"}, - {file = "polars-1.39.3.tar.gz", hash = "sha256:2e016c7f3e8d14fa777ef86fe0477cec6c67023a20ba4c94d6e8431eefe4a63c"}, + {file = "polars-1.40.1-py3-none-any.whl", hash = "sha256:c0f861219d1319cdea45c4ce4d30355a47176b8f98dcedf95ea8269f131b8abd"}, + {file = "polars-1.40.1.tar.gz", hash = "sha256:ab2694134b137596b5a59bfd7b4c54ebbc9b59f9403127f18e32d363777552e8"}, ] [package.dependencies] -polars-runtime-32 = "1.39.3" +polars-runtime-32 = "1.40.1" [package.extras] adbc = ["adbc-driver-manager[dbapi]", "adbc-driver-sqlite[dbapi]"] @@ -2705,8 +2621,8 @@ plot = ["altair (>=5.4.0)"] polars-cloud = ["polars_cloud (>=0.4.0)"] pyarrow = ["pyarrow (>=7.0.0)"] pydantic = ["pydantic"] -rt64 = ["polars-runtime-64 (==1.39.3)"] -rtcompat = ["polars-runtime-compat (==1.39.3)"] +rt64 = ["polars-runtime-64 (==1.40.1)"] +rtcompat = ["polars-runtime-compat (==1.40.1)"] sqlalchemy = ["polars[pandas]", "sqlalchemy"] style = ["great-tables (>=0.8.0)"] timezone = ["tzdata ; platform_system == \"Windows\""] @@ -2715,33 +2631,33 @@ xlsxwriter = ["xlsxwriter"] [[package]] name = "polars-runtime-32" -version = "1.39.3" +version = "1.40.1" description = "Blazingly fast DataFrame library" optional = false python-versions = ">=3.10" groups = ["test"] files = [ - {file = "polars_runtime_32-1.39.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:425c0b220b573fa097b4042edff73114cc6d23432a21dfd2dc41adf329d7d2e9"}, - {file = "polars_runtime_32-1.39.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ef5884711e3c617d7dc93519a7d038e242f5741cfe5fe9afd32d58845d86c562"}, - {file = "polars_runtime_32-1.39.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06b47f535eb1f97a9a1e5b0053ef50db3a4276e241178e37bbb1a38b1fa53b14"}, - {file = "polars_runtime_32-1.39.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bc9e13dc1d2e828331f2fe8ccbc9757554dc4933a8d3e85e906b988178f95ed"}, - {file = "polars_runtime_32-1.39.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:363d49e3a3e638fc943e2b9887940300a7d06789930855a178a4727949259dc2"}, - {file = "polars_runtime_32-1.39.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7c206bdcc7bc62ea038d6adea8e44b02f0e675e0191a54c810703b4895208ea4"}, - {file = "polars_runtime_32-1.39.3-cp310-abi3-win_amd64.whl", hash = "sha256:d66ca522517554a883446957539c40dc7b75eb0c2220357fb28bc8940d305339"}, - {file = "polars_runtime_32-1.39.3-cp310-abi3-win_arm64.whl", hash = "sha256:f49f51461de63f13e5dd4eb080421c8f23f856945f3f8bd5b2b1f59da52c2860"}, - {file = "polars_runtime_32-1.39.3.tar.gz", hash = "sha256:c728e4f469cafab501947585f36311b8fb222d3e934c6209e83791e0df20b29d"}, + {file = "polars_runtime_32-1.40.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b748ef652270cc49e9e69f99a035e0eb4d5f856d42bcd6ac4d9d80a40142aa1e"}, + {file = "polars_runtime_32-1.40.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d249b3743e05986060cec0a7aaa542d020df6c6b876e556023a310efd581f9be"}, + {file = "polars_runtime_32-1.40.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5987b30e7aa1059d069498496e8dda35afd592b0ac3d46ed87e3ff8df1ad652c"}, + {file = "polars_runtime_32-1.40.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d7f42a8b3f16fc66002cc0f6516f7dd7653396886ae0ed362ab95c0b3408b59"}, + {file = "polars_runtime_32-1.40.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e5f7becc237a7ec9d9a10878dc8e54b73bbf4e2d94a2991c37d7a0b38590d8f9"}, + {file = "polars_runtime_32-1.40.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:992d14cf191dde043d36fbdbc98a65e43fbc7e9a5024cecd45f838ac4988c1ee"}, + {file = "polars_runtime_32-1.40.1-cp310-abi3-win_amd64.whl", hash = "sha256:f78bb2abd00101cbb23cc0cb068f7e36e081057a15d2ec2dde3dda280709f030"}, + {file = "polars_runtime_32-1.40.1-cp310-abi3-win_arm64.whl", hash = "sha256:b5cbfaf6b085b420b4bfcbe24e8f665076d1cccfdb80c0484c02a023ce205537"}, + {file = "polars_runtime_32-1.40.1.tar.gz", hash = "sha256:37f3065615d1bf90d03b5326222df4c5c1f8a5d33e50470aa588e3465e6eb814"}, ] [[package]] name = "pre-commit" -version = "4.5.1" +version = "4.6.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77"}, - {file = "pre_commit-4.5.1.tar.gz", hash = "sha256:eb545fcff725875197837263e977ea257a402056661f09dae08e4b149b030a61"}, + {file = "pre_commit-4.6.0-py2.py3-none-any.whl", hash = "sha256:e2cf246f7299edcabcf15f9b0571fdce06058527f0a06535068a86d38089f29b"}, + {file = "pre_commit-4.6.0.tar.gz", hash = "sha256:718d2208cef53fdc38206e40524a6d4d9576d103eb16f0fec11c875e7716e9d9"}, ] [package.dependencies] @@ -2755,10 +2671,9 @@ virtualenv = ">=20.10.0" name = "prometheus-client" version = "0.25.0" description = "Python client for the Prometheus monitoring system." -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1"}, {file = "prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28"}, @@ -2788,10 +2703,9 @@ wcwidth = "*" name = "propcache" version = "0.4.1" description = "Accelerated property cache" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "propcache-0.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db"}, {file = "propcache-0.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8"}, @@ -2921,10 +2835,9 @@ files = [ name = "proto-plus" version = "1.27.2" description = "Beautiful, Pythonic protocol buffers" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "proto_plus-1.27.2-py3-none-any.whl", hash = "sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718"}, {file = "proto_plus-1.27.2.tar.gz", hash = "sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24"}, @@ -2942,7 +2855,7 @@ version = "6.33.6" description = "" optional = false python-versions = ">=3.9" -groups = ["main", "test"] +groups = ["test"] files = [ {file = "protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3"}, {file = "protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326"}, @@ -2955,25 +2868,23 @@ files = [ {file = "protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901"}, {file = "protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135"}, ] -markers = {main = "extra == \"remote\" or extra == \"tboard\""} [[package]] name = "py-spy" -version = "0.4.1" +version = "0.4.2" description = "" -optional = true +optional = false python-versions = "*" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ - {file = "py_spy-0.4.1-py2.py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:809094208c6256c8f4ccadd31e9a513fe2429253f48e20066879239ba12cd8cc"}, - {file = "py_spy-0.4.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:1fb8bf71ab8df95a95cc387deed6552934c50feef2cf6456bc06692a5508fd0c"}, - {file = "py_spy-0.4.1-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee776b9d512a011d1ad3907ed53ae32ce2f3d9ff3e1782236554e22103b5c084"}, - {file = "py_spy-0.4.1-py2.py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:532d3525538254d1859b49de1fbe9744df6b8865657c9f0e444bf36ce3f19226"}, - {file = "py_spy-0.4.1-py2.py3-none-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4972c21890b6814017e39ac233c22572c4a61fd874524ebc5ccab0f2237aee0a"}, - {file = "py_spy-0.4.1-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6a80ec05eb8a6883863a367c6a4d4f2d57de68466f7956b6367d4edd5c61bb29"}, - {file = "py_spy-0.4.1-py2.py3-none-win_amd64.whl", hash = "sha256:d92e522bd40e9bf7d87c204033ce5bb5c828fca45fa28d970f58d71128069fdc"}, - {file = "py_spy-0.4.1.tar.gz", hash = "sha256:e53aa53daa2e47c2eef97dd2455b47bb3a7e7f962796a86cc3e7dbde8e6f4db4"}, + {file = "py_spy-0.4.2-py2.py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:1ccf688393105111684435f035bc14ec3f22117dd2b85b2414612cf27a22755a"}, + {file = "py_spy-0.4.2-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:a0e6f6810ccf0fc5e64e85e0182a5b626c4496eec01b14fb8755154b363a4831"}, + {file = "py_spy-0.4.2-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:142887e984a4e541071c99a4401ff8c3770f255d329dbd0f64e8c1dd51882cce"}, + {file = "py_spy-0.4.2-py2.py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f1c6d9b0e2379ead5bf792df43f4cf36153aa79e6dda4fb8ac7740cf8017110"}, + {file = "py_spy-0.4.2-py2.py3-none-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:24720573f95230653b457671a1dcc3c5a381fcf4e92677761e328a430ad251b2"}, + {file = "py_spy-0.4.2-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:aeb0323409199c785f730645e9f4bb7a7b9ca2c481f2c331a55642b5d13fa52f"}, + {file = "py_spy-0.4.2-py2.py3-none-win_amd64.whl", hash = "sha256:8b06a353c177677e4e1701b288d8c58e2f8d4208ee81a8048d9f72ba800918f8"}, + {file = "py_spy-0.4.2.tar.gz", hash = "sha256:90e600b27bb6bb40479637baca5a5b4bc2ba3395c93d889e672315d93042c4ae"}, ] [package.extras] @@ -2983,10 +2894,9 @@ test = ["numpy"] name = "pyasn1" version = "0.6.3" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" -optional = true +optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde"}, {file = "pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf"}, @@ -2996,10 +2906,9 @@ files = [ name = "pyasn1-modules" version = "0.4.2" description = "A collection of ASN.1-based protocols modules" -optional = true +optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, @@ -3014,29 +2923,28 @@ version = "3.0" description = "C parser in Python" optional = false python-versions = ">=3.10" -groups = ["main", "deploy"] +groups = ["deploy", "test"] files = [ {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, ] -markers = {main = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\" and extra == \"remote\"", deploy = "sys_platform == \"linux\" and platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} +markers = {deploy = "sys_platform == \"linux\" and platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"", test = "implementation_name != \"PyPy\" and platform_python_implementation != \"PyPy\""} [[package]] name = "pydantic" -version = "2.12.5" +version = "2.13.3" description = "Data validation using Python type hints" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ - {file = "pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d"}, - {file = "pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49"}, + {file = "pydantic-2.13.3-py3-none-any.whl", hash = "sha256:6db14ac8dfc9a1e57f87ea2c0de670c251240f43cb0c30a5130e9720dc612927"}, + {file = "pydantic-2.13.3.tar.gz", hash = "sha256:af09e9d1d09f4e7fe37145c1f577e1d61ceb9a41924bf0094a36506285d0a84d"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.41.5" +pydantic-core = "2.46.3" typing-extensions = ">=4.14.1" typing-inspection = ">=0.4.2" @@ -3046,134 +2954,132 @@ timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows [[package]] name = "pydantic-core" -version = "2.41.5" +version = "2.46.3" description = "Core functionality for Pydantic validation and serialization" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" -files = [ - {file = "pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146"}, - {file = "pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2"}, - {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97"}, - {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9"}, - {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52"}, - {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941"}, - {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a"}, - {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c"}, - {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2"}, - {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556"}, - {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49"}, - {file = "pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba"}, - {file = "pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9"}, - {file = "pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6"}, - {file = "pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b"}, - {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a"}, - {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8"}, - {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e"}, - {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1"}, - {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b"}, - {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b"}, - {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284"}, - {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594"}, - {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e"}, - {file = "pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b"}, - {file = "pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe"}, - {file = "pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f"}, - {file = "pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7"}, - {file = "pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0"}, - {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69"}, - {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75"}, - {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05"}, - {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc"}, - {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c"}, - {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5"}, - {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c"}, - {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294"}, - {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1"}, - {file = "pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d"}, - {file = "pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815"}, - {file = "pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3"}, - {file = "pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9"}, - {file = "pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34"}, - {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0"}, - {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33"}, - {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e"}, - {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2"}, - {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586"}, - {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d"}, - {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740"}, - {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e"}, - {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858"}, - {file = "pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36"}, - {file = "pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11"}, - {file = "pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd"}, - {file = "pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a"}, - {file = "pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14"}, - {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1"}, - {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66"}, - {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869"}, - {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2"}, - {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375"}, - {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553"}, - {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90"}, - {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07"}, - {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb"}, - {file = "pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23"}, - {file = "pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf"}, - {file = "pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c"}, - {file = "pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008"}, - {file = "pydantic_core-2.41.5-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8bfeaf8735be79f225f3fefab7f941c712aaca36f1128c9d7e2352ee1aa87bdf"}, - {file = "pydantic_core-2.41.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:346285d28e4c8017da95144c7f3acd42740d637ff41946af5ce6e5e420502dd5"}, - {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a75dafbf87d6276ddc5b2bf6fae5254e3d0876b626eb24969a574fff9149ee5d"}, - {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7b93a4d08587e2b7e7882de461e82b6ed76d9026ce91ca7915e740ecc7855f60"}, - {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8465ab91a4bd96d36dde3263f06caa6a8a6019e4113f24dc753d79a8b3a3f82"}, - {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:299e0a22e7ae2b85c1a57f104538b2656e8ab1873511fd718a1c1c6f149b77b5"}, - {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:707625ef0983fcfb461acfaf14de2067c5942c6bb0f3b4c99158bed6fedd3cf3"}, - {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425"}, - {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0384e2e1021894b1ff5a786dbf94771e2986ebe2869533874d7e43bc79c6f504"}, - {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:f0cd744688278965817fd0839c4a4116add48d23890d468bc436f78beb28abf5"}, - {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:753e230374206729bf0a807954bcc6c150d3743928a73faffee51ac6557a03c3"}, - {file = "pydantic_core-2.41.5-cp39-cp39-win32.whl", hash = "sha256:873e0d5b4fb9b89ef7c2d2a963ea7d02879d9da0da8d9d4933dee8ee86a8b460"}, - {file = "pydantic_core-2.41.5-cp39-cp39-win_amd64.whl", hash = "sha256:e4f4a984405e91527a0d62649ee21138f8e3d0ef103be488c1dc11a80d7f184b"}, - {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034"}, - {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c"}, - {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2"}, - {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad"}, - {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd"}, - {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc"}, - {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56"}, - {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b"}, - {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8"}, - {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a"}, - {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b"}, - {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2"}, - {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093"}, - {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a"}, - {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963"}, - {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a"}, - {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26"}, - {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808"}, - {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc"}, - {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1"}, - {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84"}, - {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770"}, - {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f"}, - {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51"}, - {file = "pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e"}, +groups = ["test"] +files = [ + {file = "pydantic_core-2.46.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:1da3786b8018e60349680720158cc19161cc3b4bdd815beb0a321cd5ce1ad5b1"}, + {file = "pydantic_core-2.46.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cc0988cb29d21bf4a9d5cf2ef970b5c0e38d8d8e107a493278c05dc6c1dda69f"}, + {file = "pydantic_core-2.46.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f9067c3bfadd04c55484b89c0d267981b2f3512850f6f66e1e74204a4e4ce3"}, + {file = "pydantic_core-2.46.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a642ac886ecf6402d9882d10c405dcf4b902abeb2972cd5fb4a48c83cd59279a"}, + {file = "pydantic_core-2.46.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79f561438481f28681584b89e2effb22855e2179880314bcddbf5968e935e807"}, + {file = "pydantic_core-2.46.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57a973eae4665352a47cf1a99b4ee864620f2fe663a217d7a8da68a1f3a5bfda"}, + {file = "pydantic_core-2.46.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83d002b97072a53ea150d63e0a3adfae5670cef5aa8a6e490240e482d3b22e57"}, + {file = "pydantic_core-2.46.3-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b40ddd51e7c44b28cfaef746c9d3c506d658885e0a46f9eeef2ee815cbf8e045"}, + {file = "pydantic_core-2.46.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ac5ec7fb9b87f04ee839af2d53bcadea57ded7d229719f56c0ed895bff987943"}, + {file = "pydantic_core-2.46.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a3b11c812f61b3129c4905781a2601dfdfdea5fe1e6c1cfb696b55d14e9c054f"}, + {file = "pydantic_core-2.46.3-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:1108da631e602e5b3c38d6d04fe5bb3bfa54349e6918e3ca6cf570b2e2b2f9d4"}, + {file = "pydantic_core-2.46.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:de885175515bcfa98ae618c1df7a072f13d179f81376c8007112af20567fd08a"}, + {file = "pydantic_core-2.46.3-cp310-cp310-win32.whl", hash = "sha256:d11058e3201527d41bc6b545c79187c9e4bf85e15a236a6007f0e991518882b7"}, + {file = "pydantic_core-2.46.3-cp310-cp310-win_amd64.whl", hash = "sha256:3612edf65c8ea67ac13616c4d23af12faef1ae435a8a93e5934c2a0cbbdd1fd6"}, + {file = "pydantic_core-2.46.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ab124d49d0459b2373ecf54118a45c28a1e6d4192a533fbc915e70f556feb8e5"}, + {file = "pydantic_core-2.46.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cca67d52a5c7a16aed2b3999e719c4bcf644074eac304a5d3d62dd70ae7d4b2c"}, + {file = "pydantic_core-2.46.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c024e08c0ba23e6fd68c771a521e9d6a792f2ebb0fa734296b36394dc30390e"}, + {file = "pydantic_core-2.46.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6645ce7eec4928e29a1e3b3d5c946621d105d3e79f0c9cddf07c2a9770949287"}, + {file = "pydantic_core-2.46.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a712c7118e6c5ea96562f7b488435172abb94a3c53c22c9efc1412264a45cbbe"}, + {file = "pydantic_core-2.46.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:69a868ef3ff206343579021c40faf3b1edc64b1cc508ff243a28b0a514ccb050"}, + {file = "pydantic_core-2.46.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc7e8c32db809aa0f6ea1d6869ebc8518a65d5150fdfad8bcae6a49ae32a22e2"}, + {file = "pydantic_core-2.46.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:3481bd1341dc85779ee506bc8e1196a277ace359d89d28588a9468c3ecbe63fa"}, + {file = "pydantic_core-2.46.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8690eba565c6d68ffd3a8655525cbdd5246510b44a637ee2c6c03a7ebfe64d3c"}, + {file = "pydantic_core-2.46.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4de88889d7e88d50d40ee5b39d5dac0bcaef9ba91f7e536ac064e6b2834ecccf"}, + {file = "pydantic_core-2.46.3-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:e480080975c1ef7f780b8f99ed72337e7cc5efea2e518a20a692e8e7b278eb8b"}, + {file = "pydantic_core-2.46.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:de3a5c376f8cd94da9a1b8fd3dd1c16c7a7b216ed31dc8ce9fd7a22bf13b836e"}, + {file = "pydantic_core-2.46.3-cp311-cp311-win32.whl", hash = "sha256:fc331a5314ffddd5385b9ee9d0d2fee0b13c27e0e02dad71b1ae5d6561f51eeb"}, + {file = "pydantic_core-2.46.3-cp311-cp311-win_amd64.whl", hash = "sha256:b5b9c6cf08a8a5e502698f5e153056d12c34b8fb30317e0c5fd06f45162a6346"}, + {file = "pydantic_core-2.46.3-cp311-cp311-win_arm64.whl", hash = "sha256:5dfd51cf457482f04ec49491811a2b8fd5b843b64b11eecd2d7a1ee596ea78a6"}, + {file = "pydantic_core-2.46.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b11b59b3eee90a80a36701ddb4576d9ae31f93f05cb9e277ceaa09e6bf074a67"}, + {file = "pydantic_core-2.46.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:af8653713055ea18a3abc1537fe2ebc42f5b0bbb768d1eb79fd74eb47c0ac089"}, + {file = "pydantic_core-2.46.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75a519dab6d63c514f3a81053e5266c549679e4aa88f6ec57f2b7b854aceb1b0"}, + {file = "pydantic_core-2.46.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a6cd87cb1575b1ad05ba98894c5b5c96411ef678fa2f6ed2576607095b8d9789"}, + {file = "pydantic_core-2.46.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f80a55484b8d843c8ada81ebf70a682f3f00a3d40e378c06cf17ecb44d280d7d"}, + {file = "pydantic_core-2.46.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3861f1731b90c50a3266316b9044f5c9b405eecb8e299b0a7120596334e4fe9c"}, + {file = "pydantic_core-2.46.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb528e295ed31570ac3dcc9bfdd6e0150bc11ce6168ac87a8082055cf1a67395"}, + {file = "pydantic_core-2.46.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:367508faa4973b992b271ba1494acaab36eb7e8739d1e47be5035fb1ea225396"}, + {file = "pydantic_core-2.46.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ad3c826fe523e4becf4fe39baa44286cff85ef137c729a2c5e269afbfd0905d"}, + {file = "pydantic_core-2.46.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ec638c5d194ef8af27db69f16c954a09797c0dc25015ad6123eb2c73a4d271ca"}, + {file = "pydantic_core-2.46.3-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:28ed528c45446062ee66edb1d33df5d88828ae167de76e773a3c7f64bd14e976"}, + {file = "pydantic_core-2.46.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aed19d0c783886d5bd86d80ae5030006b45e28464218747dcf83dabfdd092c7b"}, + {file = "pydantic_core-2.46.3-cp312-cp312-win32.whl", hash = "sha256:06d5d8820cbbdb4147578c1fe7ffcd5b83f34508cb9f9ab76e807be7db6ff0a4"}, + {file = "pydantic_core-2.46.3-cp312-cp312-win_amd64.whl", hash = "sha256:c3212fda0ee959c1dd04c60b601ec31097aaa893573a3a1abd0a47bcac2968c1"}, + {file = "pydantic_core-2.46.3-cp312-cp312-win_arm64.whl", hash = "sha256:f1f8338dd7a7f31761f1f1a3c47503a9a3b34eea3c8b01fa6ee96408affb5e72"}, + {file = "pydantic_core-2.46.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:12bc98de041458b80c86c56b24df1d23832f3e166cbaff011f25d187f5c62c37"}, + {file = "pydantic_core-2.46.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:85348b8f89d2c3508b65b16c3c33a4da22b8215138d8b996912bb1532868885f"}, + {file = "pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1105677a6df914b1fb71a81b96c8cce7726857e1717d86001f29be06a25ee6f8"}, + {file = "pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:87082cd65669a33adeba5470769e9704c7cf026cc30afb9cc77fd865578ebaad"}, + {file = "pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60e5f66e12c4f5212d08522963380eaaeac5ebd795826cfd19b2dfb0c7a52b9c"}, + {file = "pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b6cdf19bf84128d5e7c37e8a73a0c5c10d51103a650ac585d42dd6ae233f2b7f"}, + {file = "pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:031bb17f4885a43773c8c763089499f242aee2ea85cf17154168775dccdecf35"}, + {file = "pydantic_core-2.46.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:bcf2a8b2982a6673693eae7348ef3d8cf3979c1d63b54fca7c397a635cc68687"}, + {file = "pydantic_core-2.46.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28e8cf2f52d72ced402a137145923a762cbb5081e48b34312f7a0c8f55928ec3"}, + {file = "pydantic_core-2.46.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:17eaface65d9fc5abb940003020309c1bf7a211f5f608d7870297c367e6f9022"}, + {file = "pydantic_core-2.46.3-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:93fd339f23408a07e98950a89644f92c54d8729719a40b30c0a30bb9ebc55d23"}, + {file = "pydantic_core-2.46.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:23cbdb3aaa74dfe0837975dbf69b469753bbde8eacace524519ffdb6b6e89eb7"}, + {file = "pydantic_core-2.46.3-cp313-cp313-win32.whl", hash = "sha256:610eda2e3838f401105e6326ca304f5da1e15393ae25dacae5c5c63f2c275b13"}, + {file = "pydantic_core-2.46.3-cp313-cp313-win_amd64.whl", hash = "sha256:68cc7866ed863db34351294187f9b729964c371ba33e31c26f478471c52e1ed0"}, + {file = "pydantic_core-2.46.3-cp313-cp313-win_arm64.whl", hash = "sha256:f64b5537ac62b231572879cd08ec05600308636a5d63bcbdb15063a466977bec"}, + {file = "pydantic_core-2.46.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:afa3aa644f74e290cdede48a7b0bee37d1c35e71b05105f6b340d484af536d9b"}, + {file = "pydantic_core-2.46.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ced3310e51aa425f7f77da8bbbb5212616655bedbe82c70944320bc1dbe5e018"}, + {file = "pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e29908922ce9da1a30b4da490bd1d3d82c01dcfdf864d2a74aacee674d0bfa34"}, + {file = "pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0c9ff69140423eea8ed2d5477df3ba037f671f5e897d206d921bc9fdc39613e7"}, + {file = "pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b675ab0a0d5b1c8fdb81195dc5bcefea3f3c240871cdd7ff9a2de8aa50772eb2"}, + {file = "pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0087084960f209a9a4af50ecd1fb063d9ad3658c07bb81a7a53f452dacbfb2ba"}, + {file = "pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed42e6cc8e1b0e2b9b96e2276bad70ae625d10d6d524aed0c93de974ae029f9f"}, + {file = "pydantic_core-2.46.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:f1771ce258afb3e4201e67d154edbbae712a76a6081079fe247c2f53c6322c22"}, + {file = "pydantic_core-2.46.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a7610b6a5242a6c736d8ad47fd5fff87fcfe8f833b281b1c409c3d6835d9227f"}, + {file = "pydantic_core-2.46.3-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:ff5e7783bcc5476e1db448bf268f11cb257b1c276d3e89f00b5727be86dd0127"}, + {file = "pydantic_core-2.46.3-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:9d2e32edcc143bc01e95300671915d9ca052d4f745aa0a49c48d4803f8a85f2c"}, + {file = "pydantic_core-2.46.3-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6e42d83d1c6b87fa56b521479cff237e626a292f3b31b6345c15a99121b454c1"}, + {file = "pydantic_core-2.46.3-cp314-cp314-win32.whl", hash = "sha256:07bc6d2a28c3adb4f7c6ae46aa4f2d2929af127f587ed44057af50bf1ce0f505"}, + {file = "pydantic_core-2.46.3-cp314-cp314-win_amd64.whl", hash = "sha256:8940562319bc621da30714617e6a7eaa6b98c84e8c685bcdc02d7ed5e7c7c44e"}, + {file = "pydantic_core-2.46.3-cp314-cp314-win_arm64.whl", hash = "sha256:5dcbbcf4d22210ced8f837c96db941bdb078f419543472aca5d9a0bb7cddc7df"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:d0fe3dce1e836e418f912c1ad91c73357d03e556a4d286f441bf34fed2dbeecf"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9ce92e58abc722dac1bf835a6798a60b294e48eb0e625ec9fd994b932ac5feee"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a03e6467f0f5ab796a486146d1b887b2dc5e5f9b3288898c1b1c3ad974e53e4a"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2798b6ba041b9d70acfb9071a2ea13c8456dd1e6a5555798e41ba7b0790e329c"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9be3e221bdc6d69abf294dcf7aff6af19c31a5cdcc8f0aa3b14be29df4bd03b1"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f13936129ce841f2a5ddf6f126fea3c43cd128807b5a59588c37cf10178c2e64"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28b5f2ef03416facccb1c6ef744c69793175fd27e44ef15669201601cf423acb"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:830d1247d77ad23852314f069e9d7ddafeec5f684baf9d7e7065ed46a049c4e6"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0793c90c1a3c74966e7975eaef3ed30ebdff3260a0f815a62a22adc17e4c01c"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:d2d0aead851b66f5245ec0c4fb2612ef457f8bbafefdf65a2bf9d6bac6140f47"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:2f40e4246676beb31c5ce77c38a55ca4e465c6b38d11ea1bd935420568e0b1ab"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:cf489cf8986c543939aeee17a09c04d6ffb43bfef8ca16fcbcc5cfdcbed24dba"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-win32.whl", hash = "sha256:ffe0883b56cfc05798bf994164d2b2ff03efe2d22022a2bb080f3b626176dd56"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-win_amd64.whl", hash = "sha256:706d9d0ce9cf4593d07270d8e9f53b161f90c57d315aeec4fb4fd7a8b10240d8"}, + {file = "pydantic_core-2.46.3-cp314-cp314t-win_arm64.whl", hash = "sha256:77706aeb41df6a76568434701e0917da10692da28cb69d5fb6919ce5fdb07374"}, + {file = "pydantic_core-2.46.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:fa3eb7c2995aa443687a825bc30395c8521b7c6ec201966e55debfd1128bcceb"}, + {file = "pydantic_core-2.46.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3d08782c4045f90724b44c95d35ebec0d67edb8a957a2ac81d5a8e4b8a200495"}, + {file = "pydantic_core-2.46.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:831eb19aa789a97356979e94c981e5667759301fb708d1c0d5adf1bc0098b873"}, + {file = "pydantic_core-2.46.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4335e87c7afa436a0dfa899e138d57a72f8aad542e2cf19c36fb428461caabd0"}, + {file = "pydantic_core-2.46.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99421e7684a60f7f3550a1d159ade5fdff1954baedb6bdd407cba6a307c9f27d"}, + {file = "pydantic_core-2.46.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd81f6907932ebac3abbe41378dac64b2380db1287e2aa64d8d88f78d170f51a"}, + {file = "pydantic_core-2.46.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f247596366f4221af52beddd65af1218797771d6989bc891a0b86ccaa019168"}, + {file = "pydantic_core-2.46.3-cp39-cp39-manylinux_2_31_riscv64.whl", hash = "sha256:6dff8cc884679df229ebc6d8eb2321ea6f8e091bc7d4886d4dc2e0e71452843c"}, + {file = "pydantic_core-2.46.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68ef2f623dda6d5a9067ac014e406c020c780b2a358930a7e5c1b73702900720"}, + {file = "pydantic_core-2.46.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d56bdb4af1767cc15b0386b3c581fdfe659bb9ee4a4f776e92c1cd9d074000d6"}, + {file = "pydantic_core-2.46.3-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:91249bcb7c165c2fb2a2f852dbc5c91636e2e218e75d96dfdd517e4078e173dd"}, + {file = "pydantic_core-2.46.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b068543bdb707f5d935dab765d99227aa2545ef2820935f2e5dd801795c7dbd"}, + {file = "pydantic_core-2.46.3-cp39-cp39-win32.whl", hash = "sha256:dcda6583921c05a40533f982321532f2d8db29326c7b95c4026941fa5074bd79"}, + {file = "pydantic_core-2.46.3-cp39-cp39-win_amd64.whl", hash = "sha256:a35cc284c8dd7edae8a31533713b4d2467dfe7c4f1b5587dd4031f28f90d1d13"}, + {file = "pydantic_core-2.46.3-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:9715525891ed524a0a1eb6d053c74d4d4ad5017677fb00af0b7c2644a31bae46"}, + {file = "pydantic_core-2.46.3-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:9d2f400712a99a013aff420ef1eb9be077f8189a36c1e3ef87660b4e1088a874"}, + {file = "pydantic_core-2.46.3-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd2aab0e2e9dc2daf36bd2686c982535d5e7b1d930a1344a7bb6e82baab42a76"}, + {file = "pydantic_core-2.46.3-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e9d76736da5f362fabfeea6a69b13b7f2be405c6d6966f06b2f6bfff7e64531"}, + {file = "pydantic_core-2.46.3-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:b12dd51f1187c2eb489af8e20f880362db98e954b54ab792fa5d92e8bcc6b803"}, + {file = "pydantic_core-2.46.3-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:f00a0961b125f1a47af7bcc17f00782e12f4cd056f83416006b30111d941dfa3"}, + {file = "pydantic_core-2.46.3-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57697d7c056aca4bbb680200f96563e841a6386ac1129370a0102592f4dddff5"}, + {file = "pydantic_core-2.46.3-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd35aa21299def8db7ef4fe5c4ff862941a9a158ca7b63d61e66fe67d30416b4"}, + {file = "pydantic_core-2.46.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:13afdd885f3d71280cf286b13b310ee0f7ccfefd1dbbb661514a474b726e2f25"}, + {file = "pydantic_core-2.46.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f91c0aff3e3ee0928edd1232c57f643a7a003e6edf1860bc3afcdc749cb513f3"}, + {file = "pydantic_core-2.46.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6529d1d128321a58d30afcc97b49e98836542f68dd41b33c2e972bb9e5290536"}, + {file = "pydantic_core-2.46.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:975c267cff4f7e7272eacbe50f6cc03ca9a3da4c4fbd66fffd89c94c1e311aa1"}, + {file = "pydantic_core-2.46.3-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2b8e4f2bbdf71415c544b4b1138b8060db7b6611bc927e8064c769f64bed651c"}, + {file = "pydantic_core-2.46.3-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:e61ea8e9fff9606d09178f577ff8ccdd7206ff73d6552bcec18e1033c4254b85"}, + {file = "pydantic_core-2.46.3-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b504bda01bafc69b6d3c7a0c7f039dcf60f47fab70e06fe23f57b5c75bdc82b8"}, + {file = "pydantic_core-2.46.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:b00b76f7142fc60c762ce579bd29c8fa44aaa56592dd3c54fab3928d0d4ca6ff"}, + {file = "pydantic_core-2.46.3.tar.gz", hash = "sha256:41c178f65b8c29807239d47e6050262eb6bf84eb695e41101e62e38df4a5bc2c"}, ] [package.dependencies] @@ -3181,14 +3087,14 @@ typing-extensions = ">=4.14.1" [[package]] name = "pydata-sphinx-theme" -version = "0.17.0" +version = "0.17.1" description = "Bootstrap-based Sphinx theme from the PyData community" optional = false python-versions = ">=3.10" groups = ["docs"] files = [ - {file = "pydata_sphinx_theme-0.17.0-py3-none-any.whl", hash = "sha256:cec5c92f41f4a11541b6df8210c446b4aa9c3badb7fcf2db7893405b786d5c99"}, - {file = "pydata_sphinx_theme-0.17.0.tar.gz", hash = "sha256:529c5631582cb3328cf4814fb9eb80611d1704c854406d282a75c9c86e3a1955"}, + {file = "pydata_sphinx_theme-0.17.1-py3-none-any.whl", hash = "sha256:320b022d7808bdf5920d9a28e573f27aace9b23e1af6ca103eecc752411df492"}, + {file = "pydata_sphinx_theme-0.17.1.tar.gz", hash = "sha256:2cfc1d926c753c77039b7ee53f0ccebcbee5e81f0db61432b01cbb10ad7fd0af"}, ] [package.dependencies] @@ -3311,7 +3217,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["test"] +groups = ["main"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -3326,12 +3232,11 @@ version = "1.2.2" description = "Python interpreter discovery" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["dev", "test"] files = [ {file = "python_discovery-1.2.2-py3-none-any.whl", hash = "sha256:e1ae95d9af875e78f15e19aed0c6137ab1bb49c200f21f5061786490c9585c7a"}, {file = "python_discovery-1.2.2.tar.gz", hash = "sha256:876e9c57139eb757cb5878cbdd9ae5379e5d96266c99ef731119e04fffe533bb"}, ] -markers = {main = "extra == \"remote\""} [package.dependencies] filelock = ">=3.15.4" @@ -3341,61 +3246,6 @@ platformdirs = ">=4.3.6,<5" docs = ["furo (>=2025.12.19)", "sphinx (>=9.1)", "sphinx-autodoc-typehints (>=3.6.3)", "sphinxcontrib-mermaid (>=2)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.5.4)", "pytest (>=8.3.5)", "pytest-mock (>=3.14)", "setuptools (>=75.1)"] -[[package]] -name = "pytokens" -version = "0.4.1" -description = "A Fast, spec compliant Python 3.14+ tokenizer that runs on older Pythons." -optional = false -python-versions = ">=3.8" -groups = ["dev"] -files = [ - {file = "pytokens-0.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a44ed93ea23415c54f3face3b65ef2b844d96aeb3455b8a69b3df6beab6acc5"}, - {file = "pytokens-0.4.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:add8bf86b71a5d9fb5b89f023a80b791e04fba57960aa790cc6125f7f1d39dfe"}, - {file = "pytokens-0.4.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:670d286910b531c7b7e3c0b453fd8156f250adb140146d234a82219459b9640c"}, - {file = "pytokens-0.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4e691d7f5186bd2842c14813f79f8884bb03f5995f0575272009982c5ac6c0f7"}, - {file = "pytokens-0.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:27b83ad28825978742beef057bfe406ad6ed524b2d28c252c5de7b4a6dd48fa2"}, - {file = "pytokens-0.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d70e77c55ae8380c91c0c18dea05951482e263982911fc7410b1ffd1dadd3440"}, - {file = "pytokens-0.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a58d057208cb9075c144950d789511220b07636dd2e4708d5645d24de666bdc"}, - {file = "pytokens-0.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b49750419d300e2b5a3813cf229d4e5a4c728dae470bcc89867a9ad6f25a722d"}, - {file = "pytokens-0.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d9907d61f15bf7261d7e775bd5d7ee4d2930e04424bab1972591918497623a16"}, - {file = "pytokens-0.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6"}, - {file = "pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083"}, - {file = "pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1"}, - {file = "pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1"}, - {file = "pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9"}, - {file = "pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68"}, - {file = "pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b"}, - {file = "pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f"}, - {file = "pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1"}, - {file = "pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4"}, - {file = "pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78"}, - {file = "pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321"}, - {file = "pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa"}, - {file = "pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d"}, - {file = "pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324"}, - {file = "pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9"}, - {file = "pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb"}, - {file = "pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3"}, - {file = "pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975"}, - {file = "pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a"}, - {file = "pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918"}, - {file = "pytokens-0.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:da5baeaf7116dced9c6bb76dc31ba04a2dc3695f3d9f74741d7910122b456edc"}, - {file = "pytokens-0.4.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11edda0942da80ff58c4408407616a310adecae1ddd22eef8c692fe266fa5009"}, - {file = "pytokens-0.4.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0fc71786e629cef478cbf29d7ea1923299181d0699dbe7c3c0f4a583811d9fc1"}, - {file = "pytokens-0.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dcafc12c30dbaf1e2af0490978352e0c4041a7cde31f4f81435c2a5e8b9cabb6"}, - {file = "pytokens-0.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:42f144f3aafa5d92bad964d471a581651e28b24434d184871bd02e3a0d956037"}, - {file = "pytokens-0.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:34bcc734bd2f2d5fe3b34e7b3c0116bfb2397f2d9666139988e7a3eb5f7400e3"}, - {file = "pytokens-0.4.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:941d4343bf27b605e9213b26bfa1c4bf197c9c599a9627eb7305b0defcfe40c1"}, - {file = "pytokens-0.4.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3ad72b851e781478366288743198101e5eb34a414f1d5627cdd585ca3b25f1db"}, - {file = "pytokens-0.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:682fa37ff4d8e95f7df6fe6fe6a431e8ed8e788023c6bcc0f0880a12eab80ad1"}, - {file = "pytokens-0.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:30f51edd9bb7f85c748979384165601d028b84f7bd13fe14d3e065304093916a"}, - {file = "pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de"}, - {file = "pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a"}, -] - -[package.extras] -dev = ["black", "build", "mypy", "pytest", "pytest-cov", "setuptools", "tox", "twine", "wheel"] - [[package]] name = "pywin32-ctypes" version = "0.2.3" @@ -3415,7 +3265,7 @@ version = "6.0.3" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main", "dev", "docs"] +groups = ["dev", "docs", "test"] files = [ {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, @@ -3491,7 +3341,6 @@ files = [ {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, ] -markers = {main = "extra == \"remote\""} [[package]] name = "questionary" @@ -3510,28 +3359,30 @@ prompt_toolkit = ">=2.0,<4.0" [[package]] name = "ray" -version = "2.54.1" +version = "2.55.1" description = "Ray provides a simple, universal API for building distributed applications." -optional = true +optional = false python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"remote\"" -files = [ - {file = "ray-2.54.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:2ea650e648acc6e76edd98c694657fd1fcb1cd97700d944a7d20da90269e9810"}, - {file = "ray-2.54.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:6425f15cfe6a298366b53c8658350f94ced2c548802ca3b69f94b87db16e97c5"}, - {file = "ray-2.54.1-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:054985194bd32f4464c93f9318d247fac61e1f32ac221565ecfdc81ab8c75d0b"}, - {file = "ray-2.54.1-cp310-cp310-win_amd64.whl", hash = "sha256:512587412e2f5e1753adabfdfa4dd9cff1dc509601e36fd5fab671e448ae4dac"}, - {file = "ray-2.54.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c0240496af274af7cd3b1b1d015f23b88e5fdafe59bfdc040e5f229e0aff5dff"}, - {file = "ray-2.54.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:86c51eafd3e84dad59c1ef4cf97b3ac8c088af0705782ee915e31bca5880597a"}, - {file = "ray-2.54.1-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:e095dfe9c521a04e5930520b4a82ea82d61903d4cd2f3270fbc5dfbdb41b9c72"}, - {file = "ray-2.54.1-cp311-cp311-win_amd64.whl", hash = "sha256:ea90bed0110e0ce3ff6571e7a0c800920a3c6d299d29b8eac020dac362667169"}, - {file = "ray-2.54.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:645ebfb73cfd32bd510a05ed9f2738a18d6db69929cae9701d749f2740dbfd9a"}, - {file = "ray-2.54.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:cd452b61ae2e0daf9271f5a554614397429cc2731681bae10fe72316dadc2749"}, - {file = "ray-2.54.1-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:4c6f7e23dda62a32f94083141c3f97e9c4246e3ae4ae2bc488bcd8fd0311f54a"}, - {file = "ray-2.54.1-cp312-cp312-win_amd64.whl", hash = "sha256:673a895c0c4a716ed772552baa3f5b8d7d1f7a4b34e04787fdfe6fe3049ed0d8"}, - {file = "ray-2.54.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:d05f477d1518a00fd5880644e889a7a3eaf64ae5d1f8f239a682d052ad2a383d"}, - {file = "ray-2.54.1-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:2766f0230806480c38a9a94502087f1d4aea919f38521a28781690613b0290a4"}, - {file = "ray-2.54.1-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:0c3ae2943176e7b239c78b825a5b2bf4135d90280083a0e19c0a75a5db4d836f"}, +groups = ["test"] +files = [ + {file = "ray-2.55.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:2d5786661e192148719accc959def6cdcabd7a24cd9008005bf3d0e3c8cfd529"}, + {file = "ray-2.55.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:baf2ec89df7838cabdef493ff9bdbec1e6a6452f8bc696ad0c1b8a6198721745"}, + {file = "ray-2.55.1-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:bb49fbbe53a1d931e1f92d17f9271338f0b738885f8f70b7f531aa33f019d8af"}, + {file = "ray-2.55.1-cp310-cp310-win_amd64.whl", hash = "sha256:86e618e9ad8c6a24331c788eb599cee9838a62d2e10dfca0227743be06cf551c"}, + {file = "ray-2.55.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:0053fd5b400f7ac56263aa1bbd3d68fb79341b08b8dc697c88782d5aca7b3ed4"}, + {file = "ray-2.55.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:0ea2f670a7725833ad2333a8c46ab69865ad06c8e5de9f65695e0f8f35331cec"}, + {file = "ray-2.55.1-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:d5382da181c03ee2f502ef46cf0ae4bbc30157b5bd9a67d7651f6a272528a85a"}, + {file = "ray-2.55.1-cp311-cp311-win_amd64.whl", hash = "sha256:5e56d2e8f304cafe990c198a2b894f5b813de018998cd7212869201f6dc17cff"}, + {file = "ray-2.55.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:137f9006eee28caab8260803cca314f37bbda3fc94fdfa31c770b5d019626ad8"}, + {file = "ray-2.55.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:26541f69bb55607ef8335baac75b2ed12ff2ce02d56313219b29eda003039221"}, + {file = "ray-2.55.1-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:263705f6bab29e7622a94f82da25fd7f9cead76cdf89a07aab28f79cdf8f9d95"}, + {file = "ray-2.55.1-cp312-cp312-win_amd64.whl", hash = "sha256:9ad56704c8bd7e92130162f9c58e4ef473609515637673d5a36e761f95335206"}, + {file = "ray-2.55.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:f9844a9272ef2e6eb5771025866072cf4234cf4c7cc1a31e235b7de7111864be"}, + {file = "ray-2.55.1-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:b415d590e062f248907e0fe42994943f11726b7178fcf4b1cf5546721fb1a5f8"}, + {file = "ray-2.55.1-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:1380e043eb57cde69b7e9199c6f2558ceeb8f0fc41c97d1d5e50ea042115f302"}, + {file = "ray-2.55.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:b062045c64c2bce39a51661624f7292c7bbf30f2a9d878627aae31d46da5712d"}, + {file = "ray-2.55.1-cp314-cp314-manylinux2014_aarch64.whl", hash = "sha256:4e618d61e1b14b6fde9a586151f3fd9d435b0b85048b997bcaa7f4a533747b2b"}, + {file = "ray-2.55.1-cp314-cp314-manylinux2014_x86_64.whl", hash = "sha256:156ed3e72ad95b645d2006cd71a8dddbcc89b56bfc00027f6225adf78bd9cb74"}, ] [package.dependencies] @@ -3563,18 +3414,18 @@ virtualenv = {version = ">=20.0.24,<20.21.1 || >20.21.1", optional = true, marke [package.extras] adag = ["cupy-cuda12x ; sys_platform != \"darwin\""] air = ["aiohttp (>=3.13.3)", "aiohttp_cors", "colorful", "fastapi", "fsspec", "grpcio (>=1.42.0)", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "pandas", "pandas (>=1.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "requests", "smart_open", "starlette", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] -all = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm_tree", "fastapi", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.42.0)", "gymnasium (==1.2.2)", "lz4", "memray ; sys_platform != \"win32\"", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (>=1.7.0)", "pandas", "pandas (>=1.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "pyyaml", "requests", "scipy", "smart_open", "starlette", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] -all-cpp = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm_tree", "fastapi", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.42.0)", "gymnasium (==1.2.2)", "lz4", "memray ; sys_platform != \"win32\"", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (>=1.7.0)", "pandas", "pandas (>=1.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "pyyaml", "ray-cpp (==2.54.1)", "requests", "scipy", "smart_open", "starlette", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] +all = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm_tree", "fastapi", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.42.0)", "gymnasium (==1.2.2)", "lz4", "memray ; sys_platform != \"win32\"", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (>=1.7.0)", "pandas", "pandas (>=1.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "pyyaml", "requests", "scipy", "smart_open", "starlette", "taskiq", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] +all-cpp = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "cupy-cuda12x ; sys_platform != \"darwin\"", "dm_tree", "fastapi", "fsspec", "grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\"", "grpcio (>=1.42.0)", "gymnasium (==1.2.2)", "lz4", "memray ; sys_platform != \"win32\"", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "ormsgpack (>=1.7.0)", "pandas", "pandas (>=1.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "pyyaml", "ray-cpp (==2.55.1)", "requests", "scipy", "smart_open", "starlette", "taskiq", "tensorboardX (>=1.9)", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] cgraph = ["cupy-cuda12x ; sys_platform != \"darwin\""] client = ["grpcio", "grpcio (!=1.56.0) ; sys_platform == \"darwin\""] -cpp = ["ray-cpp (==2.54.1)"] +cpp = ["ray-cpp (==2.55.1)"] data = ["fsspec", "numpy (>=1.20)", "pandas (>=1.3)", "pyarrow (>=9.0.0)"] default = ["aiohttp (>=3.13.3)", "aiohttp_cors", "colorful", "grpcio (>=1.42.0)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "requests", "smart_open", "virtualenv (>=20.0.24,!=20.21.1)"] -llm = ["aiohttp (>=3.13.3)", "aiohttp_cors", "async-timeout ; python_version < \"3.11\"", "colorful", "fastapi", "fsspec", "grpcio (>=1.42.0)", "hf_transfer", "jsonref (>=1.1.0)", "jsonschema", "meson", "ninja", "nixl (>=0.6.1)", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "pandas (>=1.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyarrow (>=9.0.0)", "pybind11", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "requests", "smart_open", "starlette", "transformers (>=4.57.3)", "typer", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "vllm[audio] (>=0.15.0)", "watchfiles"] +llm = ["aiohttp (>=3.13.3)", "aiohttp_cors", "async-timeout ; python_version < \"3.11\"", "colorful", "fastapi", "fsspec", "grpcio (>=1.42.0)", "hf_transfer", "jsonref (>=1.1.0)", "jsonschema", "meson", "ninja", "nixl (>=1.0.0)", "numpy (>=1.20)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "pandas (>=1.3)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyarrow (>=9.0.0)", "pybind11", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "requests", "smart_open", "starlette", "typer", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "vllm[audio] (>=0.18.0)", "watchfiles"] observability = ["memray ; sys_platform != \"win32\""] rllib = ["dm_tree", "fsspec", "gymnasium (==1.2.2)", "lz4", "ormsgpack (>=1.7.0)", "pandas", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "pyyaml", "requests", "scipy", "tensorboardX (>=1.9)"] serve = ["aiohttp (>=3.13.3)", "aiohttp_cors", "colorful", "fastapi", "grpcio (>=1.42.0)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "requests", "smart_open", "starlette", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] -serve-async-inference = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "fastapi", "grpcio (>=1.42.0)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "requests", "smart_open", "starlette", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] +serve-async-inference = ["aiohttp (>=3.13.3)", "aiohttp_cors", "celery", "colorful", "fastapi", "grpcio (>=1.42.0)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "requests", "smart_open", "starlette", "taskiq", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] serve-grpc = ["aiohttp (>=3.13.3)", "aiohttp_cors", "colorful", "fastapi", "grpcio (>=1.42.0)", "opencensus", "opentelemetry-exporter-prometheus", "opentelemetry-proto", "opentelemetry-sdk (>=1.30.0)", "prometheus_client (>=0.7.1)", "py-spy (>=0.2.0) ; python_version < \"3.12\"", "py-spy (>=0.4.0) ; python_version >= \"3.12\"", "pyOpenSSL", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "requests", "smart_open", "starlette", "uvicorn[standard]", "virtualenv (>=20.0.24,!=20.21.1)", "watchfiles"] train = ["fsspec", "pandas", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "requests", "tensorboardX (>=1.9)"] tune = ["fsspec", "pandas", "pyarrow (>=9.0.0)", "pydantic (<2.0.dev0 || >=2.12.dev0,<3)", "requests", "tensorboardX (>=1.9)"] @@ -3603,10 +3454,9 @@ md = ["cmarkgfm (>=0.8.0)"] name = "referencing" version = "0.37.0" description = "JSON Referencing + Python" -optional = true +optional = false python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231"}, {file = "referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8"}, @@ -3623,12 +3473,11 @@ version = "2.33.1" description = "Python HTTP for Humans." optional = false python-versions = ">=3.10" -groups = ["main", "deploy", "docs"] +groups = ["deploy", "docs", "test"] files = [ {file = "requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a"}, {file = "requests-2.33.1.tar.gz", hash = "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517"}, ] -markers = {main = "extra == \"remote\""} [package.dependencies] certifi = ">=2023.5.7" @@ -3672,14 +3521,14 @@ idna2008 = ["idna"] [[package]] name = "rich" -version = "14.3.3" +version = "15.0.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false -python-versions = ">=3.8.0" +python-versions = ">=3.9.0" groups = ["deploy"] files = [ - {file = "rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d"}, - {file = "rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b"}, + {file = "rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb"}, + {file = "rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36"}, ] [package.dependencies] @@ -3705,10 +3554,9 @@ files = [ name = "rpds-py" version = "0.30.0" description = "Python bindings to Rust's persistent data structures (rpds)" -optional = true +optional = false python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288"}, {file = "rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00"}, @@ -3829,30 +3677,30 @@ files = [ [[package]] name = "ruff" -version = "0.15.10" +version = "0.15.11" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.15.10-py3-none-linux_armv6l.whl", hash = "sha256:0744e31482f8f7d0d10a11fcbf897af272fefdfcb10f5af907b18c2813ff4d5f"}, - {file = "ruff-0.15.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b1e7c16ea0ff5a53b7c2df52d947e685973049be1cdfe2b59a9c43601897b22e"}, - {file = "ruff-0.15.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:93cc06a19e5155b4441dd72808fdf84290d84ad8a39ca3b0f994363ade4cebb1"}, - {file = "ruff-0.15.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83e1dd04312997c99ea6965df66a14fb4f03ba978564574ffc68b0d61fd3989e"}, - {file = "ruff-0.15.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8154d43684e4333360fedd11aaa40b1b08a4e37d8ffa9d95fee6fa5b37b6fab1"}, - {file = "ruff-0.15.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ab88715f3a6deb6bde6c227f3a123410bec7b855c3ae331b4c006189e895cef"}, - {file = "ruff-0.15.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a768ff5969b4f44c349d48edf4ab4f91eddb27fd9d77799598e130fb628aa158"}, - {file = "ruff-0.15.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ee3ef42dab7078bda5ff6a1bcba8539e9857deb447132ad5566a038674540d0"}, - {file = "ruff-0.15.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51cb8cc943e891ba99989dd92d61e29b1d231e14811db9be6440ecf25d5c1609"}, - {file = "ruff-0.15.10-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e59c9bdc056a320fb9ea1700a8d591718b8faf78af065484e801258d3a76bc3f"}, - {file = "ruff-0.15.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:136c00ca2f47b0018b073f28cb5c1506642a830ea941a60354b0e8bc8076b151"}, - {file = "ruff-0.15.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8b80a2f3c9c8a950d6237f2ca12b206bccff626139be9fa005f14feb881a1ae8"}, - {file = "ruff-0.15.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:e3e53c588164dc025b671c9df2462429d60357ea91af7e92e9d56c565a9f1b07"}, - {file = "ruff-0.15.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b0c52744cf9f143a393e284125d2576140b68264a93c6716464e129a3e9adb48"}, - {file = "ruff-0.15.10-py3-none-win32.whl", hash = "sha256:d4272e87e801e9a27a2e8df7b21011c909d9ddd82f4f3281d269b6ba19789ca5"}, - {file = "ruff-0.15.10-py3-none-win_amd64.whl", hash = "sha256:28cb32d53203242d403d819fd6983152489b12e4a3ae44993543d6fe62ab42ed"}, - {file = "ruff-0.15.10-py3-none-win_arm64.whl", hash = "sha256:601d1610a9e1f1c2165a4f561eeaa2e2ea1e97f3287c5aa258d3dab8b57c6188"}, - {file = "ruff-0.15.10.tar.gz", hash = "sha256:d1f86e67ebfdef88e00faefa1552b5e510e1d35f3be7d423dc7e84e63788c94e"}, + {file = "ruff-0.15.11-py3-none-linux_armv6l.whl", hash = "sha256:e927cfff503135c558eb581a0c9792264aae9507904eb27809cdcff2f2c847b7"}, + {file = "ruff-0.15.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7a1b5b2938d8f890b76084d4fa843604d787a912541eae85fd7e233398bbb73e"}, + {file = "ruff-0.15.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d4176f3d194afbdaee6e41b9ccb1a2c287dba8700047df474abfbe773825d1cb"}, + {file = "ruff-0.15.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b17c886fb88203ced3afe7f14e8d5ae96e9d2f4ccc0ee66aa19f2c2675a27e4"}, + {file = "ruff-0.15.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:49fafa220220afe7758a487b048de4c8f9f767f37dfefad46b9dd06759d003eb"}, + {file = "ruff-0.15.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2ab8427e74a00d93b8bda1307b1e60970d40f304af38bccb218e056c220120d"}, + {file = "ruff-0.15.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:195072c0c8e1fc8f940652073df082e37a5d9cb43b4ab1e4d0566ab8977a13b7"}, + {file = "ruff-0.15.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a3a0996d486af3920dec930a2e7daed4847dfc12649b537a9335585ada163e9e"}, + {file = "ruff-0.15.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bef2cb556d509259f1fe440bb9cd33c756222cf0a7afe90d15edf0866702431"}, + {file = "ruff-0.15.11-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:030d921a836d7d4a12cf6e8d984a88b66094ccb0e0f17ddd55067c331191bf19"}, + {file = "ruff-0.15.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0e783b599b4577788dbbb66b9addcef87e9a8832f4ce0c19e34bf55543a2f890"}, + {file = "ruff-0.15.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ae90592246625ba4a34349d68ec28d4400d75182b71baa196ddb9f82db025ef5"}, + {file = "ruff-0.15.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1f111d62e3c983ed20e0ca2e800f8d77433a5b1161947df99a5c2a3fb60514f0"}, + {file = "ruff-0.15.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:06f483d6646f59eaffba9ae30956370d3a886625f511a3108994000480621d1c"}, + {file = "ruff-0.15.11-py3-none-win32.whl", hash = "sha256:476a2aa56b7da0b73a3ee80b6b2f0e19cce544245479adde7baa65466664d5f3"}, + {file = "ruff-0.15.11-py3-none-win_amd64.whl", hash = "sha256:8b6756d88d7e234fb0c98c91511aae3cd519d5e3ed271cae31b20f39cb2a12a3"}, + {file = "ruff-0.15.11-py3-none-win_arm64.whl", hash = "sha256:063fed18cc1bbe0ee7393957284a6fe8b588c6a406a285af3ee3f46da2391ee4"}, + {file = "ruff-0.15.11.tar.gz", hash = "sha256:f092b21708bf0e7437ce9ada249dfe688ff9a0954fc94abab05dcea7dcd29c33"}, ] [[package]] @@ -3904,19 +3752,17 @@ files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] -markers = {main = "extra == \"remote\""} [[package]] name = "smart-open" -version = "7.5.1" +version = "7.6.0" description = "Utils for streaming large files (S3, HDFS, GCS, SFTP, Azure Blob Storage, gzip, bz2, zst...)" -optional = true +optional = false python-versions = "<4.0,>=3.10" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ - {file = "smart_open-7.5.1-py3-none-any.whl", hash = "sha256:3e07cbbd9c8a908bcb8e25d48becf1a5cbb4886fa975e9f34c672ed171df2318"}, - {file = "smart_open-7.5.1.tar.gz", hash = "sha256:3f08e16827c4733699e6b2cc40328a3568f900cb12ad9a3ad233ba6c872d9fe7"}, + {file = "smart_open-7.6.0-py3-none-any.whl", hash = "sha256:2a78f454610a826aa688065b54b4a0a9b12a5599fa61d5190e9bac2df5e5f53f"}, + {file = "smart_open-7.6.0.tar.gz", hash = "sha256:44717f46b5ff276fac03b88e5d13d1c416f064f3b7b081381b0fa8889004bd7e"}, ] [package.dependencies] @@ -3924,7 +3770,7 @@ wrapt = "*" [package.extras] all = ["smart_open[azure,gcs,http,s3,ssh,webhdfs,zst]"] -azure = ["azure-common", "azure-core", "azure-storage-blob"] +azure = ["azure-common", "azure-core", "azure-storage-blob (>=12.7.0)"] gcs = ["google-api-core (<2.28) ; python_version < \"3.10\"", "google-cloud-storage (>=2.6.0)"] http = ["requests"] s3 = ["boto3 (>=1.9.17)"] @@ -4257,12 +4103,11 @@ version = "2.6.5" description = "TensorBoardX lets you watch Tensors Flow without Tensorflow" optional = false python-versions = ">=3.9" -groups = ["main", "test"] +groups = ["test"] files = [ {file = "tensorboardx-2.6.5-py3-none-any.whl", hash = "sha256:c10b891d00af306537cb8b58a039b2ba41571f0da06f433a41c4ca8d6abe1373"}, {file = "tensorboardx-2.6.5.tar.gz", hash = "sha256:ca176db3997ee8c07d2eb77381225956a3fd1c10c91beafab1f17069adc47017"}, ] -markers = {main = "extra == \"tboard\""} [package.dependencies] numpy = "*" @@ -4296,6 +4141,18 @@ files = [ {file = "tomlkit-0.14.0.tar.gz", hash = "sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064"}, ] +[[package]] +name = "topohub" +version = "1.5.1" +description = "Repository of reference Gabriel graph, Internet Topology Zoo, SNDlib, CAIDA and synthetic backbone topologies for networking research" +optional = false +python-versions = "*" +groups = ["test"] +files = [ + {file = "topohub-1.5.1-py3-none-any.whl", hash = "sha256:331dd0dab419ede67d5a680b724795d2320fb7f45bd8f2cf2281f155a117cdbb"}, + {file = "topohub-1.5.1.tar.gz", hash = "sha256:5a446ce2de00b31a751d16ad21d11754c22f397c3bfe5a57f0ca25c05f646277"}, +] + [[package]] name = "twine" version = "6.2.0" @@ -4328,21 +4185,19 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "dev", "docs", "test"] +groups = ["dev", "docs", "test"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, ] -markers = {main = "extra == \"remote\"", test = "python_version < \"3.13\""} [[package]] name = "typing-inspection" version = "0.4.2" description = "Runtime typing introspection tools" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7"}, {file = "typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464"}, @@ -4357,7 +4212,7 @@ version = "2026.1" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" -groups = ["test"] +groups = ["main"] markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\"" files = [ {file = "tzdata-2026.1-py2.py3-none-any.whl", hash = "sha256:4b1d2be7ac37ceafd7327b961aa3a54e467efbdb563a23655fbfe0d39cfc42a9"}, @@ -4381,12 +4236,11 @@ version = "2.6.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" -groups = ["main", "deploy", "docs"] +groups = ["deploy", "docs", "test"] files = [ {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, ] -markers = {main = "extra == \"remote\""} [package.extras] brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] @@ -4396,14 +4250,14 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "uvicorn" -version = "0.44.0" +version = "0.46.0" description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.10" groups = ["docs"] files = [ - {file = "uvicorn-0.44.0-py3-none-any.whl", hash = "sha256:ce937c99a2cc70279556967274414c087888e8cec9f9c94644dfca11bd3ced89"}, - {file = "uvicorn-0.44.0.tar.gz", hash = "sha256:6c942071b68f07e178264b9152f1f16dfac5da85880c4ce06366a96d70d4f31e"}, + {file = "uvicorn-0.46.0-py3-none-any.whl", hash = "sha256:bbebbcbed972d162afca128605223022bedd345b7bc7855ce66deb31487a9048"}, + {file = "uvicorn-0.46.0.tar.gz", hash = "sha256:fb9da0926999cc6cb22dc7cd71a94a632f078e6ae47ff683c5c420750fb7413d"}, ] [package.dependencies] @@ -4415,22 +4269,21 @@ standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.6.3) [[package]] name = "virtualenv" -version = "21.2.1" +version = "21.2.4" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["dev", "test"] files = [ - {file = "virtualenv-21.2.1-py3-none-any.whl", hash = "sha256:bd16b49c53562b28cf1a3ad2f36edb805ad71301dee70ddc449e5c88a9f919a2"}, - {file = "virtualenv-21.2.1.tar.gz", hash = "sha256:b66ffe81301766c0d5e2208fc3576652c59d44e7b731fc5f5ed701c9b537fa78"}, + {file = "virtualenv-21.2.4-py3-none-any.whl", hash = "sha256:29d21e941795206138d0f22f4e45ff7050e5da6c6472299fb7103318763861ac"}, + {file = "virtualenv-21.2.4.tar.gz", hash = "sha256:b294ef68192638004d72524ce7ef303e9d0cf5a44c95ce2e54a7500a6381cada"}, ] -markers = {main = "extra == \"remote\""} [package.dependencies] distlib = ">=0.3.7,<1" filelock = {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""} platformdirs = ">=3.9.1,<5" -python-discovery = ">=1" +python-discovery = ">=1.2.2" [[package]] name = "watchfiles" @@ -4639,22 +4492,19 @@ files = [ [[package]] name = "wheel" -version = "0.46.3" +version = "0.47.0" description = "Command line tool for manipulating wheel files" optional = false python-versions = ">=3.9" groups = ["deploy"] files = [ - {file = "wheel-0.46.3-py3-none-any.whl", hash = "sha256:4b399d56c9d9338230118d705d9737a2a468ccca63d5e813e2a4fc7815d8bc4d"}, - {file = "wheel-0.46.3.tar.gz", hash = "sha256:e3e79874b07d776c40bd6033f8ddf76a7dad46a7b8aa1b2787a83083519a1803"}, + {file = "wheel-0.47.0-py3-none-any.whl", hash = "sha256:212281cab4dff978f6cedd499cd893e1f620791ca6ff7107cf270781e587eced"}, + {file = "wheel-0.47.0.tar.gz", hash = "sha256:cc72bd1009ba0cf63922e28f94d9d83b920aa2bb28f798a31d0691b02fa3c9b3"}, ] [package.dependencies] packaging = ">=24.0" -[package.extras] -test = ["pytest (>=6.0.0)", "setuptools (>=77)"] - [[package]] name = "win32-setctime" version = "1.2.0" @@ -4677,7 +4527,7 @@ version = "2.1.2" description = "Module for decorators, wrappers and monkey patching." optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["dev", "test"] files = [ {file = "wrapt-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4b7a86d99a14f76facb269dc148590c01aaf47584071809a70da30555228158c"}, {file = "wrapt-2.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a819e39017f95bf7aede768f75915635aa8f671f2993c036991b8d3bfe8dbb6f"}, @@ -4770,7 +4620,6 @@ files = [ {file = "wrapt-2.1.2-py3-none-any.whl", hash = "sha256:b8fd6fa2b2c4e7621808f8c62e8317f4aae56e59721ad933bac5239d913cf0e8"}, {file = "wrapt-2.1.2.tar.gz", hash = "sha256:3996a67eecc2c68fd47b4e3c564405a5777367adfd9b8abb58387b63ee83b21e"}, ] -markers = {main = "extra == \"remote\""} [package.extras] dev = ["pytest", "setuptools"] @@ -4779,10 +4628,9 @@ dev = ["pytest", "setuptools"] name = "yarl" version = "1.23.0" description = "Yet another URL library" -optional = true +optional = false python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"remote\"" +groups = ["test"] files = [ {file = "yarl-1.23.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cff6d44cb13d39db2663a22b22305d10855efa0fa8015ddeacc40bc59b9d8107"}, {file = "yarl-1.23.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c53f8347cd4200f0d70a48ad059cabaf24f5adc6ba08622a23423bc7efa10d"}, @@ -4921,16 +4769,16 @@ propcache = ">=0.2.1" [[package]] name = "zipp" -version = "3.23.0" +version = "3.23.1" description = "Backport of pathlib-compatible object wrapper for zip files" -optional = true +optional = false python-versions = ">=3.9" -groups = ["main", "deploy"] +groups = ["deploy", "test"] files = [ - {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, - {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, + {file = "zipp-3.23.1-py3-none-any.whl", hash = "sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc"}, + {file = "zipp-3.23.1.tar.gz", hash = "sha256:32120e378d32cd9714ad503c1d024619063ec28aad2248dc6672ad13edfa5110"}, ] -markers = {main = "extra == \"remote\"", deploy = "python_version == \"3.11\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} +markers = {deploy = "python_version == \"3.11\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\""} [package.extras] check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] @@ -4940,11 +4788,7 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] -[extras] -remote = ["ray"] -tboard = ["tensorboardx"] - [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.14" -content-hash = "56f4da55e27c845e2afe6a42a0edbf47bdf4dcaac37e4eda9d0b5b93f5c1dd54" +content-hash = "4e3158ce4f8dde54e8ab2f11256dd7a058738ea52a89b038b86ed93f72548340" diff --git a/pyproject.toml b/pyproject.toml index 3b619a9..79b1566 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "networkx (>=3.6.1,<4.0.0)", "loguru (>=0.7.3,<0.8.0)", "aiofiles (>=25.1.0,<26.0.0)", + "pandas (>=3.0.2,<4.0.0)", ] [project.urls] @@ -27,24 +28,18 @@ Homepage = "https://github.com/eclypse-org/eclypse" Repository = "https://github.com/eclypse-org/eclypse" -[project.optional-dependencies] -remote = ["ray[default] (>=2.54.0,<3.0.0)"] -tboard = ["tensorboardx (>=2.6.5,<3.0.0)"] - - # Dependency Groups [dependency-groups] dev = [ - "black (>=26.3.1,<27.0.0)", - "commitizen (>=4.13.9,<5.0.0)", - "pre-commit (>=4.5.1,<5.0.0)", - "mypy (>=1.20.0,<2.0.0)", + "commitizen (>=4.13.10,<5.0.0)", + "pre-commit (>=4.6.0,<5.0.0)", + "mypy (>=1.20.2,<2.0.0)", "isort (>=8.0.1,<9.0.0)", - "ruff (>=0.15.10,<0.16.0)", + "ruff (>=0.15.11,<0.16.0)", ] deploy = [ - "wheel (>=0.46.3,<0.47.0)", + "wheel (>=0.47.0,<0.48.0)", "setuptools (>=82.0.1,<83.0.0)", "twine (>=6.2.0,<7.0.0)", ] @@ -54,9 +49,10 @@ test = [ "pytest-asyncio (>=1.3.0,<2.0.0)", "pytest-cov (>=7.1.0,<8.0.0)", "pytest-xdist (>=3.8.0,<4.0.0)", - "pandas (>=3.0.2,<4.0.0)", "tensorboardx (>=2.6.5,<3.0.0)", - "polars (>=1.39.3,<2.0.0)", + "polars (>=1.40.1,<2.0.0)", + "topohub (>=1.5.1,<2.0.0)", + "ray[default] (>=2.55.1,<3.0.0)", ] docs = [ @@ -68,7 +64,7 @@ docs = [ "sphinx-favicon (>=1.1.0,<2.0.0)", "sphinx-design (>=0.7.0,<0.8.0)", "sphinx-icon (>=0.2.2,<0.3.0)", - "pydata-sphinx-theme (>=0.17.0,<0.18.0)", + "pydata-sphinx-theme (>=0.17.1,<0.18.0)", "sphinx (>=9.1.0,<10.0.0) ; python_version >= '3.12'", "sphinx (>=9.0.4,<9.1.0) ; python_version < '3.12'", ] @@ -91,6 +87,15 @@ optional = true requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" +[project.scripts] +echo = "examples.echo.main:main" +grid-analysis = "examples.grid_analysis.main:main" +image-prediction = "examples.image_prediction.main:main" +off-the-shelf = "examples.off_the_shelf.main:main" +sock-shop-mpi = "examples.sock_shop.mpi:main" +sock-shop-rest = "examples.sock_shop.rest:main" +user-distribution = "examples.user_distribution.main:main" + # ------------------------- # Third-party tool settings @@ -144,13 +149,13 @@ ignore_missing_imports = true [tool.pytest.ini_options] testpaths = ["tests"] -addopts = "--cov=eclypse --cov-report=xml --cov-report=term-missing" +addopts = "--cov=eclypse --cov-report=xml --cov-report=term-missing -n auto" asyncio_mode = "auto" filterwarnings = ["ignore::DeprecationWarning", "ignore::FutureWarning"] markers = [ "integration: end-to-end integration tests", "emulation: ray-backed remote integration tests", - "extras: tests that exercise optional dependencies", + "optional: tests that exercise optional dependencies", ] [tool.coverage.run] diff --git a/tests/fixtures/domain.py b/tests/fixtures/domain.py index fdad534..73eb531 100644 --- a/tests/fixtures/domain.py +++ b/tests/fixtures/domain.py @@ -16,6 +16,7 @@ from eclypse.placement.view import PlacementView from eclypse.remote.service.service import Service from eclypse.simulation.config import SimulationConfig +from eclypse.utils._logging import config_logger class BasicService(Service): @@ -35,6 +36,7 @@ class DummyLogger: def __init__(self): self.records: list[tuple[str, tuple[Any, ...]]] = [] + self.levels = {"ECLYPSE", "ECLYPSE_EXCEPTION"} def bind(self, **_: Any) -> DummyLogger: return self @@ -46,6 +48,9 @@ def debug(self, *args: Any): self.records.append(("debug", args)) def log(self, *args: Any): + if args and isinstance(args[0], str) and args[0] in self.levels: + self.records.append((args[0], args[1:])) + return self.records.append(("log", args)) def warning(self, *args: Any): @@ -54,12 +59,20 @@ def warning(self, *args: Any): def error(self, *args: Any): self.records.append(("error", args)) + def info(self, *args: Any): + self.records.append(("info", args)) + @pytest.fixture def dummy_logger() -> DummyLogger: return DummyLogger() +@pytest.fixture(autouse=True) +def configured_eclypse_logger() -> None: + config_logger() + + @pytest.fixture def sample_infrastructure() -> Infrastructure: infrastructure = Infrastructure("edge-cloud", include_default_assets=True, seed=7) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..70da13d --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Integration test package.""" diff --git a/tests/integration/emulation/__init__.py b/tests/integration/emulation/__init__.py new file mode 100644 index 0000000..6201a1d --- /dev/null +++ b/tests/integration/emulation/__init__.py @@ -0,0 +1 @@ +"""Ray-backed emulation integration tests.""" diff --git a/tests/integration/emulation/_helpers.py b/tests/integration/emulation/_helpers.py new file mode 100644 index 0000000..2b4baf9 --- /dev/null +++ b/tests/integration/emulation/_helpers.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +_RAY_PROBE_STATE: dict[str, str | None] = {"blocked_reason": None} + + +def run_remote_probe( + repo_root: Path, + script: str, + timeout: int = 25, +) -> dict[str, Any]: + blocked_reason = _RAY_PROBE_STATE["blocked_reason"] + + if blocked_reason is not None: + pytest.skip(blocked_reason) + + env = os.environ.copy() + env["RAY_ENABLE_UV_RUN_RUNTIME_ENV"] = "0" + env["PYTHONPATH"] = str(repo_root) + + try: + completed = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + check=False, + cwd=repo_root, + env=env, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + _RAY_PROBE_STATE["blocked_reason"] = f"Ray integration probe timed out: {exc}" + pytest.skip(_RAY_PROBE_STATE["blocked_reason"]) + + if completed.returncode != 0: + combined_output = f"{completed.stdout}\n{completed.stderr}" + blocked_markers = ( + "PermissionError", + "Operation not permitted", + "Timed out waiting for file", + "gcs_server_port_", + ) + if any(marker in combined_output for marker in blocked_markers): + _RAY_PROBE_STATE["blocked_reason"] = ( + f"Ray integration probe is not permitted here:\n{combined_output}" + ) + pytest.skip(_RAY_PROBE_STATE["blocked_reason"]) + pytest.fail( + "Ray integration probe failed.\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}" + ) + + lines = [line for line in completed.stdout.splitlines() if line.strip()] + assert lines, "Expected JSON output from the Ray integration probe." + + for line in reversed(lines): + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + return payload + + pytest.fail( + "Ray integration probe did not emit a JSON object.\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}" + ) diff --git a/tests/integration/emulation/test_benchmark_builders.py b/tests/integration/emulation/test_benchmark_builders.py new file mode 100644 index 0000000..d6e6340 --- /dev/null +++ b/tests/integration/emulation/test_benchmark_builders.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from ._helpers import run_remote_probe + + +def _build_benchmark_probe_script( + output_dir: Path, + benchmark_name: str, + builder_name: str, + communication_interface: str, + entry_service: str, +) -> str: + return textwrap.dedent( + f""" + from __future__ import annotations + + import json + import os + import time + from pathlib import Path + + os.environ["RAY_ENABLE_UV_RUN_RUNTIME_ENV"] = "0" + os.environ["ECLYPSE_RND_SEED"] = "7" + + import ray + + from eclypse.builders.application import {builder_name} + from eclypse.graph import Infrastructure + from eclypse.placement.strategies import StaticStrategy + from eclypse.remote import ray_backend + from eclypse.simulation.config import SimulationConfig + from eclypse.simulation.simulation import Simulation + + + def read_step_results(service): + results = [] + for item in list(service._step_queue): + if hasattr(item, "status_code") and hasattr(item, "body"): + results.append( + {{ + "status_code": int(item.status_code), + "body": item.body, + }} + ) + else: + results.append(item) + return {{ + "step_count": service.step_count, + "results": results, + }} + + + def read_node_state(node): + return {{ + "node_id": node.id, + "services": sorted(node.services.keys()), + }} + + + ray.shutdown() + ray.init(address="local", include_dashboard=False, ignore_reinit_error=True) + ray_backend._backend = ray + ray_backend.init = lambda runtime_env: None + + infrastructure = Infrastructure( + "benchmark-infr", + include_default_assets=True, + seed=7, + ) + infrastructure.add_node( + "edge-a", + availability=1, + cpu=128, + ram=256, + storage=2048, + gpu=16, + processing_time=1, + ) + + application = {builder_name}( + application_id={benchmark_name!r}, + communication_interface={communication_interface!r}, + include_default_assets=True, + store_step=True, + ) + + config = SimulationConfig( + path=Path({str(output_dir)!r}), + report_backend="pandas", + report_format="csv", + remote=True, + step_every_ms=100, + ) + simulation = Simulation(infrastructure, config) + simulation.register( + application, + StaticStrategy( + {{ + service_id: "edge-a" + for service_id in application.nodes + }} + ), + ) + simulation.start() + + edge_actor = ray.get_actor(f"{{infrastructure.id}}/edge-a") + deadline = time.monotonic() + 15 + payload = {{"step_count": 0, "results": []}} + while time.monotonic() < deadline: + node_state = ray.get(edge_actor.entrypoint.remote(None, read_node_state)) + if {entry_service!r} not in node_state["services"]: + time.sleep(0.1) + continue + payload = ray.get( + edge_actor.entrypoint.remote({entry_service!r}, read_step_results) + ) + if payload["step_count"] >= 1 and payload["results"]: + break + time.sleep(0.1) + + simulation.stop(blocking=False) + simulation.wait(timeout=15) + + print(json.dumps(payload)) + + ray.shutdown() + """ + ) + + +def _assert_benchmark_result(benchmark_name: str, result: dict) -> None: + assert result["step_count"] >= 1 + + if benchmark_name == "video_mpi": + assert result["results"][0]["response_type"] == "analytics_result" + assert result["results"][0]["object_count"] == 2 + assert result["results"][0]["summary"] == "person, forklift" + return + + if benchmark_name == "hotel_rest": + assert result["results"][0]["status_code"] == 201 + assert result["results"][0]["body"]["reservation_id"] == "rsv-2001" + assert result["results"][0]["body"]["status"] == "confirmed" + assert result["results"][0]["body"]["transaction_id"].startswith("txn-") + return + + if benchmark_name == "crud_mpi": + assert result["results"][0]["response_type"] == "crud_response" + assert result["results"][0]["status"] == "recorded" + assert result["results"][0]["items"][0]["id"] == "item-1" + return + + if benchmark_name == "keyword_rest": + assert result["results"][0]["status_code"] == 200 + assert result["results"][0]["body"]["command"] == "wake" + return + + if benchmark_name == "anomaly_mpi": + assert result["results"][0]["response_type"] == "anomaly_response" + assert result["results"][0]["status"] == "normal" + assert result["results"][0]["score"] == pytest.approx(2.08) + return + + if benchmark_name == "thumbnail_rest": + assert result["results"][0]["status_code"] == 200 + assert result["results"][0]["body"]["status"] == "stored" + assert result["results"][0]["body"]["uri"].endswith("/img-1.jpg") + return + + msg = f"Unhandled benchmark case: {benchmark_name}" + raise AssertionError(msg) + + +@pytest.mark.integration +@pytest.mark.emulation +@pytest.mark.parametrize( + ( + "benchmark_name", + "builder_name", + "communication_interface", + "entry_service", + ), + [ + pytest.param( + "video_mpi", + "get_video_analytics_serving", + "mpi", + "CameraGatewayService", + id="video-mpi", + ), + pytest.param( + "hotel_rest", + "get_hotel_reservation", + "rest", + "FrontendService", + id="hotel-rest", + ), + pytest.param( + "crud_mpi", + "get_crud_api", + "mpi", + "GatewayService", + id="crud-mpi", + ), + pytest.param( + "keyword_rest", + "get_keyword_spotting", + "rest", + "SensorService", + id="keyword-rest", + ), + pytest.param( + "anomaly_mpi", + "get_anomaly_detection", + "mpi", + "SensorService", + id="anomaly-mpi", + ), + pytest.param( + "thumbnail_rest", + "get_thumbnailer", + "rest", + "UploadService", + id="thumbnail-rest", + ), + ], +) +def test_ray_benchmark_entrypoints( + tmp_path: Path, + benchmark_name: str, + builder_name: str, + communication_interface: str, + entry_service: str, +): + repo_root = Path(__file__).resolve().parents[3] + output_dir = tmp_path / benchmark_name + script = _build_benchmark_probe_script( + output_dir=output_dir, + benchmark_name=benchmark_name, + builder_name=builder_name, + communication_interface=communication_interface, + entry_service=entry_service, + ) + + result = run_remote_probe(repo_root, script, timeout=25) + + _assert_benchmark_result(benchmark_name, result) diff --git a/tests/integration/emulation/test_reports.py b/tests/integration/emulation/test_reports.py new file mode 100644 index 0000000..645aeef --- /dev/null +++ b/tests/integration/emulation/test_reports.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from ._helpers import run_remote_probe + + +@pytest.mark.integration +@pytest.mark.emulation +def test_ray_emulation_runtime_generates_reports(tmp_path: Path): + repo_root = Path(__file__).resolve().parents[3] + output_dir = tmp_path / "ray-simulation" + script = textwrap.dedent( + f""" + from __future__ import annotations + + import json + import os + from pathlib import Path + + os.environ["RAY_ENABLE_UV_RUN_RUNTIME_ENV"] = "0" + os.environ["ECLYPSE_RND_SEED"] = "7" + + import ray + + from eclypse.graph import Application, Infrastructure + from eclypse.placement.strategies import StaticStrategy + from eclypse.remote import ray_backend + from eclypse.remote.service.service import Service + from eclypse.simulation.config import SimulationConfig + from eclypse.simulation.simulation import Simulation + + + class CounterService(Service): + def __init__(self, service_id: str): + super().__init__(service_id, store_step=True) + + async def step(self): + if self.step_count >= 2: + self._running = False + return self.step_count + + + infrastructure = Infrastructure("edge-cloud", include_default_assets=True, seed=7) + infrastructure.add_node("edge-a", availability=1, cpu=4, ram=8, storage=16, gpu=0, processing_time=2) + infrastructure.add_node("edge-b", availability=1, cpu=8, ram=16, storage=32, gpu=1, processing_time=3) + infrastructure.add_edge("edge-a", "edge-b", latency=5, bandwidth=10) + infrastructure.add_edge("edge-b", "edge-a", latency=7, bandwidth=12) + + application = Application("shop", include_default_assets=True, seed=7) + application.add_service(CounterService("gateway"), cpu=1, ram=2, storage=2, gpu=0) + application.add_service(CounterService("worker"), cpu=2, ram=2, storage=4, gpu=0) + application.add_edge("gateway", "worker", latency=6, bandwidth=4) + application.flows = [["gateway", "worker"]] + + ray.shutdown() + ray.init(address="local", include_dashboard=False, ignore_reinit_error=True) + ray_backend._backend = ray + ray_backend.init = lambda runtime_env: None + + config = SimulationConfig( + path=Path({str(output_dir)!r}), + report_backend="pandas", + report_format="csv", + include_default_metrics=True, + remote=True, + step_every_ms="auto", + max_steps=3, + ) + simulation = Simulation(infrastructure, config) + simulation.register(application, StaticStrategy({{"gateway": "edge-a", "worker": "edge-b"}})) + simulation.start() + simulation.wait(timeout=15) + + service_rows = simulation.report.service() + callback_ids = service_rows["callback_id"].tolist() + payload = {{ + "status": str(simulation.status), + "path_exists": simulation.path.exists(), + "csv_service_exists": (simulation.path / "csv" / "service.csv").exists(), + "step_result_callbacks": sum(callback_id == "step_result" for callback_id in callback_ids), + "service_count": len(service_rows), + }} + print(json.dumps(payload)) + + ray.shutdown() + """ + ) + + result = run_remote_probe(repo_root, script) + + assert result["path_exists"] is True + assert result["csv_service_exists"] is True + assert result["service_count"] > 0 + assert result["step_result_callbacks"] > 0 diff --git a/tests/integration/emulation/test_routing.py b/tests/integration/emulation/test_routing.py new file mode 100644 index 0000000..05095b7 --- /dev/null +++ b/tests/integration/emulation/test_routing.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from ._helpers import run_remote_probe + + +@pytest.mark.integration +@pytest.mark.emulation +def test_ray_emulation_runtime_resolves_routes_and_neighbors(tmp_path: Path): + repo_root = Path(__file__).resolve().parents[3] + output_dir = tmp_path / "ray-routing" + script = textwrap.dedent( + f""" + from __future__ import annotations + + import json + import os + from pathlib import Path + + os.environ["RAY_ENABLE_UV_RUN_RUNTIME_ENV"] = "0" + os.environ["ECLYPSE_RND_SEED"] = "7" + + import ray + + from eclypse.graph import Application, Infrastructure + from eclypse.placement.strategies import StaticStrategy + from eclypse.remote import ray_backend + from eclypse.remote.service.service import Service + from eclypse.simulation.config import SimulationConfig + from eclypse.simulation.simulation import Simulation + + + class CounterService(Service): + async def step(self): + self._running = False + return self.step_count + + + infrastructure = Infrastructure("edge-cloud", include_default_assets=True, seed=7) + infrastructure.add_node("edge-a", availability=1, cpu=4, ram=8, storage=16, gpu=0, processing_time=2) + infrastructure.add_node("edge-b", availability=1, cpu=8, ram=16, storage=32, gpu=1, processing_time=3) + infrastructure.add_edge("edge-a", "edge-b", latency=5, bandwidth=10) + infrastructure.add_edge("edge-b", "edge-a", latency=7, bandwidth=12) + + application = Application("shop", include_default_assets=True, seed=7) + application.add_service(CounterService("gateway"), cpu=1, ram=2, storage=2, gpu=0) + application.add_service(CounterService("worker"), cpu=2, ram=2, storage=4, gpu=0) + application.add_edge("gateway", "worker", latency=6, bandwidth=4) + application.flows = [["gateway", "worker"]] + + ray.shutdown() + ray.init(address="local", include_dashboard=False, ignore_reinit_error=True) + ray_backend._backend = ray + ray_backend.init = lambda runtime_env: None + + config = SimulationConfig( + path=Path({str(output_dir)!r}), + report_backend="pandas", + report_format="csv", + remote=True, + ) + simulation = Simulation(infrastructure, config) + simulation.register(application, StaticStrategy({{"gateway": "edge-a", "worker": "edge-b"}})) + ray.get(simulation.simulator.audit.remote()) + ray.get(simulation.simulator.enact.remote()) + route = ray.get(simulation.simulator.route.remote("shop", "gateway", "worker")) + neighbors = ray.get(simulation.simulator.get_neighbors.remote("shop", "gateway")) + ray.get(simulation.simulator.cleanup.remote()) + + payload = {{ + "neighbors": neighbors, + "route_sender_node": route.sender_node_id if route is not None else None, + "route_recipient_node": route.recipient_node_id if route is not None else None, + "route_hops": route.hops if route is not None else None, + }} + print(json.dumps(payload)) + + ray.shutdown() + """ + ) + + result = run_remote_probe(repo_root, script) + + assert result["neighbors"] == ["worker"] + assert result["route_sender_node"] == "edge-a" + assert result["route_recipient_node"] == "edge-b" + assert len(result["route_hops"]) == 1 + assert result["route_hops"][0][:2] == ["edge-a", "edge-b"] diff --git a/tests/integration/simulation/__init__.py b/tests/integration/simulation/__init__.py new file mode 100644 index 0000000..69e47ee --- /dev/null +++ b/tests/integration/simulation/__init__.py @@ -0,0 +1 @@ +"""Local simulation integration tests.""" diff --git a/tests/integration/simulation/_helpers.py b/tests/integration/simulation/_helpers.py new file mode 100644 index 0000000..bf9a140 --- /dev/null +++ b/tests/integration/simulation/_helpers.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import time + + +def wait_until(predicate, timeout: float = 2.0): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + assert predicate() diff --git a/tests/integration/simulation/test_placement.py b/tests/integration/simulation/test_placement.py new file mode 100644 index 0000000..e73b780 --- /dev/null +++ b/tests/integration/simulation/test_placement.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from eclypse.placement.strategies import ( + BestFitStrategy, + StaticStrategy, +) +from eclypse.simulation._simulator.local import SimulationState +from eclypse.simulation.config import SimulationConfig +from eclypse.simulation.simulation import Simulation + +from ._helpers import wait_until + + +@pytest.mark.integration +def test_manual_simulation_runtime_uses_global_strategy_when_application_has_none( + tmp_path: Path, + sample_infrastructure, + sample_application, +): + config = SimulationConfig( + path=tmp_path / "global-strategy-simulation", + report_backend="pandas", + report_format="csv", + step_every_ms=None, + default_strategy=StaticStrategy({"gateway": "edge-a", "worker": "edge-b"}), + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application) + + simulation.start() + simulation.step() + + placement = simulation.simulator.placements[sample_application.id] + wait_until(lambda: placement.mapping == {"gateway": "edge-a", "worker": "edge-b"}) + + simulation.stop() + assert simulation.status is SimulationState.IDLE + + +@pytest.mark.integration +def test_manual_simulation_runtime_handles_partial_placement( + tmp_path: Path, + sample_infrastructure, + sample_application, +): + sample_application.nodes["worker"]["cpu"] = 100 + + config = SimulationConfig( + path=tmp_path / "partial-placement-simulation", + report_backend="pandas", + report_format="csv", + step_every_ms=None, + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, BestFitStrategy()) + + simulation.start() + simulation.step() + + placement = simulation.simulator.placements[sample_application.id] + wait_until(lambda: placement.mapping == {}) + + simulation.stop() + assert simulation.status is SimulationState.IDLE + + +@pytest.mark.integration +def test_manual_simulation_runtime_handles_no_placement( + tmp_path: Path, + sample_infrastructure, + sample_application, +): + sample_application.nodes["gateway"]["cpu"] = 100 + sample_application.nodes["worker"]["cpu"] = 100 + + config = SimulationConfig( + path=tmp_path / "no-placement-simulation", + report_backend="pandas", + report_format="csv", + step_every_ms=None, + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, BestFitStrategy()) + + simulation.start() + simulation.step() + + placement = simulation.simulator.placements[sample_application.id] + wait_until(lambda: placement.mapping == {}) + + simulation.stop() + assert simulation.status is SimulationState.IDLE diff --git a/tests/integration/simulation/test_recovery.py b/tests/integration/simulation/test_recovery.py new file mode 100644 index 0000000..d4b81cc --- /dev/null +++ b/tests/integration/simulation/test_recovery.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from eclypse import policies +from eclypse.placement.strategies import ( + BestFitStrategy, +) +from eclypse.simulation._simulator.local import SimulationState +from eclypse.simulation.config import SimulationConfig +from eclypse.simulation.simulation import Simulation + +from ._helpers import wait_until + + +@pytest.mark.integration +def test_manual_simulation_runtime_applies_replay_policies_across_steps( + tmp_path: Path, + sample_infrastructure, + sample_application, + static_strategy, +): + sample_application.update_policies = [ + policies.replay.replay_nodes( + [ + {"time": 0, "node_id": "gateway", "cpu": 2}, + {"time": 1, "node_id": "gateway", "cpu": 3}, + ], + value_columns=["cpu"], + ) + ] + sample_infrastructure.update_policies = [ + policies.replay.replay_edges( + [ + {"time": 0, "source": "edge-a", "target": "edge-b", "bandwidth": 8}, + {"time": 1, "source": "edge-a", "target": "edge-b", "bandwidth": 6}, + ], + value_columns=["bandwidth"], + ) + ] + + config = SimulationConfig( + path=tmp_path / "replay-simulation", + report_backend="pandas", + report_format="csv", + step_every_ms=None, + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, static_strategy) + + simulation.start() + simulation.step() + simulation.step() + simulation.stop() + + assert simulation.status is SimulationState.IDLE + assert sample_application.nodes["gateway"]["cpu"] == 3 + assert sample_infrastructure.edges["edge-a", "edge-b"]["bandwidth"] == 6 + + +@pytest.mark.integration +def test_manual_simulation_runtime_resets_and_then_fails_placement_after_degradation( + tmp_path: Path, + sample_infrastructure, + sample_application, +): + sample_infrastructure.update_policies = [ + policies.degrade.reduce( + target=0, + epochs=1, + node_assets="cpu", + ) + ] + + config = SimulationConfig( + path=tmp_path / "placement-reset-simulation", + report_backend="pandas", + report_format="csv", + step_every_ms=None, + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, BestFitStrategy()) + + simulation.start() + simulation.step() + + placement = simulation.simulator.placements[sample_application.id] + wait_until( + lambda: ( + bool(placement.mapping) + and sample_infrastructure.nodes["edge-a"]["cpu"] == 0 + and sample_infrastructure.nodes["edge-b"]["cpu"] == 0 + ) + ) + + simulation.step() + wait_until(lambda: placement.mapping == {}) + + simulation.step() + wait_until(lambda: placement.mapping == {}) + simulation.stop() + assert simulation.status is SimulationState.IDLE + + +@pytest.mark.integration +def test_manual_simulation_runtime_resets_when_service_path_disappears( + tmp_path: Path, + sample_infrastructure, + sample_application, + static_strategy, +): + def remove_forward_path(graph): + if graph.has_edge("edge-a", "edge-b"): + graph.remove_edge("edge-a", "edge-b") + + sample_infrastructure.update_policies = [remove_forward_path] + + config = SimulationConfig( + path=tmp_path / "path-loss-simulation", + report_backend="pandas", + report_format="csv", + step_every_ms=None, + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, static_strategy) + + simulation.start() + simulation.step() + + placement = simulation.simulator.placements[sample_application.id] + wait_until( + lambda: ( + placement.mapping == {"gateway": "edge-a", "worker": "edge-b"} + and not sample_infrastructure.has_edge("edge-a", "edge-b") + ) + ) + + simulation.step() + wait_until(lambda: placement.mapping == {}) + + simulation.stop() + assert simulation.status is SimulationState.IDLE diff --git a/tests/integration/simulation/test_reports.py b/tests/integration/simulation/test_reports.py new file mode 100644 index 0000000..e796d97 --- /dev/null +++ b/tests/integration/simulation/test_reports.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import csv +from pathlib import Path + +import pytest + +from eclypse.report.report import Report +from eclypse.simulation._simulator.local import SimulationState +from eclypse.simulation.config import SimulationConfig +from eclypse.simulation.simulation import Simulation +from eclypse.workflow.event import ( + EventRole, + once_at, +) + + +@pytest.mark.integration +def test_manual_simulation_runtime_generates_reports_and_config( + tmp_path: Path, + sample_infrastructure, + sample_application, + static_strategy, +): + config = SimulationConfig( + path=tmp_path / "manual-simulation", + report_backend="pandas", + report_format="csv", + include_default_metrics=True, + step_every_ms=None, + max_steps=2, + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, static_strategy) + + simulation.start() + simulation.step() + simulation.step() + simulation.stop() + + report = simulation.report + service_rows = report.service() + simulation_rows = report.simulation() + + assert simulation.status is SimulationState.IDLE + assert (simulation.path / "config.json").exists() + assert (simulation.path / "csv" / "service.csv").exists() + assert (simulation.path / "csv" / "simulation.csv").exists() + assert "placement" in service_rows["callback_id"].tolist() + assert "required_cpu" in service_rows["callback_id"].tolist() + assert "step_number" in simulation_rows["callback_id"].tolist() + assert report.application().iloc[0]["application_id"] == sample_application.id + + +@pytest.mark.integration +def test_auto_simulation_runtime_writes_summary_and_gml_outputs( + tmp_path: Path, + sample_infrastructure, + sample_application, + static_strategy, +): + config = SimulationConfig( + path=tmp_path / "auto-simulation", + report_backend="pandas", + report_format="csv", + include_default_metrics=True, + step_every_ms="auto", + max_steps=2, + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, static_strategy) + + simulation.start() + simulation.wait(timeout=10) + + report = Report(simulation.path, backend="pandas", report_format="csv") + simulation_rows = report.simulation() + + assert simulation.status is SimulationState.IDLE + assert "seed" in simulation_rows["callback_id"].tolist() + assert "simulation_time" in simulation_rows["callback_id"].tolist() + assert (simulation.path / "gml" / "app_gml-shop.gml").exists() + assert (simulation.path / "gml" / "infr_gml-edge-cloud.gml").exists() + assert report.infrastructure().iloc[0]["callback_id"] == "alive_nodes" + + +@pytest.mark.integration +def test_wrapped_event_runtime_reports_custom_metric( + tmp_path: Path, + sample_infrastructure, + sample_application, + static_strategy, +): + @once_at( + sim_seconds=0, + event_type="simulation", + activates_on=["start", ("start", 1.0), ("start", [1])], + role=EventRole.METRIC, + report="csv", + verbose=True, + ) + def wrapped_runtime_metric(*_args): + return {"wrapped_value": 7} + + config = SimulationConfig( + path=tmp_path / "wrapped-event-simulation", + report_backend="pandas", + report_format="csv", + events=[wrapped_runtime_metric], + step_every_ms="auto", + max_steps=1, + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, static_strategy) + + simulation.start() + simulation.wait(timeout=10) + + simulation_csv = simulation.path / "csv" / "simulation.csv" + with simulation_csv.open(encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + + assert simulation.status is SimulationState.IDLE + assert simulation_csv.exists() + assert any(row["event_id"] == "start" for row in rows) + assert any(row["callback_id"] == "wrapped_runtime_metric" for row in rows) + + +@pytest.mark.integration +def test_auto_simulation_runtime_stops_after_event_failure( + tmp_path: Path, + sample_infrastructure, + sample_application, + static_strategy, +): + @once_at( + sim_seconds=0, + event_type="simulation", + activates_on=["start"], + verbose=True, + ) + def failing_runtime_event(*_args): + raise RuntimeError("boom") + + config = SimulationConfig( + path=tmp_path / "failing-event-simulation", + report_backend="pandas", + report_format="csv", + events=[failing_runtime_event], + step_every_ms="auto", + max_steps=5, + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, static_strategy) + + simulation.start() + simulation.wait(timeout=10) + + assert simulation.status is SimulationState.IDLE diff --git a/tests/integration/test_emulation_runtime.py b/tests/integration/test_emulation_runtime.py deleted file mode 100644 index 6ae1ed7..0000000 --- a/tests/integration/test_emulation_runtime.py +++ /dev/null @@ -1,237 +0,0 @@ -from __future__ import annotations - -import json -import os -import subprocess -import sys -import textwrap -from pathlib import Path -from typing import Any - -import pytest - -_RAY_PROBE_STATE: dict[str, str | None] = {"blocked_reason": None} - - -def _run_remote_probe( - repo_root: Path, script: str, timeout: int = 25 -) -> dict[str, Any]: - blocked_reason = _RAY_PROBE_STATE["blocked_reason"] - - if blocked_reason is not None: - pytest.skip(blocked_reason) - - env = os.environ.copy() - env["RAY_ENABLE_UV_RUN_RUNTIME_ENV"] = "0" - env["PYTHONPATH"] = str(repo_root) - - try: - completed = subprocess.run( - [sys.executable, "-c", script], - capture_output=True, - check=False, - cwd=repo_root, - env=env, - text=True, - timeout=timeout, - ) - except subprocess.TimeoutExpired as exc: - _RAY_PROBE_STATE["blocked_reason"] = f"Ray integration probe timed out: {exc}" - pytest.skip(_RAY_PROBE_STATE["blocked_reason"]) - - if completed.returncode != 0: - combined_output = f"{completed.stdout}\n{completed.stderr}" - blocked_markers = ( - "PermissionError", - "Operation not permitted", - "Timed out waiting for file", - "gcs_server_port_", - ) - if any(marker in combined_output for marker in blocked_markers): - _RAY_PROBE_STATE["blocked_reason"] = ( - f"Ray integration probe is not permitted here:\n{combined_output}" - ) - pytest.skip(_RAY_PROBE_STATE["blocked_reason"]) - pytest.fail( - "Ray integration probe failed.\n" - f"stdout:\n{completed.stdout}\n" - f"stderr:\n{completed.stderr}" - ) - - lines = [line for line in completed.stdout.splitlines() if line.strip()] - assert lines, "Expected JSON output from the Ray integration probe." - return json.loads(lines[-1]) - - -@pytest.mark.integration -@pytest.mark.emulation -def test_ray_emulation_runtime_generates_reports(tmp_path: Path): - repo_root = Path(__file__).resolve().parents[2] - output_dir = tmp_path / "ray-simulation" - script = textwrap.dedent( - f""" - from __future__ import annotations - - import json - import os - from pathlib import Path - - os.environ["RAY_ENABLE_UV_RUN_RUNTIME_ENV"] = "0" - os.environ["ECLYPSE_RND_SEED"] = "7" - - import ray - - from eclypse.graph import Application, Infrastructure - from eclypse.placement.strategies import StaticStrategy - from eclypse.remote import ray_backend - from eclypse.remote.service.service import Service - from eclypse.simulation.config import SimulationConfig - from eclypse.simulation.simulation import Simulation - - - class CounterService(Service): - def __init__(self, service_id: str): - super().__init__(service_id, store_step=True) - - async def step(self): - if self.step_count >= 2: - self._running = False - return self.step_count - - - infrastructure = Infrastructure("edge-cloud", include_default_assets=True, seed=7) - infrastructure.add_node("edge-a", availability=1, cpu=4, ram=8, storage=16, gpu=0, processing_time=2) - infrastructure.add_node("edge-b", availability=1, cpu=8, ram=16, storage=32, gpu=1, processing_time=3) - infrastructure.add_edge("edge-a", "edge-b", latency=5, bandwidth=10) - infrastructure.add_edge("edge-b", "edge-a", latency=7, bandwidth=12) - - application = Application("shop", include_default_assets=True, seed=7) - application.add_service(CounterService("gateway"), cpu=1, ram=2, storage=2, gpu=0) - application.add_service(CounterService("worker"), cpu=2, ram=2, storage=4, gpu=0) - application.add_edge("gateway", "worker", latency=6, bandwidth=4) - application.flows = [["gateway", "worker"]] - - ray.shutdown() - ray.init(address="local", include_dashboard=False, ignore_reinit_error=True) - ray_backend._backend = ray - ray_backend.init = lambda runtime_env: None - - config = SimulationConfig( - path=Path({str(output_dir)!r}), - report_backend="pandas", - report_format="csv", - include_default_metrics=True, - remote=True, - step_every_ms="auto", - max_steps=3, - ) - simulation = Simulation(infrastructure, config) - simulation.register(application, StaticStrategy({{"gateway": "edge-a", "worker": "edge-b"}})) - simulation.start() - simulation.wait(timeout=15) - - service_rows = simulation.report.service() - callback_ids = service_rows["callback_id"].tolist() - payload = {{ - "status": str(simulation.status), - "path_exists": simulation.path.exists(), - "csv_service_exists": (simulation.path / "csv" / "service.csv").exists(), - "step_result_callbacks": sum(callback_id == "step_result" for callback_id in callback_ids), - "service_count": len(service_rows), - }} - print(json.dumps(payload)) - - ray.shutdown() - """ - ) - - result = _run_remote_probe(repo_root, script) - - assert result["path_exists"] is True - assert result["csv_service_exists"] is True - assert result["service_count"] > 0 - assert result["step_result_callbacks"] > 0 - - -@pytest.mark.integration -@pytest.mark.emulation -def test_ray_emulation_runtime_resolves_routes_and_neighbors(tmp_path: Path): - repo_root = Path(__file__).resolve().parents[2] - output_dir = tmp_path / "ray-routing" - script = textwrap.dedent( - f""" - from __future__ import annotations - - import json - import os - from pathlib import Path - - os.environ["RAY_ENABLE_UV_RUN_RUNTIME_ENV"] = "0" - os.environ["ECLYPSE_RND_SEED"] = "7" - - import ray - - from eclypse.graph import Application, Infrastructure - from eclypse.placement.strategies import StaticStrategy - from eclypse.remote import ray_backend - from eclypse.remote.service.service import Service - from eclypse.simulation.config import SimulationConfig - from eclypse.simulation.simulation import Simulation - - - class CounterService(Service): - async def step(self): - self._running = False - return self.step_count - - - infrastructure = Infrastructure("edge-cloud", include_default_assets=True, seed=7) - infrastructure.add_node("edge-a", availability=1, cpu=4, ram=8, storage=16, gpu=0, processing_time=2) - infrastructure.add_node("edge-b", availability=1, cpu=8, ram=16, storage=32, gpu=1, processing_time=3) - infrastructure.add_edge("edge-a", "edge-b", latency=5, bandwidth=10) - infrastructure.add_edge("edge-b", "edge-a", latency=7, bandwidth=12) - - application = Application("shop", include_default_assets=True, seed=7) - application.add_service(CounterService("gateway"), cpu=1, ram=2, storage=2, gpu=0) - application.add_service(CounterService("worker"), cpu=2, ram=2, storage=4, gpu=0) - application.add_edge("gateway", "worker", latency=6, bandwidth=4) - application.flows = [["gateway", "worker"]] - - ray.shutdown() - ray.init(address="local", include_dashboard=False, ignore_reinit_error=True) - ray_backend._backend = ray - ray_backend.init = lambda runtime_env: None - - config = SimulationConfig( - path=Path({str(output_dir)!r}), - report_backend="pandas", - report_format="csv", - remote=True, - ) - simulation = Simulation(infrastructure, config) - simulation.register(application, StaticStrategy({{"gateway": "edge-a", "worker": "edge-b"}})) - ray.get(simulation.simulator.audit.remote()) - ray.get(simulation.simulator.enact.remote()) - route = ray.get(simulation.simulator.route.remote("shop", "gateway", "worker")) - neighbors = ray.get(simulation.simulator.get_neighbors.remote("shop", "gateway")) - ray.get(simulation.simulator.cleanup.remote()) - - payload = {{ - "neighbors": neighbors, - "route_sender_node": route.sender_node_id if route is not None else None, - "route_recipient_node": route.recipient_node_id if route is not None else None, - "route_hops": route.hops if route is not None else None, - }} - print(json.dumps(payload)) - - ray.shutdown() - """ - ) - - result = _run_remote_probe(repo_root, script) - - assert result["neighbors"] == ["worker"] - assert result["route_sender_node"] == "edge-a" - assert result["route_recipient_node"] == "edge-b" - assert len(result["route_hops"]) == 1 - assert result["route_hops"][0][:2] == ["edge-a", "edge-b"] diff --git a/tests/integration/test_simulation_runtime.py b/tests/integration/test_simulation_runtime.py deleted file mode 100644 index 379878d..0000000 --- a/tests/integration/test_simulation_runtime.py +++ /dev/null @@ -1,378 +0,0 @@ -from __future__ import annotations - -import csv -import time -from pathlib import Path - -import pytest - -from eclypse import policies -from eclypse.placement.strategies import ( - BestFitStrategy, - StaticStrategy, -) -from eclypse.report.report import Report -from eclypse.simulation._simulator.local import SimulationState -from eclypse.simulation.config import SimulationConfig -from eclypse.simulation.simulation import Simulation -from eclypse.workflow.event import ( - EventRole, - event, -) - - -def _wait_until(predicate, timeout: float = 2.0): - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if predicate(): - return - time.sleep(0.01) - assert predicate() - - -@pytest.mark.integration -def test_manual_simulation_runtime_generates_reports_and_config( - tmp_path: Path, - sample_infrastructure, - sample_application, - static_strategy, -): - config = SimulationConfig( - path=tmp_path / "manual-simulation", - report_backend="pandas", - report_format="csv", - include_default_metrics=True, - max_steps=2, - ) - simulation = Simulation(sample_infrastructure, config) - simulation.register(sample_application, static_strategy) - - simulation.start() - simulation.step() - simulation.step() - simulation.stop() - - report = simulation.report - service_rows = report.service() - simulation_rows = report.simulation() - - assert simulation.status is SimulationState.IDLE - assert (simulation.path / "config.json").exists() - assert (simulation.path / "csv" / "service.csv").exists() - assert (simulation.path / "csv" / "simulation.csv").exists() - assert "placement" in service_rows["callback_id"].tolist() - assert "required_cpu" in service_rows["callback_id"].tolist() - assert "step_number" in simulation_rows["callback_id"].tolist() - assert report.application().iloc[0]["application_id"] == sample_application.id - - -@pytest.mark.integration -def test_auto_simulation_runtime_writes_summary_and_gml_outputs( - tmp_path: Path, - sample_infrastructure, - sample_application, - static_strategy, -): - config = SimulationConfig( - path=tmp_path / "auto-simulation", - report_backend="pandas", - report_format="csv", - include_default_metrics=True, - step_every_ms="auto", - max_steps=2, - ) - simulation = Simulation(sample_infrastructure, config) - simulation.register(sample_application, static_strategy) - - simulation.start() - simulation.wait(timeout=10) - - report = Report(simulation.path, backend="pandas", report_format="csv") - simulation_rows = report.simulation() - - assert simulation.status is SimulationState.IDLE - assert "seed" in simulation_rows["callback_id"].tolist() - assert "simulation_time" in simulation_rows["callback_id"].tolist() - assert (simulation.path / "gml" / "app_gml-shop.gml").exists() - assert (simulation.path / "gml" / "infr_gml-edge-cloud.gml").exists() - assert report.infrastructure().iloc[0]["callback_id"] == "alive_nodes" - - -@pytest.mark.integration -def test_wrapped_event_runtime_reports_custom_metric( - tmp_path: Path, - sample_infrastructure, - sample_application, - static_strategy, -): - @event( - event_type="simulation", - activates_on=["start", ("start", 1.0), ("start", [1])], - role=EventRole.METRIC, - report="csv", - verbose=True, - ) - def wrapped_runtime_metric(*_args): - return {"wrapped_value": 7} - - config = SimulationConfig( - path=tmp_path / "wrapped-event-simulation", - report_backend="pandas", - report_format="csv", - events=[wrapped_runtime_metric], - step_every_ms="auto", - max_steps=1, - ) - simulation = Simulation(sample_infrastructure, config) - simulation.register(sample_application, static_strategy) - - simulation.start() - simulation.wait(timeout=10) - - simulation_csv = simulation.path / "csv" / "simulation.csv" - with simulation_csv.open(encoding="utf-8", newline="") as handle: - rows = list(csv.DictReader(handle)) - - assert simulation.status is SimulationState.IDLE - assert simulation_csv.exists() - assert any(row["event_id"] == "start" for row in rows) - assert any(row["callback_id"] == "wrapped_runtime_metric" for row in rows) - - -@pytest.mark.integration -def test_manual_simulation_runtime_applies_replay_policies_across_steps( - tmp_path: Path, - sample_infrastructure, - sample_application, - static_strategy, -): - sample_application.update_policies = [ - policies.replay.replay_nodes( - [ - {"time": 0, "node_id": "gateway", "cpu": 2}, - {"time": 1, "node_id": "gateway", "cpu": 3}, - ], - value_columns=["cpu"], - ) - ] - sample_infrastructure.update_policies = [ - policies.replay.replay_edges( - [ - {"time": 0, "source": "edge-a", "target": "edge-b", "bandwidth": 8}, - {"time": 1, "source": "edge-a", "target": "edge-b", "bandwidth": 6}, - ], - value_columns=["bandwidth"], - ) - ] - - config = SimulationConfig( - path=tmp_path / "replay-simulation", - report_backend="pandas", - report_format="csv", - ) - simulation = Simulation(sample_infrastructure, config) - simulation.register(sample_application, static_strategy) - - simulation.start() - simulation.step() - simulation.step() - simulation.stop() - - assert simulation.status is SimulationState.IDLE - assert sample_application.nodes["gateway"]["cpu"] == 3 - assert sample_infrastructure.edges["edge-a", "edge-b"]["bandwidth"] == 6 - - -@pytest.mark.integration -def test_manual_simulation_runtime_resets_and_then_fails_placement_after_degradation( - tmp_path: Path, - sample_infrastructure, - sample_application, -): - sample_infrastructure.update_policies = [ - policies.degrade.reduce( - target=0, - epochs=1, - node_assets="cpu", - ) - ] - - config = SimulationConfig( - path=tmp_path / "placement-reset-simulation", - report_backend="pandas", - report_format="csv", - ) - simulation = Simulation(sample_infrastructure, config) - simulation.register(sample_application, BestFitStrategy()) - - simulation.start() - simulation.step() - - placement = simulation.simulator.placements[sample_application.id] - _wait_until( - lambda: ( - bool(placement.mapping) - and sample_infrastructure.nodes["edge-a"]["cpu"] == 0 - and sample_infrastructure.nodes["edge-b"]["cpu"] == 0 - ) - ) - - simulation.step() - _wait_until(lambda: placement.mapping == {}) - - simulation.step() - _wait_until(lambda: placement.mapping == {}) - simulation.stop() - assert simulation.status is SimulationState.IDLE - - -@pytest.mark.integration -def test_manual_simulation_runtime_uses_global_strategy_when_application_has_none( - tmp_path: Path, - sample_infrastructure, - sample_application, -): - sample_infrastructure.strategy = StaticStrategy( - {"gateway": "edge-a", "worker": "edge-b"} - ) - - config = SimulationConfig( - path=tmp_path / "global-strategy-simulation", - report_backend="pandas", - report_format="csv", - ) - simulation = Simulation(sample_infrastructure, config) - simulation.register(sample_application) - - simulation.start() - simulation.step() - - placement = simulation.simulator.placements[sample_application.id] - _wait_until(lambda: placement.mapping == {"gateway": "edge-a", "worker": "edge-b"}) - - simulation.stop() - assert simulation.status is SimulationState.IDLE - - -@pytest.mark.integration -def test_manual_simulation_runtime_handles_partial_placement( - tmp_path: Path, - sample_infrastructure, - sample_application, -): - sample_application.nodes["worker"]["cpu"] = 100 - - config = SimulationConfig( - path=tmp_path / "partial-placement-simulation", - report_backend="pandas", - report_format="csv", - ) - simulation = Simulation(sample_infrastructure, config) - simulation.register(sample_application, BestFitStrategy()) - - simulation.start() - simulation.step() - - placement = simulation.simulator.placements[sample_application.id] - _wait_until(lambda: placement.mapping == {}) - - simulation.stop() - assert simulation.status is SimulationState.IDLE - - -@pytest.mark.integration -def test_manual_simulation_runtime_handles_no_placement( - tmp_path: Path, - sample_infrastructure, - sample_application, -): - sample_application.nodes["gateway"]["cpu"] = 100 - sample_application.nodes["worker"]["cpu"] = 100 - - config = SimulationConfig( - path=tmp_path / "no-placement-simulation", - report_backend="pandas", - report_format="csv", - ) - simulation = Simulation(sample_infrastructure, config) - simulation.register(sample_application, BestFitStrategy()) - - simulation.start() - simulation.step() - - placement = simulation.simulator.placements[sample_application.id] - _wait_until(lambda: placement.mapping == {}) - - simulation.stop() - assert simulation.status is SimulationState.IDLE - - -@pytest.mark.integration -def test_manual_simulation_runtime_resets_when_service_path_disappears( - tmp_path: Path, - sample_infrastructure, - sample_application, - static_strategy, -): - def remove_forward_path(graph): - if graph.has_edge("edge-a", "edge-b"): - graph.remove_edge("edge-a", "edge-b") - - sample_infrastructure.update_policies = [remove_forward_path] - - config = SimulationConfig( - path=tmp_path / "path-loss-simulation", - report_backend="pandas", - report_format="csv", - ) - simulation = Simulation(sample_infrastructure, config) - simulation.register(sample_application, static_strategy) - - simulation.start() - simulation.step() - - placement = simulation.simulator.placements[sample_application.id] - _wait_until( - lambda: ( - placement.mapping == {"gateway": "edge-a", "worker": "edge-b"} - and not sample_infrastructure.has_edge("edge-a", "edge-b") - ) - ) - - simulation.step() - _wait_until(lambda: placement.mapping == {}) - - simulation.stop() - assert simulation.status is SimulationState.IDLE - - -@pytest.mark.integration -def test_auto_simulation_runtime_stops_after_event_failure( - tmp_path: Path, - sample_infrastructure, - sample_application, - static_strategy, -): - @event( - event_type="simulation", - activates_on=["start"], - verbose=True, - ) - def failing_runtime_event(*_args): - raise RuntimeError("boom") - - config = SimulationConfig( - path=tmp_path / "failing-event-simulation", - report_backend="pandas", - report_format="csv", - events=[failing_runtime_event], - step_every_ms="auto", - max_steps=5, - ) - simulation = Simulation(sample_infrastructure, config) - simulation.register(sample_application, static_strategy) - - simulation.start() - simulation.wait(timeout=10) - - assert simulation.status is SimulationState.IDLE diff --git a/tests/extras/test_config_and_optional_imports.py b/tests/optional/test_config_and_optional_imports.py similarity index 86% rename from tests/extras/test_config_and_optional_imports.py rename to tests/optional/test_config_and_optional_imports.py index 6cd4fd2..514fb72 100644 --- a/tests/extras/test_config_and_optional_imports.py +++ b/tests/optional/test_config_and_optional_imports.py @@ -37,8 +37,8 @@ def __call__(self, *_args, **_kwargs): return {"value": 1} -@pytest.mark.extras -def test_config_import_targets_are_available_for_installed_extras(): +@pytest.mark.optional +def test_config_import_targets_are_available_for_installed_optional_dependencies(): _require_installed("pandas") _require_installed("polars") _require_installed("ray") @@ -46,12 +46,12 @@ def test_config_import_targets_are_available_for_installed_extras(): _require_module("pandas") _require_module("polars") - _require_module("ray", extras_name="remote") - _require_module("tensorboardX", extras_name="tboard") + _require_module("ray") + _require_module("tensorboardX") -@pytest.mark.extras -def test_simulation_config_accepts_optional_backend_extras(tmp_path: Path): +@pytest.mark.optional +def test_simulation_config_accepts_optional_backend_dependencies(tmp_path: Path): _require_installed("pandas") _require_installed("polars") @@ -73,14 +73,14 @@ def test_simulation_config_accepts_optional_backend_extras(tmp_path: Path): assert lazy_config.report_backend == "polars_lazy" -@pytest.mark.extras +@pytest.mark.optional def test_simulation_config_resolves_optional_reporters_and_remote(tmp_path: Path): _require_installed("ray") _require_installed("polars") _require_installed("tensorboardX") config = SimulationConfig( - path=tmp_path / "extra-reporters", + path=tmp_path / "optional-reporters", report_backend="pandas", remote=True, events=[ diff --git a/tests/extras/test_ray_and_reporter_extras.py b/tests/optional/test_ray_and_reporter_optional.py similarity index 98% rename from tests/extras/test_ray_and_reporter_extras.py rename to tests/optional/test_ray_and_reporter_optional.py index 67dd53f..34e492b 100644 --- a/tests/extras/test_ray_and_reporter_extras.py +++ b/tests/optional/test_ray_and_reporter_optional.py @@ -27,7 +27,7 @@ def _require_installed(module_name: str): pytest.skip(f"Optional dependency {module_name!r} is not installed.") -@pytest.mark.extras +@pytest.mark.optional @pytest.mark.asyncio async def test_optional_reporters_initialise_with_real_dependencies(tmp_path: Path): _require_installed("polars") @@ -73,7 +73,7 @@ async def test_optional_reporters_initialise_with_real_dependencies(tmp_path: Pa await tensorboard_reporter.close() -@pytest.mark.extras +@pytest.mark.optional def test_ray_interface_smoke_round_trip(): _require_installed("ray") diff --git a/tests/unit/builders/application/_service_test_helpers.py b/tests/unit/builders/application/_service_test_helpers.py new file mode 100644 index 0000000..92aebe4 --- /dev/null +++ b/tests/unit/builders/application/_service_test_helpers.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from types import SimpleNamespace + + +class FakeRestResponse: + def __init__(self, body): + self.body = body + self.data = body + + +class AwaitableResult: + def __init__(self, result): + self.result = result + + def __await__(self): + async def _resolve(): + return self.result + + return _resolve().__await__() + + +class FakeRESTInterface: + def __init__(self, handlers): + self.handlers = handlers + self.calls: list[tuple[str, str, dict[str, object]]] = [] + + async def get(self, url: str, **kwargs): + self.calls.append(("GET", url, kwargs)) + handler = self.handlers[("GET", url)] + result = handler(**kwargs) if callable(handler) else handler + return FakeRestResponse(result) + + async def post(self, url: str, **kwargs): + self.calls.append(("POST", url, kwargs)) + handler = self.handlers[("POST", url)] + result = handler(**kwargs) if callable(handler) else handler + return FakeRestResponse(result) + + +class FakeMPIInterface: + def __init__(self, messages): + self.messages = list(messages) + self.sent: list[tuple[str, dict[str, object]]] = [] + + async def recv(self): + return self.messages.pop(0) + + def send(self, recipient_id: str, body: dict[str, object]): + self.sent.append((recipient_id, body)) + return AwaitableResult((recipient_id, body)) + + +def attach_service_logger(service): + service.attach_node( + SimpleNamespace( + _logger=SimpleNamespace( + bind=lambda **_: SimpleNamespace(info=lambda *_args: None) + ) + ) + ) + return service + + +def set_mpi(service, messages): + service._comm = FakeMPIInterface(messages) + return service._comm diff --git a/tests/unit/builders/application/test_anomaly_detection.py b/tests/unit/builders/application/test_anomaly_detection.py new file mode 100644 index 0000000..b91aae1 --- /dev/null +++ b/tests/unit/builders/application/test_anomaly_detection.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application import get_anomaly_detection +from eclypse.remote.service.service import Service + + +def test_anomaly_detection_builder(): + plain_app = get_anomaly_detection(include_default_assets=True) + mpi_app = get_anomaly_detection( + include_default_assets=True, + communication_interface="mpi", + ) + rest_app = get_anomaly_detection( + include_default_assets=True, + communication_interface="rest", + ) + + assert plain_app.has_service_implementations is False + assert mpi_app.has_service_implementations is True + assert rest_app.has_service_implementations is True + assert all(isinstance(service, Service) for service in mpi_app.services.values()) + assert all(isinstance(service, Service) for service in rest_app.services.values()) + assert plain_app.has_edge("SensorService", "FeatureService") + assert len(plain_app.flows) == 1 + + +def test_anomaly_detection_rejects_unknown_interfaces(): + with pytest.raises(ValueError, match="Unknown communication interface"): + get_anomaly_detection(communication_interface="grpc") # type: ignore[arg-type] diff --git a/tests/unit/builders/application/test_anomaly_detection_services.py b/tests/unit/builders/application/test_anomaly_detection_services.py new file mode 100644 index 0000000..ab65a84 --- /dev/null +++ b/tests/unit/builders/application/test_anomaly_detection_services.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application.anomaly_detection import mpi_services as anomaly_mpi +from eclypse.builders.application.anomaly_detection import rest_services as anomaly_rest +from tests.unit.builders.application._service_test_helpers import ( + FakeRESTInterface, + attach_service_logger, + set_mpi, +) + + +@pytest.mark.asyncio +async def test_anomaly_detection_services(monkeypatch): + feature = attach_service_logger(anomaly_rest.FeatureService("FeatureService")) + inference = attach_service_logger(anomaly_rest.InferenceService("InferenceService")) + alert = attach_service_logger(anomaly_rest.AlertService("AlertService")) + sensor = attach_service_logger(anomaly_rest.SensorService("SensorService")) + + code, body = feature.features(1, [0.8, 1.2, 4.5]) + assert code == 200 + assert body["features"]["max"] == 4.5 + + code, body = inference.score(1, {"max": 4.5, "mean": 2.1667}) + assert code == 200 + assert body["score"] == pytest.approx(2.08) + + code, body = alert.alert(1, 2.08) + assert code == 200 + assert body["status"] == "normal" + + code, body = alert.alert(2, 2.7) + assert body["status"] == "alert" + + sensor_rest = FakeRESTInterface( + { + ("POST", "FeatureService/features"): { + "features": {"max": 4.5, "mean": 2.1667} + }, + ("POST", "InferenceService/score"): {"score": 2.08}, + ("POST", "AlertService/alert"): {"status": "normal", "score": 2.08}, + } + ) + monkeypatch.setattr(type(sensor), "rest", property(lambda self: sensor_rest)) + response = await sensor.step() + assert response.body["status"] == "normal" + + mpi_feature = attach_service_logger(anomaly_mpi.FeatureService("FeatureService")) + mpi_inference = attach_service_logger( + anomaly_mpi.InferenceService("InferenceService") + ) + mpi_alert = attach_service_logger(anomaly_mpi.AlertService("AlertService")) + mpi_sensor = attach_service_logger(anomaly_mpi.SensorService("SensorService")) + + feature_comm = set_mpi( + mpi_feature, + [ + { + "sender_id": "SensorService", + "request_type": "extract_features", + "window_id": 1, + "samples": [0.8, 1.2, 4.5], + } + ], + ) + await mpi_feature.step() + assert feature_comm.sent[0][1]["features"]["max"] == 4.5 + + inference_comm = set_mpi( + mpi_inference, + [ + { + "sender_id": "FeatureService", + "request_type": "score_window", + "window_id": 1, + "features": {"max": 4.5, "mean": 2.1667}, + } + ], + ) + await mpi_inference.step() + assert inference_comm.sent[0][1]["score"] == pytest.approx(2.08) + + alert_comm = set_mpi( + mpi_alert, + [ + { + "sender_id": "InferenceService", + "request_type": "emit_alert", + "window_id": 1, + "score": 2.7, + } + ], + ) + await mpi_alert.step() + assert alert_comm.sent[0][1]["status"] == "alert" + + sensor_comm = set_mpi(mpi_sensor, [{"status": "normal", "score": 2.08}]) + response = await mpi_sensor.step() + assert sensor_comm.sent[0][0] == "FeatureService" + assert response["status"] == "normal" diff --git a/tests/unit/builders/application/test_consistency.py b/tests/unit/builders/application/test_consistency.py new file mode 100644 index 0000000..b8248d1 --- /dev/null +++ b/tests/unit/builders/application/test_consistency.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import ast +from itertools import pairwise +from pathlib import Path + +import pytest + +from eclypse.builders.application import ( + get_anomaly_detection, + get_crud_api, + get_hotel_reservation, + get_keyword_spotting, + get_media_service, + get_social_network, + get_sock_shop, + get_thumbnailer, + get_video_analytics_serving, +) + +_FLOW_CONSISTENT_BUILDERS = [ + get_video_analytics_serving, + get_hotel_reservation, + get_social_network, + get_media_service, + get_crud_api, + get_keyword_spotting, + get_anomaly_detection, + get_thumbnailer, +] + +_CALL_CONSISTENT_BUILDERS = [ + *_FLOW_CONSISTENT_BUILDERS, + get_sock_shop, +] + +_REST_METHODS = {"delete", "get", "patch", "post", "put"} + + +class _RuntimeCallCollector(ast.NodeVisitor): + def __init__(self) -> None: + self.calls: set[tuple[str, str]] = set() + self._class_name: str | None = None + + def visit_ClassDef(self, node: ast.ClassDef) -> None: # noqa: N802 + previous_class_name = self._class_name + self._class_name = node.name + self.generic_visit(node) + self._class_name = previous_class_name + + def visit_Call(self, node: ast.Call) -> None: # noqa: N802 + if self._class_name is not None: + mpi_target = _extract_mpi_send_target(node) + if mpi_target is not None: + self.calls.add((self._class_name, mpi_target)) + + rest_target = _extract_rest_target(node) + if rest_target is not None: + self.calls.add((self._class_name, rest_target)) + + self.generic_visit(node) + + def visit_Return(self, node: ast.Return) -> None: # noqa: N802 + if self._class_name is not None: + target = _extract_return_target(node.value) + if target is not None: + self.calls.add((self._class_name, target)) + + self.generic_visit(node) + + +def _extract_mpi_send_target(node: ast.Call) -> str | None: + if _is_self_method_call(node, namespace="mpi", methods={"send"}): + return _first_string_argument(node) + return None + + +def _extract_rest_target(node: ast.Call) -> str | None: + if not _is_self_method_call(node, namespace="rest", methods=_REST_METHODS): + return None + + endpoint = _first_string_argument(node) + if endpoint is None: + return None + return endpoint.split("/", maxsplit=1)[0] + + +def _extract_return_target(node: ast.AST | None) -> str | None: + if not isinstance(node, ast.Tuple): + return None + if not node.elts: + return None + first_element = node.elts[0] + if isinstance(first_element, ast.Constant) and isinstance( + first_element.value, + str, + ): + return first_element.value + return None + + +def _first_string_argument(node: ast.Call) -> str | None: + if not node.args: + return None + first_argument = node.args[0] + if isinstance(first_argument, ast.Constant) and isinstance( + first_argument.value, + str, + ): + return first_argument.value + return None + + +def _is_self_method_call( + node: ast.Call, + namespace: str, + methods: set[str], +) -> bool: + if not isinstance(node.func, ast.Attribute): + return False + if node.func.attr not in methods: + return False + if not isinstance(node.func.value, ast.Attribute): + return False + if node.func.value.attr != namespace: + return False + if not isinstance(node.func.value.value, ast.Name): + return False + return node.func.value.value.id == "self" + + +def _collect_runtime_calls(builder) -> set[tuple[str, str]]: + package_path = Path(builder.__globals__["__file__"]).resolve().parent + calls: set[tuple[str, str]] = set() + for services_directory in ("mpi_services", "rest_services"): + for file_path in sorted((package_path / services_directory).glob("*.py")): + if file_path.name == "__init__.py": + continue + collector = _RuntimeCallCollector() + collector.visit(ast.parse(file_path.read_text(), filename=str(file_path))) + calls.update(collector.calls) + return calls + + +@pytest.mark.parametrize("builder", _FLOW_CONSISTENT_BUILDERS) +def test_flows_match_topology(builder): + app = builder() + + for flow in app.flows: + for source, target in pairwise(flow): + assert app.has_edge(source, target), (builder.__name__, source, target) + + +@pytest.mark.parametrize("builder", _CALL_CONSISTENT_BUILDERS) +def test_calls_match_topology(builder): + app = builder() + + for source, target in sorted(_collect_runtime_calls(builder)): + assert app.has_edge(source, target), (builder.__name__, source, target) diff --git a/tests/unit/builders/application/test_crud_api.py b/tests/unit/builders/application/test_crud_api.py new file mode 100644 index 0000000..4603144 --- /dev/null +++ b/tests/unit/builders/application/test_crud_api.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application import get_crud_api +from eclypse.remote.service.service import Service + + +def test_crud_api_builder(): + plain_app = get_crud_api(include_default_assets=True) + mpi_app = get_crud_api( + include_default_assets=True, + communication_interface="mpi", + ) + rest_app = get_crud_api( + include_default_assets=True, + communication_interface="rest", + ) + + assert plain_app.has_service_implementations is False + assert mpi_app.has_service_implementations is True + assert rest_app.has_service_implementations is True + assert all(isinstance(service, Service) for service in mpi_app.services.values()) + assert all(isinstance(service, Service) for service in rest_app.services.values()) + assert plain_app.has_edge("GatewayService", "AuthService") + assert len(plain_app.flows) == 2 + + +def test_crud_api_rejects_unknown_interfaces(): + with pytest.raises(ValueError, match="Unknown communication interface"): + get_crud_api(communication_interface="grpc") # type: ignore[arg-type] diff --git a/tests/unit/builders/application/test_crud_api_services.py b/tests/unit/builders/application/test_crud_api_services.py new file mode 100644 index 0000000..a600a60 --- /dev/null +++ b/tests/unit/builders/application/test_crud_api_services.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application.crud_api import mpi_services as crud_mpi +from eclypse.builders.application.crud_api import rest_services as crud_rest +from eclypse.remote.communication.rest.codes import HTTPStatusCode +from tests.unit.builders.application._service_test_helpers import ( + FakeRESTInterface, + attach_service_logger, + set_mpi, +) + + +@pytest.mark.asyncio +async def test_crud_api_services(monkeypatch): + auth = attach_service_logger(crud_rest.AuthService("AuthService")) + audit = attach_service_logger(crud_rest.AuditService("AuditService")) + item = attach_service_logger(crud_rest.ItemService("ItemService")) + gateway = attach_service_logger(crud_rest.GatewayService("GatewayService")) + + code, body = auth.auth("demo-key") + assert code == 200 + assert body["token"] == "token:demo-key" + + code, body = audit.record_event("token:demo", "item-1", "create") + assert code == 200 + assert body["message"] == "token:demo:create:item-1" + + item_rest = FakeRESTInterface( + {("POST", "AuditService/events"): {"status": "recorded"}} + ) + monkeypatch.setattr(type(item), "rest", property(lambda self: item_rest)) + code, body = await item.create_item( + token="token:demo", + item={"id": "item-1", "name": "demo", "status": "active"}, + ) + assert code is HTTPStatusCode.CREATED + assert body["items"][0]["id"] == "item-1" + + gateway_rest = FakeRESTInterface( + { + ("POST", "AuthService/auth"): {"token": "token:demo-key"}, + ("POST", "ItemService/items"): { + "status": "recorded", + "items": [{"id": "item-1"}], + }, + } + ) + monkeypatch.setattr(type(gateway), "rest", property(lambda self: gateway_rest)) + response = await gateway.step() + assert response.body["status"] == "recorded" + + mpi_auth = attach_service_logger(crud_mpi.AuthService("AuthService")) + mpi_audit = attach_service_logger(crud_mpi.AuditService("AuditService")) + mpi_item = attach_service_logger(crud_mpi.ItemService("ItemService")) + mpi_gateway = attach_service_logger(crud_mpi.GatewayService("GatewayService")) + + auth_comm = set_mpi( + mpi_auth, + [ + { + "sender_id": "GatewayService", + "request_type": "authenticate", + "api_key": "demo-key", + } + ], + ) + await mpi_auth.step() + assert auth_comm.sent[0][1]["token"] == "token:demo-key" + + audit_comm = set_mpi( + mpi_audit, + [ + { + "sender_id": "ItemService", + "request_type": "record_event", + "item_id": "item-1", + "action": "create", + } + ], + ) + await mpi_audit.step() + assert audit_comm.sent[0][1]["status"] == "recorded" + + item_comm = set_mpi( + mpi_item, + [ + { + "sender_id": "GatewayService", + "request_type": "create_item", + "token": "token:demo", + "item": {"id": "item-1", "name": "demo", "status": "active"}, + }, + {"sender_id": "AuditService", "status": "recorded"}, + ], + ) + await mpi_item.step() + assert item_comm.sent[0][0] == "AuditService" + assert item_comm.sent[1][0] == "GatewayService" + assert item_comm.sent[1][1]["items"][0]["id"] == "item-1" + + gateway_comm = set_mpi( + mpi_gateway, + [ + {"token": "token:demo-key"}, + {"status": "recorded", "items": [{"id": "item-1"}]}, + ], + ) + response = await mpi_gateway.step() + assert gateway_comm.sent[0][0] == "AuthService" + assert gateway_comm.sent[1][0] == "ItemService" + assert response["status"] == "recorded" diff --git a/tests/unit/builders/application/test_hotel_reservation.py b/tests/unit/builders/application/test_hotel_reservation.py new file mode 100644 index 0000000..b456f92 --- /dev/null +++ b/tests/unit/builders/application/test_hotel_reservation.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application import get_hotel_reservation +from eclypse.remote.service.service import Service + + +def test_hotel_reservation_builder(): + plain_app = get_hotel_reservation(include_default_assets=True) + mpi_app = get_hotel_reservation( + include_default_assets=True, + communication_interface="mpi", + ) + rest_app = get_hotel_reservation( + include_default_assets=True, + communication_interface="rest", + ) + + assert plain_app.has_service_implementations is False + assert mpi_app.has_service_implementations is True + assert rest_app.has_service_implementations is True + assert all(isinstance(service, Service) for service in mpi_app.services.values()) + assert all(isinstance(service, Service) for service in rest_app.services.values()) + assert plain_app.has_edge("FrontendService", "SearchService") + assert len(plain_app.flows) == 3 + + +def test_hotel_reservation_rejects_unknown_interfaces(): + with pytest.raises(ValueError, match="Unknown communication interface"): + get_hotel_reservation(communication_interface="grpc") # type: ignore[arg-type] diff --git a/tests/unit/builders/application/test_hotel_reservation_services.py b/tests/unit/builders/application/test_hotel_reservation_services.py new file mode 100644 index 0000000..6d9317d --- /dev/null +++ b/tests/unit/builders/application/test_hotel_reservation_services.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application.deathstarbench.hotel_reservation import ( + mpi_services as hotel_mpi, +) +from eclypse.builders.application.deathstarbench.hotel_reservation import ( + rest_services as hotel_rest, +) +from eclypse.remote.communication.rest.codes import HTTPStatusCode +from tests.unit.builders.application._service_test_helpers import ( + FakeRESTInterface, + attach_service_logger, + set_mpi, +) + + +@pytest.mark.asyncio +async def test_hotel_reservation_services(monkeypatch): + search = attach_service_logger(hotel_rest.SearchService("SearchService")) + profile = attach_service_logger(hotel_rest.ProfileService("ProfileService")) + payment = attach_service_logger(hotel_rest.PaymentService("PaymentService")) + reservation = attach_service_logger( + hotel_rest.ReservationService("ReservationService") + ) + frontend = attach_service_logger(hotel_rest.FrontendService("FrontendService")) + + code, body = search.search("Pisa", 2) + assert code == 200 + assert body["hotels"][0]["id"] == "h1" + + code, body = profile.profile(101) + assert code == 200 + assert body["user"]["name"] == "Ada Lovelace" + + monkeypatch.setattr( + "eclypse.builders.application.deathstarbench.hotel_reservation.rest_services.payment.rnd.randint", + lambda _low, _high: 1234, + ) + code, body = payment.pay("rsv-2001", 129.0) + assert code == 200 + assert body["transaction_id"] == "txn-1234" + + reservation_rest = FakeRESTInterface( + { + ("POST", "PaymentService/pay"): { + "transaction_id": "txn-1234", + "status": "confirmed", + } + } + ) + monkeypatch.setattr( + type(reservation), "rest", property(lambda self: reservation_rest) + ) + code, body = await reservation.reserve( + hotel={"name": "Arno View", "price": 129.0}, + user={"name": "Ada Lovelace"}, + ) + assert code is HTTPStatusCode.CREATED + assert body["transaction_id"] == "txn-1234" + + frontend_rest = FakeRESTInterface( + { + ("GET", "SearchService/search"): { + "hotels": [{"id": "h1", "name": "Arno View", "price": 129.0}] + }, + ("GET", "ProfileService/profile"): { + "user": {"user_id": 101, "name": "Ada Lovelace"} + }, + ("POST", "ReservationService/reserve"): { + "reservation_id": "rsv-2001", + "status": "confirmed", + }, + } + ) + monkeypatch.setattr(type(frontend), "rest", property(lambda self: frontend_rest)) + response = await frontend.step() + assert response.body["reservation_id"] == "rsv-2001" + + mpi_search = attach_service_logger(hotel_mpi.SearchService("SearchService")) + mpi_profile = attach_service_logger(hotel_mpi.ProfileService("ProfileService")) + mpi_payment = attach_service_logger(hotel_mpi.PaymentService("PaymentService")) + mpi_reservation = attach_service_logger( + hotel_mpi.ReservationService("ReservationService") + ) + mpi_frontend = attach_service_logger(hotel_mpi.FrontendService("FrontendService")) + + search_comm = set_mpi( + mpi_search, + [ + { + "sender_id": "FrontendService", + "request_type": "search_hotels", + "city": "Pisa", + "nights": 2, + } + ], + ) + await mpi_search.step() + assert search_comm.sent[0][0] == "FrontendService" + assert search_comm.sent[0][1]["response_type"] == "search_results" + + profile_comm = set_mpi( + mpi_profile, + [ + { + "sender_id": "FrontendService", + "request_type": "get_profile", + "user_id": 101, + } + ], + ) + await mpi_profile.step() + assert profile_comm.sent[0][1]["user"]["name"] == "Ada Lovelace" + + monkeypatch.setattr( + "eclypse.builders.application.deathstarbench.hotel_reservation.mpi_services.payment.rnd.randint", + lambda _low, _high: 1234, + ) + payment_comm = set_mpi( + mpi_payment, + [ + { + "sender_id": "ReservationService", + "request_type": "charge_card", + "reservation_id": "rsv-2001", + "amount": 129.0, + } + ], + ) + await mpi_payment.step() + assert payment_comm.sent == [ + ( + "ReservationService", + { + "response_type": "payment_response", + "reservation_id": "rsv-2001", + "transaction_id": "txn-1234", + "status": "confirmed", + }, + ) + ] + + reservation_comm = set_mpi( + mpi_reservation, + [ + { + "sender_id": "FrontendService", + "request_type": "create_reservation", + "hotel": {"name": "Arno View", "price": 129.0}, + "user": {"name": "Ada Lovelace"}, + }, + { + "sender_id": "PaymentService", + "transaction_id": "txn-1234", + "status": "confirmed", + }, + ], + ) + await mpi_reservation.step() + assert reservation_comm.sent[0][0] == "PaymentService" + assert reservation_comm.sent[1][0] == "FrontendService" + + frontend_comm = set_mpi( + mpi_frontend, + [ + {"hotels": [{"id": "h1", "name": "Arno View", "price": 129.0}]}, + {"user": {"user_id": 101, "name": "Ada Lovelace"}}, + {"reservation_id": "rsv-2001", "status": "confirmed"}, + ], + ) + frontend_response = await mpi_frontend.step() + assert frontend_comm.sent[0][0] == "SearchService" + assert frontend_comm.sent[1][0] == "ProfileService" + assert frontend_comm.sent[2][0] == "ReservationService" + assert frontend_response["reservation_id"] == "rsv-2001" diff --git a/tests/unit/builders/application/test_keyword_spotting.py b/tests/unit/builders/application/test_keyword_spotting.py new file mode 100644 index 0000000..5092a51 --- /dev/null +++ b/tests/unit/builders/application/test_keyword_spotting.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application import get_keyword_spotting +from eclypse.remote.service.service import Service + + +def test_keyword_spotting_builder(): + plain_app = get_keyword_spotting(include_default_assets=True) + mpi_app = get_keyword_spotting( + include_default_assets=True, + communication_interface="mpi", + ) + rest_app = get_keyword_spotting( + include_default_assets=True, + communication_interface="rest", + ) + + assert plain_app.has_service_implementations is False + assert mpi_app.has_service_implementations is True + assert rest_app.has_service_implementations is True + assert all(isinstance(service, Service) for service in mpi_app.services.values()) + assert all(isinstance(service, Service) for service in rest_app.services.values()) + assert plain_app.has_edge("SensorService", "PreprocessService") + assert len(plain_app.flows) == 1 + + +def test_keyword_spotting_rejects_unknown_interfaces(): + with pytest.raises(ValueError, match="Unknown communication interface"): + get_keyword_spotting(communication_interface="grpc") # type: ignore[arg-type] diff --git a/tests/unit/builders/application/test_keyword_spotting_services.py b/tests/unit/builders/application/test_keyword_spotting_services.py new file mode 100644 index 0000000..985e252 --- /dev/null +++ b/tests/unit/builders/application/test_keyword_spotting_services.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application.keyword_spotting import mpi_services as kws_mpi +from eclypse.builders.application.keyword_spotting import rest_services as kws_rest +from tests.unit.builders.application._service_test_helpers import ( + FakeRESTInterface, + attach_service_logger, + set_mpi, +) + + +@pytest.mark.asyncio +async def test_keyword_spotting_services(monkeypatch): + preprocess = attach_service_logger(kws_rest.PreprocessService("PreprocessService")) + inference = attach_service_logger(kws_rest.InferenceService("InferenceService")) + action = attach_service_logger(kws_rest.ActionService("ActionService")) + sensor = attach_service_logger(kws_rest.SensorService("SensorService")) + + code, body = preprocess.preprocess(1, [0.1, 0.3, 0.2]) + assert code == 200 + assert body["features"] == [1.0, 3.0, 2.0] + + code, body = inference.infer(1, [1.0, 3.0, 2.0]) + assert code == 200 + assert body["keyword"] == "eclypse" + + code, body = inference.infer(2, [1.0, 1.0]) + assert body["keyword"] == "background" + + code, body = action.action(1, "eclypse") + assert code == 200 + assert body["command"] == "wake" + + code, body = action.action(2, "background") + assert body["command"] == "idle" + + sensor_rest = FakeRESTInterface( + { + ("POST", "PreprocessService/preprocess"): {"features": [1.0, 3.0, 2.0]}, + ("POST", "InferenceService/infer"): {"keyword": "eclypse"}, + ("POST", "ActionService/action"): {"command": "wake"}, + } + ) + monkeypatch.setattr(type(sensor), "rest", property(lambda self: sensor_rest)) + response = await sensor.step() + assert response.body["command"] == "wake" + + mpi_preprocess = attach_service_logger( + kws_mpi.PreprocessService("PreprocessService") + ) + mpi_inference = attach_service_logger(kws_mpi.InferenceService("InferenceService")) + mpi_action = attach_service_logger(kws_mpi.ActionService("ActionService")) + mpi_sensor = attach_service_logger(kws_mpi.SensorService("SensorService")) + + preprocess_comm = set_mpi( + mpi_preprocess, + [ + { + "sender_id": "SensorService", + "request_type": "preprocess_audio", + "window_id": 1, + "samples": [0.1, 0.3, 0.2], + } + ], + ) + await mpi_preprocess.step() + assert preprocess_comm.sent[0][1]["features"] == [1.0, 3.0, 2.0] + + inference_comm = set_mpi( + mpi_inference, + [ + { + "sender_id": "PreprocessService", + "request_type": "run_inference", + "window_id": 1, + "features": [1.0, 3.0, 2.0], + } + ], + ) + await mpi_inference.step() + assert inference_comm.sent[0][1]["keyword"] == "eclypse" + + action_comm = set_mpi( + mpi_action, + [ + { + "sender_id": "InferenceService", + "request_type": "dispatch_action", + "window_id": 1, + "keyword": "background", + } + ], + ) + await mpi_action.step() + assert action_comm.sent[0][1]["command"] == "idle" + + sensor_comm = set_mpi(mpi_sensor, [{"command": "wake"}]) + response = await mpi_sensor.step() + assert sensor_comm.sent[0][0] == "PreprocessService" + assert response["command"] == "wake" diff --git a/tests/unit/builders/application/test_media_service.py b/tests/unit/builders/application/test_media_service.py new file mode 100644 index 0000000..261c2aa --- /dev/null +++ b/tests/unit/builders/application/test_media_service.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application import get_media_service +from eclypse.remote.service.service import Service + + +def test_media_service_builder(): + plain_app = get_media_service(include_default_assets=True) + mpi_app = get_media_service( + include_default_assets=True, + communication_interface="mpi", + ) + rest_app = get_media_service( + include_default_assets=True, + communication_interface="rest", + ) + + assert plain_app.has_service_implementations is False + assert mpi_app.has_service_implementations is True + assert rest_app.has_service_implementations is True + assert all(isinstance(service, Service) for service in mpi_app.services.values()) + assert all(isinstance(service, Service) for service in rest_app.services.values()) + assert plain_app.has_edge("ComposeReviewService", "UniqueIdService") + assert len(plain_app.flows) == 4 + + +def test_media_service_rejects_unknown_interfaces(): + with pytest.raises(ValueError, match="Unknown communication interface"): + get_media_service(communication_interface="grpc") # type: ignore[arg-type] diff --git a/tests/unit/builders/application/test_media_service_services.py b/tests/unit/builders/application/test_media_service_services.py new file mode 100644 index 0000000..8b88e99 --- /dev/null +++ b/tests/unit/builders/application/test_media_service_services.py @@ -0,0 +1,413 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application.deathstarbench.media_service import ( + mpi_services as media_mpi, +) +from eclypse.builders.application.deathstarbench.media_service import ( + rest_services as media_rest, +) +from tests.unit.builders.application._service_test_helpers import ( + FakeRESTInterface, + attach_service_logger, + set_mpi, +) + + +@pytest.mark.asyncio +async def test_media_service_services(monkeypatch): + compose = attach_service_logger( + media_rest.ComposeReviewService("ComposeReviewService"), + ) + unique_id = attach_service_logger(media_rest.UniqueIdService("UniqueIdService")) + movie_id = attach_service_logger(media_rest.MovieIdService("MovieIdService")) + text = attach_service_logger(media_rest.TextService("TextService")) + rating = attach_service_logger(media_rest.RatingService("RatingService")) + user = attach_service_logger(media_rest.UserService("UserService")) + review_storage = attach_service_logger( + media_rest.ReviewStorageService("ReviewStorageService"), + ) + user_review = attach_service_logger( + media_rest.UserReviewService("UserReviewService"), + ) + movie_review = attach_service_logger( + media_rest.MovieReviewService("MovieReviewService"), + ) + movie_info = attach_service_logger(media_rest.MovieInfoService("MovieInfoService")) + cast_info = attach_service_logger(media_rest.CastInfoService("CastInfoService")) + plot = attach_service_logger(media_rest.PlotService("PlotService")) + + compose_rest = FakeRESTInterface( + { + ("POST", "UniqueIdService/compose"): { + "review_id": 7001, + "status": "stored", + } + } + ) + monkeypatch.setattr(type(compose), "rest", property(lambda self: compose_rest)) + compose_response = await compose.step() + assert compose_response.body["review_id"] == 7001 + + unique_rest = FakeRESTInterface( + { + ("POST", "MovieIdService/compose"): { + "review_id": 7001, + "movie_id": "m1", + } + } + ) + monkeypatch.setattr(type(unique_id), "rest", property(lambda self: unique_rest)) + code, body = await unique_id.compose( + req_id=1, + reply_to="ComposeReviewService", + user_id=101, + username="ada", + movie_title="The Matrix", + rating=5, + text="Great movie", + ) + assert code == 200 + assert unique_rest.calls[0][2]["review_id"] == 7001 + assert body["movie_id"] == "m1" + + code, body = movie_id.lookup(movie_title="The Matrix") + assert code == 200 + assert body["movie_id"] == "m1" + + movie_id_rest = FakeRESTInterface( + {("POST", "TextService/compose"): {"text": "Great movie", "rating": 5}} + ) + monkeypatch.setattr(type(movie_id), "rest", property(lambda self: movie_id_rest)) + code, body = await movie_id.compose( + movie_title="The Matrix", + review_id=7001, + req_id=1, + rating=5, + text="Great movie", + ) + assert code == 200 + assert movie_id_rest.calls[0][2]["movie_id"] == "m1" + assert body["rating"] == 5 + + text_rest = FakeRESTInterface( + {("POST", "RatingService/compose"): {"rating": 5, "text": "Great movie"}} + ) + monkeypatch.setattr(type(text), "rest", property(lambda self: text_rest)) + code, body = await text.compose(text=" Great movie ", review_id=7001, req_id=1) + assert code == 200 + assert text_rest.calls[0][2]["text"] == "Great movie" + assert body["text"] == "Great movie" + + rating_rest = FakeRESTInterface( + {("POST", "UserService/compose"): {"user": {"user_id": 101}}} + ) + monkeypatch.setattr(type(rating), "rest", property(lambda self: rating_rest)) + code, body = await rating.compose(rating=8, review_id=7001, req_id=1) + assert code == 200 + assert rating_rest.calls[0][2]["rating"] == 5 + assert body["user"]["user_id"] == 101 + + code, body = user.user(user_id=101, username="ada") + assert code == 200 + assert body["user"]["username"] == "ada" + + user_rest = FakeRESTInterface( + {("POST", "ReviewStorageService/store"): {"status": "stored"}} + ) + monkeypatch.setattr(type(user), "rest", property(lambda self: user_rest)) + code, body = await user.compose( + user_id=101, + username="ada", + review_id=7001, + req_id=1, + ) + assert code == 200 + assert user_rest.calls[0][2]["user"]["username"] == "ada" + assert body["status"] == "stored" + + review_storage.reviews[7001] = {"review_id": 7001, "movie_id": "m1"} + code, body = review_storage.read_many(review_ids=[7001]) + assert code == 200 + assert body["reviews"][0]["review_id"] == 7001 + + review_storage_rest = FakeRESTInterface( + { + ("POST", "UserReviewService/write"): { + "status": "stored", + "review_id": 7001, + } + } + ) + monkeypatch.setattr( + type(review_storage), + "rest", + property(lambda self: review_storage_rest), + ) + code, body = await review_storage.store( + review_id=7001, + movie_id="m1", + movie_title="The Matrix", + rating=5, + text="Great movie", + user={"user_id": 101, "username": "ada"}, + reply_to="ComposeReviewService", + ) + assert code == 201 + assert review_storage_rest.calls[0][1] == "UserReviewService/write" + assert body["status"] == "stored" + + user_review_rest = FakeRESTInterface( + { + ("POST", "MovieReviewService/write"): { + "status": "stored", + "review_id": 7001, + } + } + ) + monkeypatch.setattr( + type(user_review), + "rest", + property(lambda self: user_review_rest), + ) + code, body = await user_review.write( + review={ + "review_id": 7001, + "movie_id": "m1", + "movie_title": "The Matrix", + "user": {"user_id": 101, "username": "ada"}, + }, + reply_to="ComposeReviewService", + ) + assert code == 200 + assert user_review_rest.calls[0][1] == "MovieReviewService/write" + assert body["review_id"] == 7001 + + code, body = movie_review.write( + review={ + "review_id": 7001, + "movie_id": "m1", + "movie_title": "The Matrix", + }, + reply_to="ComposeReviewService", + ) + assert code == 201 + assert body["review_count"] == 1 + + code, body = movie_review.read(movie_id="m1") + assert code == 200 + assert body["reviews"][0]["review_id"] == 7001 + + code, body = cast_info.cast(movie_id="m1") + assert code == 200 + assert "Keanu Reeves" in body["cast"] + + code, body = plot.plot(movie_id="m1") + assert code == 200 + assert body["plot"].startswith("A hacker") + + movie_info_rest = FakeRESTInterface( + { + ("GET", "CastInfoService/cast"): { + "cast": ["Keanu Reeves", "Carrie-Anne Moss"] + }, + ( + "GET", + "PlotService/plot", + ): {"plot": "A hacker discovers the world is a simulation."}, + ( + "GET", + "MovieReviewService/read", + ): {"reviews": [{"review_id": 7001}]}, + } + ) + monkeypatch.setattr( + type(movie_info), "rest", property(lambda self: movie_info_rest) + ) + code, body = await movie_info.details(movie_id="m1", movie_title="The Matrix") + assert code == 200 + assert len(body["cast"]) == 2 + assert body["reviews"][0]["review_id"] == 7001 + + mpi_compose = attach_service_logger( + media_mpi.ComposeReviewService("ComposeReviewService"), + ) + mpi_unique_id = attach_service_logger(media_mpi.UniqueIdService("UniqueIdService")) + mpi_movie_id = attach_service_logger(media_mpi.MovieIdService("MovieIdService")) + mpi_text = attach_service_logger(media_mpi.TextService("TextService")) + mpi_rating = attach_service_logger(media_mpi.RatingService("RatingService")) + mpi_user = attach_service_logger(media_mpi.UserService("UserService")) + mpi_review_storage = attach_service_logger( + media_mpi.ReviewStorageService("ReviewStorageService"), + ) + mpi_user_review = attach_service_logger( + media_mpi.UserReviewService("UserReviewService"), + ) + mpi_movie_review = attach_service_logger( + media_mpi.MovieReviewService("MovieReviewService"), + ) + mpi_movie_info = attach_service_logger( + media_mpi.MovieInfoService("MovieInfoService") + ) + mpi_cast = attach_service_logger(media_mpi.CastInfoService("CastInfoService")) + mpi_plot = attach_service_logger(media_mpi.PlotService("PlotService")) + + compose_comm = set_mpi( + mpi_compose, + [{"response_type": "compose_review_response", "review_id": 7001}], + ) + compose_response = await mpi_compose.step() + assert compose_comm.sent[0][0] == "UniqueIdService" + assert compose_response["review_id"] == 7001 + + unique_comm = set_mpi( + mpi_unique_id, + [{"sender_id": "ComposeReviewService", "request_type": "compose_review"}], + ) + await mpi_unique_id.step() + assert unique_comm.sent[0][0] == "MovieIdService" + assert unique_comm.sent[0][1]["review_id"] == 7001 + + movie_id_comm = set_mpi( + mpi_movie_id, + [ + { + "sender_id": "UniqueIdService", + "request_type": "compose_review", + "movie_title": "The Matrix", + } + ], + ) + await mpi_movie_id.step() + assert movie_id_comm.sent[0][0] == "TextService" + assert movie_id_comm.sent[0][1]["movie_id"] == "m1" + + text_comm = set_mpi( + mpi_text, + [{"sender_id": "MovieIdService", "text": " Great movie "}], + ) + await mpi_text.step() + assert text_comm.sent[0][0] == "RatingService" + assert text_comm.sent[0][1]["text"] == "Great movie" + + rating_comm = set_mpi( + mpi_rating, + [{"sender_id": "TextService", "rating": 8}], + ) + await mpi_rating.step() + assert rating_comm.sent[0][0] == "UserService" + assert rating_comm.sent[0][1]["rating"] == 5 + + user_comm = set_mpi( + mpi_user, + [{"sender_id": "RatingService", "user_id": 101, "username": "ada"}], + ) + await mpi_user.step() + assert user_comm.sent[0][0] == "ReviewStorageService" + assert user_comm.sent[0][1]["user"]["username"] == "ada" + + storage_comm = set_mpi( + mpi_review_storage, + [ + { + "sender_id": "UserService", + "request_type": "compose_review", + "review_id": 7001, + "movie_id": "m1", + "movie_title": "The Matrix", + "rating": 5, + "text": "Great movie", + "user": {"user_id": 101, "username": "ada"}, + "reply_to": "ComposeReviewService", + } + ], + ) + await mpi_review_storage.step() + assert storage_comm.sent[0][0] == "UserReviewService" + assert storage_comm.sent[0][1]["request_type"] == "write_user_review" + + user_review_comm = set_mpi( + mpi_user_review, + [ + { + "sender_id": "ReviewStorageService", + "request_type": "write_user_review", + "review": { + "review_id": 7001, + "movie_id": "m1", + "movie_title": "The Matrix", + "user": {"user_id": 101, "username": "ada"}, + }, + "reply_to": "ComposeReviewService", + } + ], + ) + await mpi_user_review.step() + assert user_review_comm.sent[0][0] == "MovieReviewService" + + movie_review_comm = set_mpi( + mpi_movie_review, + [ + { + "sender_id": "UserReviewService", + "request_type": "write_movie_review", + "review": { + "review_id": 7001, + "movie_id": "m1", + "movie_title": "The Matrix", + }, + "reply_to": "ComposeReviewService", + } + ], + ) + await mpi_movie_review.step() + assert movie_review_comm.sent[0][0] == "ComposeReviewService" + assert movie_review_comm.sent[0][1]["review_count"] == 1 + + cast_comm = set_mpi( + mpi_cast, + [ + { + "sender_id": "MovieInfoService", + "request_type": "get_cast", + "movie_id": "m1", + } + ], + ) + await mpi_cast.step() + assert cast_comm.sent[0][0] == "MovieInfoService" + assert "Keanu Reeves" in cast_comm.sent[0][1]["cast"] + + plot_comm = set_mpi( + mpi_plot, + [ + { + "sender_id": "MovieInfoService", + "request_type": "get_plot", + "movie_id": "m1", + } + ], + ) + await mpi_plot.step() + assert plot_comm.sent[0][0] == "MovieInfoService" + assert plot_comm.sent[0][1]["plot"].startswith("A hacker") + + movie_info_comm = set_mpi( + mpi_movie_info, + [ + { + "sender_id": "FrontendService", + "movie_id": "m1", + "movie_title": "The Matrix", + }, + {"cast": ["Keanu Reeves", "Carrie-Anne Moss"]}, + {"plot": "A hacker discovers the world is a simulation."}, + {"reviews": [{"review_id": 7001}]}, + ], + ) + movie_info_response = await mpi_movie_info.step() + assert movie_info_comm.sent[0][0] == "CastInfoService" + assert movie_info_comm.sent[1][0] == "PlotService" + assert movie_info_comm.sent[2][0] == "MovieReviewService" + assert movie_info_response["reviews"][0]["review_id"] == 7001 diff --git a/tests/unit/builders/application/test_social_network.py b/tests/unit/builders/application/test_social_network.py new file mode 100644 index 0000000..2bab7cd --- /dev/null +++ b/tests/unit/builders/application/test_social_network.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application import get_social_network +from eclypse.remote.service.service import Service + + +def test_social_network_builder(): + plain_app = get_social_network(include_default_assets=True) + mpi_app = get_social_network( + include_default_assets=True, + communication_interface="mpi", + ) + rest_app = get_social_network( + include_default_assets=True, + communication_interface="rest", + ) + + assert plain_app.has_service_implementations is False + assert mpi_app.has_service_implementations is True + assert rest_app.has_service_implementations is True + assert all(isinstance(service, Service) for service in mpi_app.services.values()) + assert all(isinstance(service, Service) for service in rest_app.services.values()) + assert plain_app.has_edge("ComposePostService", "UniqueIdService") + assert len(plain_app.flows) == 3 + + +def test_social_network_rejects_unknown_interfaces(): + with pytest.raises(ValueError, match="Unknown communication interface"): + get_social_network(communication_interface="grpc") # type: ignore[arg-type] diff --git a/tests/unit/builders/application/test_social_network_services.py b/tests/unit/builders/application/test_social_network_services.py new file mode 100644 index 0000000..2f28860 --- /dev/null +++ b/tests/unit/builders/application/test_social_network_services.py @@ -0,0 +1,445 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application.deathstarbench.social_network import ( + mpi_services as social_mpi, +) +from eclypse.builders.application.deathstarbench.social_network import ( + rest_services as social_rest, +) +from tests.unit.builders.application._service_test_helpers import ( + FakeRESTInterface, + attach_service_logger, + set_mpi, +) + + +@pytest.mark.asyncio +async def test_social_network_services(monkeypatch): + compose = attach_service_logger( + social_rest.ComposePostService("ComposePostService"), + ) + unique_id = attach_service_logger(social_rest.UniqueIdService("UniqueIdService")) + text = attach_service_logger(social_rest.TextService("TextService")) + mentions = attach_service_logger( + social_rest.UserMentionService("UserMentionService"), + ) + urls = attach_service_logger( + social_rest.UrlShortenService("UrlShortenService"), + ) + media = attach_service_logger(social_rest.MediaService("MediaService")) + user = attach_service_logger(social_rest.UserService("UserService")) + post_storage = attach_service_logger( + social_rest.PostStorageService("PostStorageService"), + ) + user_timeline = attach_service_logger( + social_rest.UserTimelineService("UserTimelineService"), + ) + home_timeline = attach_service_logger( + social_rest.HomeTimelineService("HomeTimelineService"), + ) + social_graph = attach_service_logger( + social_rest.SocialGraphService("SocialGraphService"), + ) + + compose_rest = FakeRESTInterface( + { + ("POST", "UniqueIdService/compose"): { + "post_id": 5001, + "status": "posted", + "follower_count": 2, + } + } + ) + monkeypatch.setattr(type(compose), "rest", property(lambda self: compose_rest)) + compose_response = await compose.step() + assert compose_response.body["post_id"] == 5001 + + unique_rest = FakeRESTInterface( + {("POST", "TextService/compose"): {"post_id": 5001, "status": "posted"}} + ) + monkeypatch.setattr(type(unique_id), "rest", property(lambda self: unique_rest)) + code, body = await unique_id.compose( + req_id=1, + reply_to="ComposePostService", + username="alice", + user_id=101, + text="Hello @bob check https://example.com", + media_ids=[11], + media_types=["image"], + post_type="POST", + ) + assert code == 200 + assert unique_rest.calls[0][1] == "TextService/compose" + assert unique_rest.calls[0][2]["post_id"] == 5001 + assert body["status"] == "posted" + + text_rest = FakeRESTInterface( + { + ("POST", "UserMentionService/compose"): { + "user_mentions": [{"user_id": 201, "username": "bob"}] + } + } + ) + monkeypatch.setattr(type(text), "rest", property(lambda self: text_rest)) + code, body = await text.compose( + text="Hello @bob check https://example.com", + post_id=5001, + req_id=1, + ) + assert code == 200 + assert text_rest.calls[0][2]["mentions"] == ["bob"] + assert text_rest.calls[0][2]["urls"] == ["https://example.com"] + assert body["user_mentions"][0]["username"] == "bob" + + mention_rest = FakeRESTInterface( + { + ("POST", "UrlShortenService/compose"): { + "shortened_urls": [{"shortened_url": "https://t.ec/1"}] + } + } + ) + monkeypatch.setattr(type(mentions), "rest", property(lambda self: mention_rest)) + code, body = await mentions.compose( + mentions=["bob"], + post_id=5001, + req_id=1, + ) + assert code == 200 + assert mention_rest.calls[0][2]["user_mentions"][0]["user_id"] == 201 + assert body["shortened_urls"][0]["shortened_url"] == "https://t.ec/1" + + url_rest = FakeRESTInterface( + { + ("POST", "MediaService/compose"): { + "media": [{"media_id": 11, "media_type": "image"}] + } + } + ) + monkeypatch.setattr(type(urls), "rest", property(lambda self: url_rest)) + code, body = await urls.compose( + urls=["https://example.com"], + post_id=5001, + req_id=1, + ) + assert code == 200 + assert url_rest.calls[0][2]["shortened_urls"][0]["expanded_url"] == ( + "https://example.com" + ) + assert body["media"][0]["media_type"] == "image" + + media_rest = FakeRESTInterface( + {("POST", "UserService/compose"): {"creator": {"user_id": 101}}} + ) + monkeypatch.setattr(type(media), "rest", property(lambda self: media_rest)) + code, body = await media.compose( + media_ids=[11], + media_types=["image"], + post_id=5001, + req_id=1, + ) + assert code == 200 + assert media_rest.calls[0][2]["media"][0]["media_id"] == 11 + assert body["creator"]["user_id"] == 101 + + code, body = user.creator(user_id=101, username="alice") + assert code == 200 + assert body["creator"]["username"] == "alice" + + user_rest = FakeRESTInterface( + {("POST", "PostStorageService/store"): {"status": "posted"}} + ) + monkeypatch.setattr(type(user), "rest", property(lambda self: user_rest)) + code, body = await user.compose( + user_id=101, + username="alice", + post_id=5001, + req_id=1, + ) + assert code == 200 + assert user_rest.calls[0][2]["creator"]["username"] == "alice" + assert body["status"] == "posted" + + post_storage.posts[5001] = {"post_id": 5001, "text": "Hello"} + code, body = post_storage.read_many(post_ids=[5001]) + assert code == 200 + assert body["posts"][0]["post_id"] == 5001 + + post_storage_rest = FakeRESTInterface( + { + ("POST", "UserTimelineService/write"): { + "status": "posted", + "post_id": 5001, + } + } + ) + monkeypatch.setattr( + type(post_storage), + "rest", + property(lambda self: post_storage_rest), + ) + code, body = await post_storage.store( + post_id=5001, + creator={"user_id": 101, "username": "alice"}, + text="Hello", + user_mentions=[], + media=[], + shortened_urls=[], + reply_to="ComposePostService", + ) + assert code == 201 + assert post_storage_rest.calls[0][1] == "UserTimelineService/write" + assert body["status"] == "posted" + + user_timeline.timelines[101] = [5001] + user_timeline_read_rest = FakeRESTInterface( + { + ("GET", "PostStorageService/read_many"): { + "posts": [{"post_id": 5001, "text": "Hello"}] + } + } + ) + monkeypatch.setattr( + type(user_timeline), + "rest", + property(lambda self: user_timeline_read_rest), + ) + code, body = await user_timeline.read(user_id=101) + assert code == 200 + assert body["posts"][0]["post_id"] == 5001 + + user_timeline_write_rest = FakeRESTInterface( + {("POST", "HomeTimelineService/write"): {"status": "posted", "post_id": 5001}} + ) + monkeypatch.setattr( + type(user_timeline), + "rest", + property(lambda self: user_timeline_write_rest), + ) + code, body = await user_timeline.write( + creator={"user_id": 101, "username": "alice"}, + post_id=5001, + post={"post_id": 5001, "text": "Hello"}, + reply_to="ComposePostService", + ) + assert code == 200 + assert user_timeline_write_rest.calls[0][1] == "HomeTimelineService/write" + assert body["status"] == "posted" + + home_timeline.timelines[101] = [5001] + home_read_rest = FakeRESTInterface( + { + ("GET", "PostStorageService/read_many"): { + "posts": [{"post_id": 5001, "text": "Hello"}] + } + } + ) + monkeypatch.setattr( + type(home_timeline), + "rest", + property(lambda self: home_read_rest), + ) + code, body = await home_timeline.read(user_id=101) + assert code == 200 + assert body["posts"][0]["text"] == "Hello" + + home_write_rest = FakeRESTInterface( + {("GET", "SocialGraphService/followers"): {"followers": [202, 303]}} + ) + monkeypatch.setattr( + type(home_timeline), + "rest", + property(lambda self: home_write_rest), + ) + code, body = await home_timeline.write( + creator={"user_id": 101, "username": "alice"}, + post_id=5001, + post={"post_id": 5001, "text": "Hello"}, + reply_to="ComposePostService", + ) + assert code == 201 + assert body["follower_count"] == 2 + assert 202 in body["delivered_to"] + + code, body = social_graph.followers(user_id=101) + assert code == 200 + assert body["followers"] == [202, 303] + + code, body = social_graph.follow(user_id=101, follower_id=404) + assert code == 200 + assert 404 in body["followers"] + + mpi_compose = attach_service_logger( + social_mpi.ComposePostService("ComposePostService"), + ) + mpi_unique_id = attach_service_logger(social_mpi.UniqueIdService("UniqueIdService")) + mpi_text = attach_service_logger(social_mpi.TextService("TextService")) + mpi_mentions = attach_service_logger( + social_mpi.UserMentionService("UserMentionService"), + ) + mpi_urls = attach_service_logger( + social_mpi.UrlShortenService("UrlShortenService"), + ) + mpi_media = attach_service_logger(social_mpi.MediaService("MediaService")) + mpi_user = attach_service_logger(social_mpi.UserService("UserService")) + mpi_post_storage = attach_service_logger( + social_mpi.PostStorageService("PostStorageService"), + ) + mpi_user_timeline = attach_service_logger( + social_mpi.UserTimelineService("UserTimelineService"), + ) + mpi_home_timeline = attach_service_logger( + social_mpi.HomeTimelineService("HomeTimelineService"), + ) + mpi_social_graph = attach_service_logger( + social_mpi.SocialGraphService("SocialGraphService"), + ) + + compose_comm = set_mpi( + mpi_compose, + [{"response_type": "compose_post_response", "post_id": 5001}], + ) + compose_response = await mpi_compose.step() + assert compose_comm.sent[0][0] == "UniqueIdService" + assert compose_response["post_id"] == 5001 + + unique_comm = set_mpi( + mpi_unique_id, + [ + { + "sender_id": "ComposePostService", + "request_type": "compose_post", + "reply_to": "ComposePostService", + "text": "Hello @bob check https://example.com", + "media_ids": [11], + "media_types": ["image"], + } + ], + ) + await mpi_unique_id.step() + assert unique_comm.sent[0][0] == "TextService" + assert unique_comm.sent[0][1]["post_id"] == 5001 + + text_comm = set_mpi( + mpi_text, + [ + { + "sender_id": "UniqueIdService", + "text": "Hello @bob check https://example.com", + } + ], + ) + await mpi_text.step() + assert text_comm.sent[0][0] == "UserMentionService" + assert text_comm.sent[0][1]["mentions"] == ["bob"] + + mention_comm = set_mpi( + mpi_mentions, + [{"sender_id": "TextService", "mentions": ["bob"]}], + ) + await mpi_mentions.step() + assert mention_comm.sent[0][0] == "UrlShortenService" + assert mention_comm.sent[0][1]["user_mentions"][0]["username"] == "bob" + + url_comm = set_mpi( + mpi_urls, + [{"sender_id": "UserMentionService", "urls": ["https://example.com"]}], + ) + await mpi_urls.step() + assert url_comm.sent[0][0] == "MediaService" + assert url_comm.sent[0][1]["shortened_urls"][0]["shortened_url"] == "https://t.ec/1" + + media_comm = set_mpi( + mpi_media, + [ + { + "sender_id": "UrlShortenService", + "media_ids": [11], + "media_types": ["image"], + } + ], + ) + await mpi_media.step() + assert media_comm.sent[0][0] == "UserService" + assert media_comm.sent[0][1]["media"][0]["media_id"] == 11 + + user_comm = set_mpi( + mpi_user, + [{"sender_id": "MediaService", "user_id": 101, "username": "alice"}], + ) + await mpi_user.step() + assert user_comm.sent[0][0] == "PostStorageService" + assert user_comm.sent[0][1]["creator"]["username"] == "alice" + + storage_comm = set_mpi( + mpi_post_storage, + [ + { + "sender_id": "UserService", + "request_type": "compose_post", + "post_id": 5001, + "creator": {"user_id": 101, "username": "alice"}, + "text": "Hello", + "user_mentions": [], + "media": [], + "shortened_urls": [], + "reply_to": "ComposePostService", + } + ], + ) + await mpi_post_storage.step() + assert storage_comm.sent[0][0] == "UserTimelineService" + assert storage_comm.sent[0][1]["request_type"] == "write_user_timeline" + + user_timeline_comm = set_mpi( + mpi_user_timeline, + [ + { + "sender_id": "PostStorageService", + "request_type": "write_user_timeline", + "creator": {"user_id": 101, "username": "alice"}, + "post_id": 5001, + "post": {"post_id": 5001, "text": "Hello"}, + "reply_to": "ComposePostService", + } + ], + ) + await mpi_user_timeline.step() + assert user_timeline_comm.sent[0][0] == "HomeTimelineService" + + social_graph_comm = set_mpi( + mpi_social_graph, + [ + { + "sender_id": "HomeTimelineService", + "request_type": "get_followers", + "user_id": 101, + } + ], + ) + await mpi_social_graph.step() + assert social_graph_comm.sent[0][0] == "HomeTimelineService" + assert social_graph_comm.sent[0][1]["followers"] == [202, 303] + + home_timeline_comm = set_mpi( + mpi_home_timeline, + [ + { + "sender_id": "UserTimelineService", + "request_type": "write_home_timeline", + "creator": {"user_id": 101, "username": "alice"}, + "post_id": 5001, + "post": {"post_id": 5001, "text": "Hello"}, + "reply_to": "ComposePostService", + }, + { + "sender_id": "SocialGraphService", + "followers": [202, 303], + }, + ], + ) + await mpi_home_timeline.step() + assert home_timeline_comm.sent[0][0] == "SocialGraphService" + assert home_timeline_comm.sent[1][0] == "ComposePostService" + assert home_timeline_comm.sent[1][1]["follower_count"] == 2 diff --git a/tests/unit/builders/application/test_sock_shop.py b/tests/unit/builders/application/test_sock_shop.py index d72a902..7995d4c 100644 --- a/tests/unit/builders/application/test_sock_shop.py +++ b/tests/unit/builders/application/test_sock_shop.py @@ -19,9 +19,9 @@ def test_sock_shop_builder_configures_supported_interfaces_and_flows(): flows=[["FrontendService", "CatalogService"]], ) - assert sock_shop.has_logic is False - assert remote_sock_shop.has_logic is True - assert rest_sock_shop.has_logic is True + assert sock_shop.has_service_implementations is False + assert remote_sock_shop.has_service_implementations is True + assert rest_sock_shop.has_service_implementations is True assert all( isinstance(service, Service) for service in remote_sock_shop.services.values() ) @@ -30,6 +30,7 @@ def test_sock_shop_builder_configures_supported_interfaces_and_flows(): ) assert sock_shop.has_edge("FrontendService", "CatalogService") assert sock_shop.has_edge("CatalogService", "FrontendService") + assert sock_shop.flows[2] == ["FrontendService", "CartService", "FrontendService"] assert len(sock_shop.flows) == 5 assert rest_sock_shop.flows == [["FrontendService", "CatalogService"]] diff --git a/tests/unit/builders/application/test_sock_shop_services.py b/tests/unit/builders/application/test_sock_shop_services.py index d1a83bc..e9d7586 100644 --- a/tests/unit/builders/application/test_sock_shop_services.py +++ b/tests/unit/builders/application/test_sock_shop_services.py @@ -132,11 +132,11 @@ def test_rest_service_exports_and_endpoint_payloads(): assert rest_services.CatalogService is RESTCatalogService assert rest_services.UserService is RESTUserService - cart_service = RESTCartService("CartService") - catalog_service = RESTCatalogService("CatalogService") - payment_service = RESTPaymentService("PaymentService") - shipping_service = RESTShippingService("ShippingService") - user_service = RESTUserService("UserService") + cart_service = _attach_service_logger(RESTCartService("CartService")) + catalog_service = _attach_service_logger(RESTCatalogService("CatalogService")) + payment_service = _attach_service_logger(RESTPaymentService("PaymentService")) + shipping_service = _attach_service_logger(RESTShippingService("ShippingService")) + user_service = _attach_service_logger(RESTUserService("UserService")) cart_code, cart_body = cart_service.get_cart() catalog_code, catalog_body = catalog_service.get_catalog() @@ -256,7 +256,7 @@ async def test_mpi_service_exports_and_workflows(monkeypatch): assert catalog_recipient == "FrontendService" assert catalog_body["response_type"] == "catalog_response" assert cart_recipient == "FrontendService" - assert cart_body["items"][0]["product_id"] == "1" + assert cart_body["items"][0]["id"] == "1" assert user_recipient == "FrontendService" assert user_body["name"] == "John Doe" assert shipping_recipient == "OrderService" @@ -268,7 +268,7 @@ async def test_mpi_service_exports_and_workflows(monkeypatch): [ {"products": [{"id": "1", "price": 19.99}]}, {"name": "Jane"}, - {"items": [{"product_id": "1", "quantity": 2}]}, + {"items": [{"id": "1", "quantity": 2}]}, {"status": "success"}, ] ) @@ -287,23 +287,22 @@ async def test_mpi_service_exports_and_workflows(monkeypatch): { "request_type": "order_request", "user_id": 12345, - "items": [{"product_id": "1", "quantity": 2}], + "items": [{"id": "1", "amount": 39.98}], }, ), ] order_mpi = FakeMPIInterface( [ - {"sender_id": "FrontendService", "items": [{"id": "1"}, {"id": "2"}]}, + { + "sender_id": "FrontendService", + "items": [{"id": "1", "amount": 25.0}, {"id": "2", "amount": 25.0}], + }, {"sender_id": "PaymentService", "transaction_id": 7777}, {"sender_id": "ShippingService", "details": {"carrier": "UPS"}}, ] ) monkeypatch.setattr(type(order_service), "mpi", property(lambda self: order_mpi)) - monkeypatch.setattr( - "eclypse.builders.application.sock_shop.mpi_services.order.rnd.randint", - lambda low, high: 25, - ) await order_service.step() @@ -312,7 +311,7 @@ async def test_mpi_service_exports_and_workflows(monkeypatch): assert order_mpi.sent == [ ( "PaymentService", - {"request_type": "payment_request", "order_id": 54321, "amount": 50}, + {"request_type": "payment_request", "order_id": 54321, "amount": 50.0}, ), ("ShippingService", {"request_type": "shipping_request", "order_id": 54321}), ( @@ -356,6 +355,10 @@ async def test_mpi_services_handle_invalid_requests_and_step_entrypoints(monkeyp "eclypse.builders.application.sock_shop.mpi_services.payment.rnd.choice", lambda options: options[0], ) + monkeypatch.setattr( + "eclypse.builders.application.sock_shop.mpi_services.payment.rnd.randint", + lambda low, high: low, + ) await catalog_service.step() await cart_service.step() diff --git a/tests/unit/builders/application/test_thumbnailer.py b/tests/unit/builders/application/test_thumbnailer.py new file mode 100644 index 0000000..93431b0 --- /dev/null +++ b/tests/unit/builders/application/test_thumbnailer.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application import get_thumbnailer +from eclypse.remote.service.service import Service + + +def test_thumbnailer_builder(): + plain_app = get_thumbnailer(include_default_assets=True) + mpi_app = get_thumbnailer( + include_default_assets=True, + communication_interface="mpi", + ) + rest_app = get_thumbnailer( + include_default_assets=True, + communication_interface="rest", + ) + + assert plain_app.has_service_implementations is False + assert mpi_app.has_service_implementations is True + assert rest_app.has_service_implementations is True + assert all(isinstance(service, Service) for service in mpi_app.services.values()) + assert all(isinstance(service, Service) for service in rest_app.services.values()) + assert plain_app.has_edge("UploadService", "TransformService") + assert len(plain_app.flows) == 1 + + +def test_thumbnailer_rejects_unknown_interfaces(): + with pytest.raises(ValueError, match="Unknown communication interface"): + get_thumbnailer(communication_interface="grpc") # type: ignore[arg-type] diff --git a/tests/unit/builders/application/test_thumbnailer_services.py b/tests/unit/builders/application/test_thumbnailer_services.py new file mode 100644 index 0000000..1baade6 --- /dev/null +++ b/tests/unit/builders/application/test_thumbnailer_services.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application.thumbnailer import mpi_services as thumb_mpi +from eclypse.builders.application.thumbnailer import rest_services as thumb_rest +from tests.unit.builders.application._service_test_helpers import ( + FakeRESTInterface, + attach_service_logger, + set_mpi, +) + + +@pytest.mark.asyncio +async def test_thumbnailer_services(monkeypatch): + transform = attach_service_logger(thumb_rest.TransformService("TransformService")) + storage = attach_service_logger(thumb_rest.StorageService("StorageService")) + notification = attach_service_logger( + thumb_rest.NotificationService("NotificationService") + ) + upload = attach_service_logger(thumb_rest.UploadService("UploadService")) + + code, body = transform.thumbnail("img-1", [1920, 1080]) + assert code == 200 + assert body["thumbnail"]["width"] == 320 + + code, body = storage.store( + "img-1", + {"width": 320, "height": 180, "format": "jpeg"}, + ) + assert code == 200 + assert body["uri"].endswith("/img-1.jpg") + + code, body = notification.notify("img-1", "s3://thumbs/img-1.jpg") + assert code == 200 + assert body["status"] == "stored" + + upload_rest = FakeRESTInterface( + { + ("POST", "TransformService/thumbnail"): { + "thumbnail": {"width": 320, "height": 180, "format": "jpeg"} + }, + ("POST", "StorageService/store"): {"uri": "s3://thumbs/img-1.jpg"}, + ("POST", "NotificationService/notify"): {"status": "stored"}, + } + ) + monkeypatch.setattr(type(upload), "rest", property(lambda self: upload_rest)) + response = await upload.step() + assert response.body["status"] == "stored" + + mpi_transform = attach_service_logger( + thumb_mpi.TransformService("TransformService") + ) + mpi_storage = attach_service_logger(thumb_mpi.StorageService("StorageService")) + mpi_notification = attach_service_logger( + thumb_mpi.NotificationService("NotificationService") + ) + mpi_upload = attach_service_logger(thumb_mpi.UploadService("UploadService")) + + transform_comm = set_mpi( + mpi_transform, + [ + { + "sender_id": "UploadService", + "request_type": "create_thumbnail", + "image_id": "img-1", + "resolution": [1920, 1080], + } + ], + ) + await mpi_transform.step() + assert transform_comm.sent[0][1]["thumbnail"]["format"] == "jpeg" + + storage_comm = set_mpi( + mpi_storage, + [ + { + "sender_id": "TransformService", + "request_type": "store_thumbnail", + "image_id": "img-1", + "thumbnail": {"width": 320, "height": 180, "format": "jpeg"}, + } + ], + ) + await mpi_storage.step() + assert storage_comm.sent[0][1]["uri"].endswith("/img-1.jpg") + + notification_comm = set_mpi( + mpi_notification, + [ + { + "sender_id": "StorageService", + "request_type": "notify_upload", + "image_id": "img-1", + "uri": "s3://thumbs/img-1.jpg", + } + ], + ) + await mpi_notification.step() + assert notification_comm.sent[0][1]["status"] == "stored" + + upload_comm = set_mpi( + mpi_upload, + [{"status": "stored", "uri": "s3://thumbs/img-1.jpg"}], + ) + response = await mpi_upload.step() + assert upload_comm.sent[0][0] == "TransformService" + assert response["status"] == "stored" diff --git a/tests/unit/builders/application/test_video_analytics_serving.py b/tests/unit/builders/application/test_video_analytics_serving.py new file mode 100644 index 0000000..fa65735 --- /dev/null +++ b/tests/unit/builders/application/test_video_analytics_serving.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application import get_video_analytics_serving +from eclypse.remote.service.service import Service + + +def test_video_analytics_serving_builder(): + plain_app = get_video_analytics_serving(include_default_assets=True) + mpi_app = get_video_analytics_serving( + include_default_assets=True, + communication_interface="mpi", + ) + rest_app = get_video_analytics_serving( + include_default_assets=True, + communication_interface="rest", + ) + + assert plain_app.has_service_implementations is False + assert mpi_app.has_service_implementations is True + assert rest_app.has_service_implementations is True + assert all(isinstance(service, Service) for service in mpi_app.services.values()) + assert all(isinstance(service, Service) for service in rest_app.services.values()) + assert plain_app.has_edge("CameraGatewayService", "DetectionService") + assert len(plain_app.flows) == 2 + + +def test_video_analytics_serving_rejects_unknown_interfaces(): + with pytest.raises(ValueError, match="Unknown communication interface"): + get_video_analytics_serving(communication_interface="grpc") # type: ignore[arg-type] diff --git a/tests/unit/builders/application/test_video_analytics_serving_services.py b/tests/unit/builders/application/test_video_analytics_serving_services.py new file mode 100644 index 0000000..9d9a417 --- /dev/null +++ b/tests/unit/builders/application/test_video_analytics_serving_services.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.application.video_analytics_serving import ( + mpi_services as video_mpi, +) +from eclypse.builders.application.video_analytics_serving import ( + rest_services as video_rest, +) +from tests.unit.builders.application._service_test_helpers import ( + FakeRESTInterface, + attach_service_logger, + set_mpi, +) + + +@pytest.mark.asyncio +async def test_video_analytics_services(monkeypatch): + assert video_rest.AnalyticsService is not None + assert video_mpi.AnalyticsService is not None + + detection = attach_service_logger(video_rest.DetectionService("DetectionService")) + tracking = attach_service_logger(video_rest.TrackingService("TrackingService")) + analytics = attach_service_logger(video_rest.AnalyticsService("AnalyticsService")) + gateway = attach_service_logger( + video_rest.CameraGatewayService("CameraGatewayService") + ) + + code, body = detection.detect(1, "camera-a", ["person", "forklift"]) + assert code == 200 + assert body["detections"] == ["person", "forklift"] + + code, body = tracking.track(1, "camera-a", ["person", "forklift"]) + assert code == 200 + assert body["tracks"][1]["track_id"] == 2 + + code, body = analytics.analyse( + 1, + "camera-a", + [{"label": "person", "track_id": 1}, {"label": "forklift", "track_id": 2}], + ) + assert code == 200 + assert body["summary"] == "person, forklift" + + gateway_rest = FakeRESTInterface( + { + ("POST", "DetectionService/detect"): {"detections": ["person", "forklift"]}, + ( + "POST", + "TrackingService/track", + ): {"tracks": [{"label": "person", "track_id": 1}]}, + ( + "POST", + "AnalyticsService/analyse", + ): {"summary": "person", "object_count": 1}, + } + ) + monkeypatch.setattr(type(gateway), "rest", property(lambda self: gateway_rest)) + response = await gateway.step() + assert response.body["object_count"] == 1 + assert gateway_rest.calls[-1][1] == "AnalyticsService/analyse" + + mpi_detection = attach_service_logger( + video_mpi.DetectionService("DetectionService") + ) + mpi_tracking = attach_service_logger(video_mpi.TrackingService("TrackingService")) + mpi_analytics = attach_service_logger( + video_mpi.AnalyticsService("AnalyticsService") + ) + mpi_gateway = attach_service_logger( + video_mpi.CameraGatewayService("CameraGatewayService") + ) + + detection_comm = set_mpi( + mpi_detection, + [ + { + "sender_id": "CameraGatewayService", + "request_type": "analyse_frame", + "frame_id": 1, + "stream_id": "camera-a", + "objects": ["person", "forklift"], + } + ], + ) + await mpi_detection.step() + assert detection_comm.sent == [ + ( + "TrackingService", + { + "request_type": "track_objects", + "frame_id": 1, + "stream_id": "camera-a", + "detections": ["person", "forklift"], + }, + ) + ] + + tracking_comm = set_mpi( + mpi_tracking, + [ + { + "sender_id": "DetectionService", + "request_type": "track_objects", + "frame_id": 1, + "stream_id": "camera-a", + "detections": ["person"], + } + ], + ) + await mpi_tracking.step() + assert tracking_comm.sent[0][0] == "AnalyticsService" + assert tracking_comm.sent[0][1]["tracks"][0]["label"] == "person" + + analytics_comm = set_mpi( + mpi_analytics, + [ + { + "sender_id": "TrackingService", + "request_type": "aggregate_events", + "frame_id": 1, + "stream_id": "camera-a", + "tracks": [{"label": "person", "track_id": 1}], + } + ], + ) + await mpi_analytics.step() + assert analytics_comm.sent == [ + ( + "CameraGatewayService", + { + "response_type": "analytics_result", + "frame_id": 1, + "stream_id": "camera-a", + "object_count": 1, + "summary": "person", + }, + ) + ] + + gateway_comm = set_mpi( + mpi_gateway, + [{"response_type": "analytics_result", "object_count": 1, "summary": "person"}], + ) + gateway_response = await mpi_gateway.step() + assert gateway_comm.sent[0][0] == "DetectionService" + assert gateway_response["summary"] == "person" diff --git a/tests/unit/builders/infrastructure/test_generators.py b/tests/unit/builders/infrastructure/test_generators.py index 6239b5b..51d44c9 100644 --- a/tests/unit/builders/infrastructure/test_generators.py +++ b/tests/unit/builders/infrastructure/test_generators.py @@ -2,42 +2,59 @@ import pytest -from eclypse.builders.infrastructure import get_orion_cev -from eclypse.builders.infrastructure.generators.b_cube import b_cube -from eclypse.builders.infrastructure.generators.fat_tree import fat_tree +from eclypse.builders.infrastructure._helpers import connect_round_robin +from eclypse.builders.infrastructure import ( + get_b_cube, + get_fat_tree, + get_hierarchical, + get_random, + get_scale_free, + get_small_world, + get_star, +) from eclypse.builders.infrastructure.generators.hierarchical import ( _get_connectivity_functions, _uniform_level_connectivity, - hierarchical, ) -from eclypse.builders.infrastructure.generators.random import random -from eclypse.builders.infrastructure.generators.star import star +from eclypse.graph import Infrastructure -def test_star_random_and_hierarchical_generators_build_expected_topologies(): - star_infra = star( +def test_star(): + infrastructure = get_star( 3, symmetric=True, include_default_assets=True, center_assets_values={"cpu": 9}, outer_assets_values={"cpu": 1}, ) - random_infra = random(3, p=1.0, symmetric=True, seed=7) - hierarchy = hierarchical( + + assert set(infrastructure.nodes) == {"center", "outer_0", "outer_1", "outer_2"} + assert infrastructure.nodes["center"]["cpu"] == 9 + assert len(infrastructure.edges) == 6 + + +def test_random(): + infrastructure = get_random(3, p=1.0, symmetric=True, seed=7) + + assert len(infrastructure.nodes) == 3 + assert len(infrastructure.edges) == 6 + + +def test_hierarchical(): + infrastructure = get_hierarchical( 4, node_partitioning=[0.5, 0.5], connectivity=[1.0], cross_level_connectivity=[0.0, 0.0], seed=3, ) + default_infrastructure = get_hierarchical(20, seed=3) - assert set(star_infra.nodes) == {"center", "outer_0", "outer_1", "outer_2"} - assert star_infra.nodes["center"]["cpu"] == 9 - assert len(star_infra.edges) == 6 - assert len(random_infra.edges) == 6 - assert len(hierarchy.nodes) == 4 - assert any(node.startswith("l0_") for node in hierarchy.nodes) - assert any(node.startswith("l1_") for node in hierarchy.nodes) + assert len(infrastructure.nodes) == 4 + assert len(default_infrastructure.nodes) == 20 + assert any(node.startswith("l0_") for node in infrastructure.nodes) + assert any(node.startswith("l1_") for node in infrastructure.nodes) + assert any(node.startswith("l0_") for node in default_infrastructure.nodes) assert list(_uniform_level_connectivity(["a"], ["b", "c"], p=0.0, seed=1)) == [ ("a", "b"), ("a", "c"), @@ -48,24 +65,47 @@ def test_star_random_and_hierarchical_generators_build_expected_topologies(): ) with pytest.raises(ValueError, match="sum of the node distribution"): - hierarchical(4, node_partitioning=[0.4, 0.4]) + get_hierarchical(4, node_partitioning=[0.4, 0.4]) with pytest.raises(ValueError, match="function for each level"): _get_connectivity_functions(connectivity=[1.0], length=2) + with pytest.raises(ValueError, match="function or a list"): + _get_connectivity_functions(connectivity=1.0, length=1) -def test_fat_tree_b_cube_and_orion_build_expected_topologies(): +def test_fat_tree(): with pytest.raises(ValueError, match="even number"): - fat_tree(3) - - fat_tree_infra = fat_tree(2) - bcube_infra = b_cube(1, 2) - orion = get_orion_cev(include_default_assets=True) - - assert len(fat_tree_infra.nodes) == 7 - assert len(fat_tree_infra.edges) == 12 - assert len(bcube_infra.nodes) == 7 - assert len(bcube_infra.edges) == 12 - assert "DU11" in orion.nodes - assert orion.has_edge("DU11", "NS11") - assert orion.nodes["NS11"]["processing_time"] == 1 + get_fat_tree(3) + + infrastructure = get_fat_tree(2) + + assert len(infrastructure.nodes) == 7 + assert len(infrastructure.edges) == 12 + + +def test_b_cube(): + infrastructure = get_b_cube(1, 2) + + assert len(infrastructure.nodes) == 7 + assert len(infrastructure.edges) == 12 + + +def test_small_world(): + infrastructure = get_small_world(6, k=2, p=0.0, symmetric=True, seed=7) + + assert len(infrastructure.nodes) == 6 + assert all(node.startswith("n") for node in infrastructure.nodes) + assert len(infrastructure.edges) == 12 + + +def test_scale_free(): + infrastructure = get_scale_free(6, m=1, symmetric=True, seed=3) + helper_infrastructure = Infrastructure() + helper_infrastructure.add_node("source") + + assert len(infrastructure.nodes) == 6 + assert all(node.startswith("n") for node in infrastructure.nodes) + assert len(infrastructure.edges) >= 10 + + with pytest.raises(ValueError, match="At least one target node"): + connect_round_robin(helper_infrastructure, ["source"], []) diff --git a/tests/unit/builders/infrastructure/test_patterns.py b/tests/unit/builders/infrastructure/test_patterns.py new file mode 100644 index 0000000..b0de07c --- /dev/null +++ b/tests/unit/builders/infrastructure/test_patterns.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import pytest + +from eclypse.builders.infrastructure import ( + get_continuum_tiered, + get_factory_cells, + get_industrial_tsn, + get_mec_5g, + get_multi_region_wan, + get_vehicular_edge, +) + + +def test_continuum_tiered(): + infrastructure = get_continuum_tiered( + device_count=4, + edge_count=2, + fog_count=1, + cloud_count=1, + include_default_assets=True, + seed=5, + ) + + assert any(node.startswith("device_") for node in infrastructure.nodes) + assert any(node.startswith("edge_") for node in infrastructure.nodes) + assert any(node.startswith("fog_") for node in infrastructure.nodes) + assert any(node.startswith("cloud_") for node in infrastructure.nodes) + assert infrastructure.nodes["cloud_0"]["processing_time"] == 1.0 + assert infrastructure.nodes["device_0"]["processing_time"] == 8.0 + + custom_infrastructure = get_continuum_tiered( + device_count=2, + edge_count=1, + fog_count=0, + cloud_count=1, + connectivity=[0.5, 1.0], + cross_level_connectivity=[0.1, 0.2, 0.3], + include_default_assets=True, + seed=5, + ) + assert len(custom_infrastructure.nodes) == 4 + + with pytest.raises(ValueError, match="At least one tier"): + get_continuum_tiered(0, 0, 0, 0) + with pytest.raises(ValueError, match="non-negative"): + get_continuum_tiered(-1, 1) + + +def test_mec_5g(): + infrastructure = get_mec_5g( + user_count=4, + ran_count=2, + mec_count=2, + cloud_count=1, + include_default_assets=True, + ) + + assert "user_0" in infrastructure.nodes + assert "ran_0" in infrastructure.nodes + assert "mec_0" in infrastructure.nodes + assert "cloud_0" in infrastructure.nodes + assert infrastructure.has_edge("user_0", "ran_0") + assert infrastructure.has_edge("ran_0", "mec_0") + assert infrastructure.has_edge("mec_0", "cloud_0") + + with pytest.raises(ValueError, match="RAN"): + get_mec_5g(user_count=1, ran_count=0) + with pytest.raises(ValueError, match="MEC host"): + get_mec_5g(user_count=1, ran_count=1, mec_count=0) + + +def test_multi_region_wan(): + infrastructure = get_multi_region_wan( + region_count=2, + nodes_per_region=3, + path_algorithm=lambda graph, source, target: [source, target], + include_default_assets=True, + ) + + assert "region_0_gateway" in infrastructure.nodes + assert "region_1_gateway" in infrastructure.nodes + assert "region_0_node_0" in infrastructure.nodes + assert infrastructure.has_edge("region_0_gateway", "region_1_gateway") + assert infrastructure.has_edge("region_0_node_0", "region_0_gateway") + + with pytest.raises(ValueError, match="region"): + get_multi_region_wan(region_count=0, nodes_per_region=1) + + +def test_industrial_tsn(): + infrastructure = get_industrial_tsn( + endpoint_count=4, + switch_count=2, + controller_count=1, + edge_count=1, + include_default_assets=True, + ) + + assert "switch_0" in infrastructure.nodes + assert "controller_0" in infrastructure.nodes + assert "endpoint_0" in infrastructure.nodes + assert infrastructure.has_edge("switch_0", "switch_1") + + with pytest.raises(ValueError, match="switch"): + get_industrial_tsn(endpoint_count=1, switch_count=0) + + +def test_factory_cells(): + infrastructure = get_factory_cells( + cell_count=2, + machines_per_cell=2, + sensors_per_cell=2, + plant_edge_count=1, + cloud_count=1, + include_default_assets=True, + ) + + assert "cell_0_controller" in infrastructure.nodes + assert "cell_1_machine_0" in infrastructure.nodes + assert "plant_edge_0" in infrastructure.nodes + assert infrastructure.has_edge("cell_0_controller", "plant_edge_0") + assert infrastructure.has_edge("plant_edge_0", "cloud_0") + + with pytest.raises(ValueError, match="cell"): + get_factory_cells(cell_count=0, machines_per_cell=1, sensors_per_cell=1) + + +def test_vehicular_edge(): + infrastructure = get_vehicular_edge( + vehicle_count=4, + rsu_count=2, + mec_count=1, + cloud_count=1, + include_default_assets=True, + ) + + assert "vehicle_0" in infrastructure.nodes + assert "rsu_0" in infrastructure.nodes + assert "mec_0" in infrastructure.nodes + assert infrastructure.has_edge("vehicle_0", "rsu_0") + assert infrastructure.has_edge("rsu_0", "mec_0") + + with pytest.raises(ValueError, match="RSU"): + get_vehicular_edge(vehicle_count=1, rsu_count=0) + with pytest.raises(ValueError, match="MEC host"): + get_vehicular_edge(vehicle_count=1, rsu_count=1, mec_count=0) diff --git a/tests/unit/builders/infrastructure/test_references.py b/tests/unit/builders/infrastructure/test_references.py new file mode 100644 index 0000000..2059ab4 --- /dev/null +++ b/tests/unit/builders/infrastructure/test_references.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import sys +import types + +import networkx as nx +import pytest + +from eclypse.builders.infrastructure import ( + get_backbone, + get_caida, + get_gabriel, + get_orion_cev, + get_sndlib, + get_topohub, + get_topology_zoo, +) + + +def _install_fake_topohub(monkeypatch: pytest.MonkeyPatch): + requests: list[tuple[str, bool]] = [] + + def get(path: str, use_names: bool = False): + requests.append((path, use_names)) + + graph = nx.Graph(name=path, demands={"A->B": 7}, stats={"avg_degree": 1.0}) + if path.startswith(("sndlib/", "topozoo/")): + graph.add_node(0, name="Alpha", pos=[10.0, 20.0]) + graph.add_node(1, name="Beta", pos=[30.0, 40.0]) + else: + graph.add_node(0, pos=[10.0, 20.0]) + graph.add_node(1, pos=[30.0, 40.0]) + + graph.add_edge( + 0, + 1, + dist=400.0, + capacity=123.0, + ecmp_fwd={"uni": 0.1}, + ecmp_bwd={"uni": 0.2}, + ) + return nx.node_link_data(graph, edges="edges") + + module = types.SimpleNamespace(get=get) + monkeypatch.setitem(sys.modules, "topohub", module) + return requests + + +def test_get_orion_cev(): + infrastructure = get_orion_cev(include_default_assets=True) + + assert "DU11" in infrastructure.nodes + assert infrastructure.has_edge("DU11", "NS11") + assert infrastructure.nodes["NS11"]["processing_time"] == 1 + + +def test_get_sndlib(monkeypatch: pytest.MonkeyPatch): + requests = _install_fake_topohub(monkeypatch) + + infrastructure = get_sndlib("polska", include_default_assets=True) + + assert requests == [("sndlib/polska", True)] + assert "Alpha" in infrastructure.nodes + assert "name" not in infrastructure.nodes["Alpha"] + assert infrastructure.graph["dataset_path"] == "sndlib/polska" + assert infrastructure.graph["demands"] == {"A->B": 7} + assert infrastructure.has_edge("Alpha", "Beta") + assert infrastructure["Alpha"]["Beta"]["latency"] == 2.0 + assert infrastructure["Alpha"]["Beta"]["bandwidth"] == 123.0 + + +def test_get_topology_zoo(monkeypatch: pytest.MonkeyPatch): + requests = _install_fake_topohub(monkeypatch) + + infrastructure = get_topology_zoo("Abilene", include_default_assets=True) + + assert requests == [("topozoo/Abilene", True)] + assert "Alpha" in infrastructure.nodes + assert infrastructure.graph["dataset_path"] == "topozoo/Abilene" + assert infrastructure.nodes["Alpha"]["topohub_id"] == 0 + + +def test_get_backbone(monkeypatch: pytest.MonkeyPatch): + requests = _install_fake_topohub(monkeypatch) + + infrastructure = get_backbone("africa", include_default_assets=True) + + assert requests == [("backbone/africa", False)] + assert "n0" in infrastructure.nodes + assert infrastructure.graph["dataset_path"] == "backbone/africa" + assert "cpu" in infrastructure.nodes["n0"] + assert infrastructure.nodes["n0"]["topohub_id"] == 0 + + +def test_get_caida(monkeypatch: pytest.MonkeyPatch): + requests = _install_fake_topohub(monkeypatch) + + infrastructure = get_caida("2024-01", include_default_assets=True) + + assert requests == [("caida/2024-01", False)] + assert "n0" in infrastructure.nodes + assert infrastructure.graph["dataset_path"] == "caida/2024-01" + + +def test_get_gabriel(monkeypatch: pytest.MonkeyPatch): + requests = _install_fake_topohub(monkeypatch) + + infrastructure = get_gabriel(25, sample=2, include_default_assets=True) + + assert requests == [("gabriel/25/2", False)] + assert "n0" in infrastructure.nodes + assert infrastructure.graph["dataset_path"] == "gabriel/25/2" + + +def test_get_topohub(monkeypatch: pytest.MonkeyPatch): + requests: list[tuple[str, bool]] = [] + + def get(path: str, use_names: bool = False): + requests.append((path, use_names)) + + graph = nx.Graph(name=path) + graph.add_node("alpha", pos=[1.0, 2.0]) + graph.add_node("beta", pos=[3.0, 4.0]) + graph.add_node("gamma", pos=[5.0, 6.0]) + graph.add_edge( + "alpha", + "beta", + latency=7.5, + bandwidth=99.0, + capacity=123.0, + ) + graph.add_edge("beta", "gamma") + return nx.node_link_data(graph, edges="edges") + + monkeypatch.setitem(sys.modules, "topohub", types.SimpleNamespace(get=get)) + + infrastructure = get_topohub( + "sndlib/polska", use_names=True, include_default_assets=True + ) + raw_infrastructure = get_topohub( + "sndlib/polska", use_names=True, include_default_assets=False + ) + + assert requests == [("sndlib/polska", True), ("sndlib/polska", True)] + assert "alpha" in infrastructure.nodes + assert infrastructure.graph["dataset_path"] == "sndlib/polska" + assert infrastructure["alpha"]["beta"]["latency"] == 7.5 + assert infrastructure["alpha"]["beta"]["bandwidth"] == 99.0 + assert infrastructure["alpha"]["beta"]["capacity"] == 123.0 + assert "bandwidth" not in raw_infrastructure["beta"]["gamma"] + assert "latency" not in raw_infrastructure["beta"]["gamma"] + + +def test_get_topohub_preserves_name_metadata_when_not_used_as_id( + monkeypatch: pytest.MonkeyPatch, +): + def get(path: str, use_names: bool = False): + del use_names + graph = nx.Graph(name=path) + graph.add_node(0, name="Alpha", pos=[1.0, 2.0]) + graph.add_node(1, name="Beta", pos=[3.0, 4.0]) + graph.add_edge(0, 1, dist=400.0) + return nx.node_link_data(graph, edges="edges") + + monkeypatch.setitem(sys.modules, "topohub", types.SimpleNamespace(get=get)) + + infrastructure = get_topohub( + "sndlib/polska", use_names=False, include_default_assets=True + ) + + assert "n0" in infrastructure.nodes + assert infrastructure.nodes["n0"]["name"] == "Alpha" + assert infrastructure.nodes["n0"]["topohub_id"] == 0 + + +def test_get_sndlib_requires_topohub(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delitem(sys.modules, "topohub", raising=False) + monkeypatch.setattr( + "eclypse.builders.infrastructure.references.topohub._helpers._require_module", + lambda *args, **kwargs: (_ for _ in ()).throw( + ImportError( + "topohub is not installed. Please install it with 'pip install topohub'." + ) + ), + ) + + with pytest.raises(ImportError, match="pip install topohub"): + get_sndlib("polska") diff --git a/tests/unit/builders/test_exports.py b/tests/unit/builders/test_exports.py index 110f797..c2993c7 100644 --- a/tests/unit/builders/test_exports.py +++ b/tests/unit/builders/test_exports.py @@ -2,8 +2,32 @@ from eclypse.builders import application as application_builders from eclypse.builders import infrastructure as infrastructure_builders +from eclypse.builders import workflow as workflow_builders def test_builder_exports_are_available(): + assert callable(application_builders.get_anomaly_detection) + assert callable(application_builders.get_crud_api) + assert callable(application_builders.get_hotel_reservation) + assert callable(application_builders.get_keyword_spotting) + assert callable(application_builders.get_media_service) assert callable(application_builders.get_sock_shop) + assert callable(application_builders.get_social_network) + assert callable(application_builders.get_thumbnailer) + assert callable(application_builders.get_video_analytics_serving) + assert callable(infrastructure_builders.get_continuum_tiered) + assert callable(infrastructure_builders.get_factory_cells) + assert callable(infrastructure_builders.get_backbone) + assert callable(infrastructure_builders.get_caida) + assert callable(infrastructure_builders.get_gabriel) assert callable(infrastructure_builders.get_orion_cev) + assert callable(infrastructure_builders.get_sndlib) + assert callable(infrastructure_builders.get_topohub) + assert callable(infrastructure_builders.get_topology_zoo) + assert callable(infrastructure_builders.get_industrial_tsn) + assert callable(infrastructure_builders.get_mec_5g) + assert callable(infrastructure_builders.get_multi_region_wan) + assert callable(infrastructure_builders.get_scale_free) + assert callable(infrastructure_builders.get_small_world) + assert callable(infrastructure_builders.get_vehicular_edge) + assert callable(workflow_builders.get_workflow) diff --git a/tests/unit/builders/workflow/test_workflow.py b/tests/unit/builders/workflow/test_workflow.py new file mode 100644 index 0000000..d27d90f --- /dev/null +++ b/tests/unit/builders/workflow/test_workflow.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +import re +import sys +import types +from dataclasses import ( + dataclass, + field, +) + +import networkx as nx +import pytest + +from eclypse.builders.workflow import get_workflow + +_BYTES_PER_MIB = 2**20 + + +@dataclass +class _FakeFile: + file_id: str + size: int + + +@dataclass +class _FakeTaskType: + name: str + + +@dataclass +class _FakeTask: + name: str + task_id: str + runtime: float + cores: int + memory: int | None = None + bytes_read: int | None = None + bytes_written: int | None = None + input_files: list[_FakeFile] = field(default_factory=list) + output_files: list[_FakeFile] = field(default_factory=list) + category: str | None = None + program: str | None = None + args: list[str] = field(default_factory=list) + machines: list[str] = field(default_factory=list) + priority: int | None = None + type: _FakeTaskType | None = None + start_time: str | None = None + + +class _FakeWorkflow(nx.DiGraph): + def __init__(self): + super().__init__() + self.name = "Montage-synthetic-instance" + self.description = "Synthetic workflow" + self.workflow_id = "wf-1" + self.created_at = "2026-01-01T00:00:00+00:00" + self.executed_at = None + self.makespan = 42.0 + self.schema_version = "1.5" + self.runtime_system_name = "WfCommons" + self.runtime_system_url = "https://wfcommons.org" + self.runtime_system_version = "1.0" + self.author_name = "OpenAI" + self.author_email = "test@example.com" + self.author_institution = "OpenAI" + self.author_country = "IT" + self.graph["source"] = "fake" + + def roots(self): + return [node_id for node_id, degree in self.in_degree() if degree == 0] + + def leaves(self): + return [node_id for node_id, degree in self.out_degree() if degree == 0] + + +def _install_fake_wfcommons(monkeypatch: pytest.MonkeyPatch): + calls: dict[str, object] = {} + + class _FakeBaseMethod: + ERROR_TABLE = "ERROR_TABLE" + SMALLEST = "SMALLEST" + BIGGEST = "BIGGEST" + RANDOM = "RANDOM" + + class _FakeRecipe: + def __init__(self, **kwargs): + calls["recipe_kwargs"] = kwargs + + class _FakeWorkflowGenerator: + def __init__(self, recipe): + calls["recipe"] = recipe + + def build_workflow(self, workflow_name=None): + calls["workflow_name"] = workflow_name + + workflow = _FakeWorkflow() + + raw = _FakeFile("raw.fits", 100) + projected = _FakeFile("projected.fits", 200) + diff = _FakeFile("diff.tbl", 300) + + source = _FakeTask( + name="mProject", + task_id="mProject_0001", + runtime=12.5, + cores=2, + memory=64, + input_files=[raw], + output_files=[projected], + program="mProject", + args=["--input", "raw.fits"], + type=_FakeTaskType("COMPUTE"), + start_time="2026-01-01T00:00:00+00:00", + ) + middle = _FakeTask( + name="mDiffFit", + task_id="mDiffFit_0002", + runtime=20.0, + cores=4, + bytes_read=200, + bytes_written=300, + input_files=[projected], + output_files=[diff], + program="mDiffFit", + type=_FakeTaskType("AUXILIARY"), + ) + sink = _FakeTask( + name="mAdd", + task_id="mAdd_0003", + runtime=5.0, + cores=1, + input_files=[diff], + output_files=[], + program="mAdd", + type=_FakeTaskType("TRANSFER"), + ) + + workflow.add_node(source.task_id, task=source, label=source.task_id) + workflow.add_node(middle.task_id, task=middle, label=middle.task_id) + workflow.add_node(sink.task_id, task=sink, label=sink.task_id) + workflow.add_edge(source.task_id, middle.task_id, weight=1) + workflow.add_edge(middle.task_id, sink.task_id, weight=2) + return workflow + + wfcommons = types.ModuleType("wfcommons") + for recipe_name in ( + "BlastRecipe", + "BwaRecipe", + "CyclesRecipe", + "EpigenomicsRecipe", + "GenomeRecipe", + "MontageRecipe", + "RnaseqRecipe", + "SeismologyRecipe", + "SoykbRecipe", + "SrasearchRecipe", + ): + setattr(wfcommons, recipe_name, _FakeRecipe) + wfcommons.WorkflowGenerator = _FakeWorkflowGenerator + + wfchef_module = types.ModuleType("wfcommons.wfchef") + abstract_module = types.ModuleType("wfcommons.wfchef.wfchef_abstract_recipe") + abstract_module.BaseMethod = _FakeBaseMethod + + monkeypatch.setitem(sys.modules, "wfcommons", wfcommons) + monkeypatch.setitem(sys.modules, "wfcommons.wfchef", wfchef_module) + monkeypatch.setitem( + sys.modules, + "wfcommons.wfchef.wfchef_abstract_recipe", + abstract_module, + ) + monkeypatch.setattr( + "eclypse.builders.workflow._helpers._require_module", + lambda *_args, **_kwargs: None, + ) + return calls + + +def test_get_workflow(monkeypatch: pytest.MonkeyPatch): + calls = _install_fake_wfcommons(monkeypatch) + + application = get_workflow( + workflow="montage", + num_tasks=100, + data_footprint=1024, + runtime_factor=2.0, + input_file_size_factor=3.0, + output_file_size_factor=4.0, + base_method="random", + workflow_name="wf-name", + seed=7, + ) + + assert calls["recipe_kwargs"] == { + "data_footprint": 1024, + "num_tasks": 100, + "exclude_graphs": None, + "runtime_factor": 2.0, + "input_file_size_factor": 3.0, + "output_file_size_factor": 4.0, + "base_method": "RANDOM", + } + assert calls["workflow_name"] == "wf-name" + + assert application.id == "Montage-synthetic-instance" + assert application.graph["workflow_backend"] == "wfcommons" + assert application.graph["workflow_family"] == "montage" + assert application.graph["source"] == "fake" + + assert application.flows == [ + ["mProject_0001", "mDiffFit_0002", "mAdd_0003"], + ] + + assert application.nodes["mProject_0001"]["cpu"] == 2.0 + assert application.nodes["mProject_0001"]["ram"] == 64.0 + assert application.nodes["mProject_0001"]["storage"] == pytest.approx( + 300 / _BYTES_PER_MIB, + ) + assert application.nodes["mProject_0001"]["processing_time"] == 12.5 + assert application.nodes["mProject_0001"]["workflow_task_type"] == "COMPUTE" + assert application.nodes["mProject_0001"][ + "workflow_input_size_mib" + ] == pytest.approx( + 100 / _BYTES_PER_MIB, + ) + assert application.nodes["mProject_0001"][ + "workflow_output_size_mib" + ] == pytest.approx( + 200 / _BYTES_PER_MIB, + ) + + assert application.nodes["mDiffFit_0002"]["storage"] == pytest.approx( + 500 / _BYTES_PER_MIB, + ) + assert application["mProject_0001"]["mDiffFit_0002"]["bandwidth"] == pytest.approx( + 200 / _BYTES_PER_MIB, + ) + assert application["mProject_0001"]["mDiffFit_0002"][ + "workflow_transferred_size_mib" + ] == pytest.approx(200 / _BYTES_PER_MIB) + assert application["mProject_0001"]["mDiffFit_0002"]["weight"] == 1 + + assert "latency" in application["mProject_0001"]["mDiffFit_0002"] + assert "availability" in application.nodes["mProject_0001"] + + +def test_get_workflow_uses_family_minimum_when_num_tasks_is_omitted( + monkeypatch: pytest.MonkeyPatch, +): + calls = _install_fake_wfcommons(monkeypatch) + + get_workflow(workflow="genome") + + assert calls["recipe_kwargs"]["num_tasks"] == 54 + + +def test_get_workflow_allows_custom_application_id_and_flows( + monkeypatch: pytest.MonkeyPatch, +): + _install_fake_wfcommons(monkeypatch) + + application = get_workflow( + workflow="montage", + application_id="custom-workflow", + flows=[["mProject_0001", "mAdd_0003"]], + ) + + assert application.id == "custom-workflow" + assert application.flows == [["mProject_0001", "mAdd_0003"]] + + +def test_get_workflow_rejects_too_small_num_tasks( + monkeypatch: pytest.MonkeyPatch, +): + _install_fake_wfcommons(monkeypatch) + + with pytest.raises( + ValueError, + match=re.escape("Workflow family 'genome' requires num_tasks >= 54, got 10."), + ): + get_workflow("genome", num_tasks=10) + + +def test_get_workflow_requires_wfcommons(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "eclypse.builders.workflow._helpers._require_module", + lambda *args, **kwargs: (_ for _ in ()).throw( + ImportError( + "wfcommons is not installed. Please install it with 'pip install wfcommons'." + ) + ), + ) + + with pytest.raises(ImportError, match="pip install wfcommons"): + get_workflow("montage") diff --git a/tests/unit/graph/assets/test_assets.py b/tests/unit/graph/assets/test_assets.py index 4c7dc46..7433167 100644 --- a/tests/unit/graph/assets/test_assets.py +++ b/tests/unit/graph/assets/test_assets.py @@ -13,6 +13,8 @@ Symbolic, ) from eclypse.graph.assets.defaults import ( + bandwidth, + cpu, get_default_edge_assets, get_default_node_assets, get_default_path_aggregators, @@ -37,6 +39,11 @@ def test_asset_init_supports_primitive_callable_and_asset_space(): assert choice_asset._init(rnd) in {3, 4} # pylint: disable=protected-access +def test_asset_init_rejects_unsupported_init_type(): + with pytest.raises(TypeError, match="Unsupported type for `init_fn`"): + Additive(0, 10, object()) + + @pytest.mark.parametrize( ("asset", "values", "expected"), [ @@ -50,6 +57,12 @@ def test_numeric_asset_aggregation(asset, values, expected): assert asset.aggregate(*values) == expected +def test_multiplicative_asset_aggregation_without_values_returns_lower_bound(): + asset = Multiplicative(0.5, 10) + + assert asset.aggregate() == 0.5 + + @pytest.mark.parametrize( ("asset", "featured", "required", "consistent", "inconsistent"), [ @@ -121,6 +134,13 @@ def test_asset_bucket_aggregates_validates_consumes_and_flips(): assert isinstance(bucket.flip()["latency"], Convex) +def test_asset_bucket_rejects_non_asset_values(): + bucket = AssetBucket() + + with pytest.raises(ValueError, match="Asset cpu is not an instance of Asset"): + bucket["cpu"] = 3 + + def test_default_asset_factories_expose_expected_keys(): node_assets = get_default_node_assets() edge_assets = get_default_edge_assets() @@ -132,3 +152,40 @@ def test_default_asset_factories_expose_expected_keys(): assert {"latency", "bandwidth"} == set(edge_assets) assert path_aggregators["latency"]([1, 2, 3]) == 6 assert path_aggregators["bandwidth"]([8, 3, 5]) == 3 + + +def test_default_asset_getters_define_default_initialisers(): + rnd = random.Random(1) + node_assets = get_default_node_assets() + edge_assets = get_default_edge_assets() + + assert node_assets["cpu"]._init(rnd) in {2**i for i in range(1, 9)} # pylint: disable=protected-access + assert 0.99 <= node_assets["availability"]._init(rnd) <= 1.0 # pylint: disable=protected-access + assert 1 <= edge_assets["latency"]._init(rnd) <= 40 # pylint: disable=protected-access + assert 50 <= edge_assets["bandwidth"]._init(rnd) <= 1500 # pylint: disable=protected-access + + +def test_default_asset_getters_can_return_bare_assets_without_initialisers(): + node_assets = get_default_node_assets(with_init=False) + edge_assets = get_default_edge_assets(with_init=False) + + assert all(asset.init_fn is None for asset in node_assets.values()) + assert all(asset.init_fn is None for asset in edge_assets.values()) + + +def test_asset_factories_without_init_use_asset_defaults(): + rnd = random.Random(1) + + assert cpu().init_fn is None + assert bandwidth().init_fn is None + with pytest.raises(ValueError, match="init_fn"): + cpu()._init(rnd) # pylint: disable=protected-access + + +def test_asset_string_representation_includes_main_fields(): + rendered = str(Additive(0, 10, 5)) + + assert "Type: Additive" in rendered + assert "Lower Bound: 0" in rendered + assert "Upper Bound: 10" in rendered + assert "Functional: True" in rendered diff --git a/tests/unit/graph/test_application.py b/tests/unit/graph/test_application.py index ac8b7bc..2435a8f 100644 --- a/tests/unit/graph/test_application.py +++ b/tests/unit/graph/test_application.py @@ -6,10 +6,15 @@ from eclypse.remote.service.service import Service +class ConcreteService(Service): + async def step(self): + return None + + def test_application_add_service_and_set_flows(): app = Application("demo") - gateway = Service("gateway") - worker = Service("worker") + gateway = ConcreteService("gateway") + worker = ConcreteService("worker") app.add_service(gateway) app.add_service(worker) @@ -17,14 +22,14 @@ def test_application_add_service_and_set_flows(): app.set_flows() assert app.flows == [["gateway", "worker"]] - assert app.has_logic + assert app.has_service_implementations with pytest.raises(TypeError): app.add_service("not-a-service") # type: ignore[arg-type] def test_application_rejects_reassigning_service_to_another_app(): - gateway = Service("gateway") + gateway = ConcreteService("gateway") first = Application("first") second = Application("second") @@ -36,15 +41,15 @@ def test_application_rejects_reassigning_service_to_another_app(): def test_application_set_flows_handles_missing_gateway_and_missing_path(): app = Application("demo") - worker = Service("worker") - helper = Service("helper") + worker = ConcreteService("worker") + helper = ConcreteService("helper") app.add_service(worker) app.add_service(helper) app.set_flows() assert app.flows == [] - gateway = Service("gateway") + gateway = ConcreteService("gateway") app.add_service(gateway) app.add_edge("gateway", "worker") app.set_flows() @@ -56,4 +61,4 @@ def test_application_detects_missing_service_logic(): app = Application("broken") app.add_node("orphan") - assert not app.has_logic + assert not app.has_service_implementations diff --git a/tests/unit/graph/test_asset_graph.py b/tests/unit/graph/test_asset_graph.py index 0525f1d..d3938c5 100644 --- a/tests/unit/graph/test_asset_graph.py +++ b/tests/unit/graph/test_asset_graph.py @@ -2,6 +2,7 @@ import pytest +import eclypse.graph.asset_graph as asset_graph_module from eclypse.graph.asset_graph import AssetGraph from eclypse.graph.assets import Additive @@ -27,6 +28,81 @@ def test_asset_graph_validates_nodes_edges_and_dynamic_flags(): graph.add_edge("missing", "a", bandwidth=1) +def test_asset_graph_handles_invalid_init_and_non_strict_violations(monkeypatch): + messages: list[tuple[str, str]] = [] + traces: list[dict[str, object]] = [] + + class DummyBoundLogger: + def warning(self, message: str): + messages.append(("warning", message)) + + def debug(self, message: str): + messages.append(("debug", message)) + + def trace(self, message: str): + messages.append(("trace", message)) + + class DummyLogger: + def bind(self, **_kwargs): + return DummyBoundLogger() + + monkeypatch.setattr(asset_graph_module, "logger", DummyLogger()) + monkeypatch.setattr( + asset_graph_module, + "log_assets_violations", + lambda _logger, _bucket, violations: traces.append(violations), + ) + + with pytest.raises(ValueError, match="attr_init can be 'min' or 'max'"): + AssetGraph("invalid", attr_init="mid") # type: ignore[arg-type] + + graph = AssetGraph( + "warnings", + node_assets={"cpu": Additive(0, 10)}, + edge_assets={"bandwidth": Additive(0, 10)}, + update_policies=[lambda current: current.nodes["a"].update(cpu=2)], + ) + graph.add_node("a", cpu=11, strict=False) + graph.add_node("b", cpu=2) + graph.add_edge("a", "b", bandwidth=11, strict=False) + graph.evolve() + + assert any("a has inconsistent assets" in message for _, message in messages) + assert any("(a -> b) has inconsistent assets" in message for _, message in messages) + assert any("Applying 1 update policies." in message for _, message in messages) + assert traces == [ + {"cpu": 11}, + {"bandwidth": 11}, + ] + + +def test_asset_graph_rejects_missing_edge_target(): + graph = AssetGraph( + "targets", + node_assets={"cpu": Additive(0, 10)}, + edge_assets={"bandwidth": Additive(0, 10)}, + ) + graph.add_node("a", cpu=5) + + with pytest.raises(ValueError, match="Node b not found in the graph"): + graph.add_edge("a", "b", bandwidth=1) + + +def test_asset_graph_rejects_strict_edge_violations_and_allows_static_evolve(): + graph = AssetGraph( + "strict-edge", + node_assets={"cpu": Additive(0, 10)}, + edge_assets={"bandwidth": Additive(0, 10)}, + ) + graph.add_node("a", cpu=1) + graph.add_node("b", cpu=2) + + with pytest.raises(ValueError, match=r"\(a -> b\) has inconsistent assets"): + graph.add_edge("a", "b", bandwidth=11) + + graph.evolve() + + def test_asset_graph_evolve_runs_registered_policies(): graph = AssetGraph( "dynamic", diff --git a/tests/unit/graph/test_infrastructure.py b/tests/unit/graph/test_infrastructure.py index 9b6068e..706279c 100644 --- a/tests/unit/graph/test_infrastructure.py +++ b/tests/unit/graph/test_infrastructure.py @@ -9,7 +9,6 @@ _cost_changed, _default_weight_function, ) -from eclypse.placement.strategies import StaticStrategy def test_infrastructure_path_resources_and_cache_behaviour(sample_infrastructure): @@ -62,7 +61,7 @@ def test_infrastructure_contains_and_helper_functions(sample_infrastructure): requirements.add_node("edge-b", cpu=1) requirements.add_edge("edge-a", "edge-b", latency=0, bandwidth=1000, strict=False) - not_respected = sample_infrastructure.contains(requirements) + not_respected = sample_infrastructure.validate(requirements) assert "edge-a" in not_respected assert "edge-b" in not_respected @@ -70,13 +69,8 @@ def test_infrastructure_contains_and_helper_functions(sample_infrastructure): assert _cost_changed(10, 0) -def test_infrastructure_same_node_resources_and_strategy_flag(sample_infrastructure): +def test_infrastructure_same_node_resources(sample_infrastructure): assert sample_infrastructure.path_resources("edge-a", "edge-a") == ( sample_infrastructure.edge_assets.upper_bound ) assert sample_infrastructure.processing_time("edge-a", "edge-a") == 0.0 - assert sample_infrastructure.has_strategy is False - - sample_infrastructure.strategy = StaticStrategy({"gateway": "edge-a"}) - - assert sample_infrastructure.has_strategy is True diff --git a/tests/unit/placement/strategies/test_strategies.py b/tests/unit/placement/strategies/test_strategies.py index 00445e6..9f9b9e1 100644 --- a/tests/unit/placement/strategies/test_strategies.py +++ b/tests/unit/placement/strategies/test_strategies.py @@ -6,8 +6,8 @@ RandomStrategy, RoundRobinStrategy, StaticStrategy, + PlacementStrategy, ) -from eclypse.placement.strategies.strategy import PlacementStrategy from eclypse.placement.view import PlacementView from eclypse.utils.constants import RND_SEED diff --git a/tests/unit/placement/test_manager.py b/tests/unit/placement/test_manager.py index aec0f28..ddd8392 100644 --- a/tests/unit/placement/test_manager.py +++ b/tests/unit/placement/test_manager.py @@ -13,10 +13,10 @@ def test_manager_generate_mapping_uses_global_strategy_and_enact_resets( sample_infrastructure, sample_application, ): - sample_infrastructure.strategy = StaticStrategy( - {"gateway": "edge-a", "worker": "edge-b"} + manager = PlacementManager( + sample_infrastructure, + default_strategy=StaticStrategy({"gateway": "edge-a", "worker": "edge-b"}), ) - manager = PlacementManager(sample_infrastructure) manager.register(sample_application) placement = manager.get(sample_application.id) @@ -92,7 +92,7 @@ def test_manager_handles_empty_mappings_and_respected_audits( ): manager = PlacementManager(sample_infrastructure) standalone = Placement(sample_infrastructure, sample_application) - sample_infrastructure.strategy = SimpleNamespace( + manager.default_strategy = SimpleNamespace( place=lambda *_args, **_kwargs: {"gateway": None, "worker": None} ) monkeypatch.setattr("eclypse.placement._manager.logger", dummy_logger) diff --git a/tests/unit/policies/degrade/test_degrade.py b/tests/unit/policies/degrade/test_degrade.py index 8b895c6..9485bbe 100644 --- a/tests/unit/policies/degrade/test_degrade.py +++ b/tests/unit/policies/degrade/test_degrade.py @@ -134,3 +134,52 @@ def test_increase_and_reduce_can_be_composed_explicitly(): assert graph.nodes["a"]["ram"] == 64 assert graph.edges["a", "b"]["bandwidth"] == 25 assert graph.edges["a", "b"]["latency"] == 20 + + +def test_additional_degrade_policies_transform_values(): + graph = build_graph() + + policies.degrade.set_value( + 10, + node_assets="cpu", + edge_values={"bandwidth": 40}, + )(graph) + assert graph.nodes["a"]["cpu"] == 10 + assert graph.edges["a", "b"]["bandwidth"] == 40 + + policies.degrade.scale(2, node_assets="cpu")(graph) + assert graph.nodes["a"]["cpu"] == 20 + + policies.degrade.decay(0.5, edge_assets="bandwidth")(graph) + assert graph.edges["a", "b"]["bandwidth"] == 20 + + policies.degrade.clamp_values(upper=15, node_assets="cpu")(graph) + assert graph.nodes["a"]["cpu"] == 15 + + ramp = policies.degrade.ramp_to(60, epochs=2, node_assets="cpu") + ramp(graph) + ramp(graph) + assert graph.nodes["a"]["cpu"] == 60 + + graph.nodes["a"]["cpu"] = 10 + restore = policies.degrade.restore( + epochs=2, + node_assets="cpu", + node_values={"cpu": 80}, + ) + restore(graph) + restore(graph) + assert graph.nodes["a"]["cpu"] == 80 + + +def test_additional_degrade_policies_validate_inputs(): + with pytest.raises(ValueError): + policies.degrade.scale(1.0) + with pytest.raises(ValueError): + policies.degrade.decay(1.5, node_assets="cpu") + with pytest.raises(ValueError): + policies.degrade.clamp_values(lower=2, upper=1, node_assets="cpu") + with pytest.raises(ValueError): + policies.degrade.ramp_to(1, epochs=0, node_assets="cpu") + with pytest.raises(ValueError): + policies.degrade.restore(epochs=0, node_assets="cpu") diff --git a/tests/unit/policies/distribution/test_distribution.py b/tests/unit/policies/distribution/test_distribution.py index bcc26cc..f230117 100644 --- a/tests/unit/policies/distribution/test_distribution.py +++ b/tests/unit/policies/distribution/test_distribution.py @@ -323,3 +323,65 @@ def test_categorical_distribution_validates_inputs(): with pytest.raises(ValueError): policies.distribution.categorical(node_asset_weights={"cpu": [1.0]}) + + +def test_new_distribution_policies_apply_numeric_multipliers(): + graph = build_graph() + + policies.distribution.constant( + node_assets="cpu", + edge_assets="bandwidth", + node_distribution=2.0, + edge_distribution=0.5, + )(graph) + assert graph.nodes["a"]["cpu"] == 160 + assert graph.edges["a", "b"]["bandwidth"] == 50 + + policies.distribution.bernoulli( + node_assets="ram", + node_distribution=(1.0, 2.0, 0.0), + )(graph) + assert graph.nodes["a"]["ram"] == 64 + + policies.distribution.empirical(node_assets="cpu", node_distribution=[0.5])(graph) + assert graph.nodes["a"]["cpu"] == 80 + + policies.distribution.discrete( + edge_assets="latency", + edge_distribution=[(3.0, 1.0)], + )(graph) + assert graph.edges["a", "b"]["latency"] == 30 + + +def test_new_distribution_policies_validate_and_use_seeded_rng(): + with pytest.raises(ValueError): + policies.distribution.bernoulli( + node_assets="cpu", node_distribution=(1.5, 1, 0) + ) + with pytest.raises(ValueError): + policies.distribution.exponential(node_assets="cpu", node_distribution=0) + with pytest.raises(ValueError): + policies.distribution.poisson(node_assets="cpu", node_distribution=-1) + with pytest.raises(ValueError): + policies.distribution.pareto(node_assets="cpu", node_distribution=0) + with pytest.raises(ValueError): + policies.distribution.weibull(node_assets="cpu", node_distribution=(0, 1)) + with pytest.raises(ValueError): + policies.distribution.empirical(node_assets="cpu", node_distribution=[]) + with pytest.raises(ValueError): + policies.distribution.discrete(node_assets="cpu", node_distribution=[(1, 0)]) + + first_graph = build_graph() + second_graph = build_graph() + for builder, distribution in [ + (policies.distribution.exponential, 1.0), + (policies.distribution.poisson, 2.0), + (policies.distribution.pareto, 2.0), + (policies.distribution.weibull, (1.0, 2.0)), + ]: + first_policy = builder(node_assets="cpu", node_distribution=distribution) + second_policy = builder(node_assets="cpu", node_distribution=distribution) + first_policy(first_graph) + second_policy(second_graph) + + assert first_graph.nodes["a"]["cpu"] == second_graph.nodes["a"]["cpu"] diff --git a/tests/unit/policies/failure/test_failure.py b/tests/unit/policies/failure/test_failure.py index 71a03eb..4423edf 100644 --- a/tests/unit/policies/failure/test_failure.py +++ b/tests/unit/policies/failure/test_failure.py @@ -55,3 +55,43 @@ def test_failure_policy_validation_and_alternative_branches(): policies.failure.latency_spike(1.0, factor=2.0)(graph) assert graph.edges["a", "b"]["latency"] == 20 + + +def test_edge_and_correlated_failure_policies(): + graph = build_graph() + graph.edges["a", "b"]["availability"] = 1.0 + graph.nodes["a"]["zone"] = "z1" + graph.nodes["b"]["zone"] = "z1" + + policies.failure.kill_edges(1.0)(graph) + assert graph.edges["a", "b"]["availability"] == 0.0 + + policies.failure.revive_edges(1.0)(graph) + assert graph.edges["a", "b"]["availability"] == 1.0 + + policies.failure.edge_availability_flap(1.0)(graph) + assert graph.edges["a", "b"]["availability"] == 0.0 + + policies.failure.correlated_failure(1.0, group_key="zone")(graph) + assert graph.nodes["a"]["availability"] == 0.0 + assert graph.nodes["b"]["availability"] == 0.0 + + +def test_partition_brownout_and_resource_exhaustion_policies(): + graph = build_graph() + graph.add_node("c", cpu=20, ram=8, availability=1.0) + graph.add_edge("b", "c", latency=30, bandwidth=50, availability=1.0) + graph.edges["a", "b"]["availability"] = 1.0 + + policies.failure.network_partition([["a"], ["b", "c"]])(graph) + assert graph.edges["a", "b"]["availability"] == 0.0 + assert graph.edges["b", "c"]["availability"] == 1.0 + + policies.failure.resource_exhaustion(1.0, factor=0.5, node_assets="cpu")(graph) + assert graph.nodes["a"]["cpu"] == 40 + + policies.failure.brownout(1.0, factor=0.5, edge_assets="bandwidth")(graph) + assert graph.edges["a", "b"]["bandwidth"] == 50 + + with pytest.raises(ValueError): + policies.failure.network_partition([["a"]]) diff --git a/tests/unit/policies/noise/test_noise.py b/tests/unit/policies/noise/test_noise.py index dd9d8e5..d6407cd 100644 --- a/tests/unit/policies/noise/test_noise.py +++ b/tests/unit/policies/noise/test_noise.py @@ -129,3 +129,53 @@ def test_impulse_validation(): with pytest.raises(ValueError): policies.noise.impulse(node_assets="cpu", node_factor_range=(2.0, 1.0)) + + +def test_additional_noise_policies_apply_expected_changes(): + graph = build_graph() + + policies.noise.additive_jitter(node_ranges={"cpu": (5, 5)})(graph) + assert graph.nodes["a"]["cpu"] == 85 + + policies.noise.gaussian_jitter(edge_parameters={"latency": (5, 0)})(graph) + assert graph.edges["a", "b"]["latency"] == 15 + + policies.noise.multiplicative_jitter( + node_assets="cpu", + node_factor_range=(2, 2), + )(graph) + assert graph.nodes["a"]["cpu"] == 170 + + policies.noise.correlated_noise( + node_assets=["cpu", "ram"], + delta_range=(1, 1), + )(graph) + assert graph.nodes["a"]["cpu"] == 171 + assert graph.nodes["a"]["ram"] == 33 + + policies.noise.dropout(node_assets="cpu", probability=1.0, value=0)(graph) + assert graph.nodes["a"]["cpu"] == 0 + + +def test_seasonal_noise_and_validation_paths(): + graph = build_graph() + policy = policies.noise.seasonal_noise( + amplitude=10, + period=4, + node_assets="cpu", + ) + policy(graph) + policy(graph) + + assert graph.nodes["a"]["cpu"] == 90 + + with pytest.raises(ValueError): + policies.noise.additive_jitter() + with pytest.raises(ValueError): + policies.noise.gaussian_jitter(node_parameters={"cpu": (0, -1)}) + with pytest.raises(ValueError): + policies.noise.correlated_noise(delta_range=(2, 1), node_assets="cpu") + with pytest.raises(ValueError): + policies.noise.seasonal_noise(amplitude=1, period=0, node_assets="cpu") + with pytest.raises(ValueError): + policies.noise.dropout() diff --git a/tests/unit/policies/replay/test_replay.py b/tests/unit/policies/replay/test_replay.py index d606c8d..72dcf83 100644 --- a/tests/unit/policies/replay/test_replay.py +++ b/tests/unit/policies/replay/test_replay.py @@ -158,3 +158,73 @@ def test_replay_filters_start_step_and_edge_missing_behaviour(): ) with pytest.raises(KeyError): failing_edge_policy(graph) + + +def test_replay_cyclic_graph_mapping_events_and_interpolation(monkeypatch): + graph = build_graph() + + cyclic = policies.replay.replay_nodes( + [ + {"time": 0, "node_id": "a", "cpu": 1}, + {"time": 1, "node_id": "a", "cpu": 2}, + ], + value_columns=["cpu"], + cyclic=True, + ) + cyclic(graph) + cyclic(graph) + cyclic(graph) + assert graph.nodes["a"]["cpu"] == 1 + + graph_policy = policies.replay.replay_graph( + node_records=[{"time": 0, "node_id": "a", "ram": 44}], + edge_records=[{"time": 0, "source": "a", "target": "b", "latency": 22}], + node_value_columns=["ram"], + edge_value_columns=["latency"], + ) + graph_policy(graph) + assert graph.nodes["a"]["ram"] == 44 + assert graph.edges["a", "b"]["latency"] == 22 + + mapped = policies.replay.replay_with_mapping( + [{"time": 0, "external": "A", "value": 33}], + target="nodes", + column_mapping={"external": "node_id", "value": "cpu"}, + id_mapping={"A": "a"}, + value_columns=["cpu"], + ) + mapped(graph) + assert graph.nodes["a"]["cpu"] == 33 + + events = policies.replay.replay_events( + [ + { + "time": 0, + "policy": lambda target_graph: target_graph.nodes["a"].update(cpu=12), + } + ] + ) + events(graph) + assert graph.nodes["a"]["cpu"] == 12 + + interpolated = policies.replay.interpolated_replay( + [ + {"time": 0, "node_id": "a", "cpu": 0}, + {"time": 2, "node_id": "a", "cpu": 20}, + ], + target="nodes", + value_columns=["cpu"], + ) + interpolated(graph) + interpolated(graph) + assert graph.nodes["a"]["cpu"] == 10 + + class FakePandas: + @staticmethod + def read_csv(path): + assert path == "trace.csv" + return FakeDataFrame([{"time": 0, "node_id": "a", "cpu": 5}]) + + monkeypatch.setitem(__import__("sys").modules, "pandas", FakePandas) + policies.replay.from_csv("trace.csv", target="nodes", value_columns=["cpu"])(graph) + assert graph.nodes["a"]["cpu"] == 5 diff --git a/tests/unit/policies/schedule/test_schedule.py b/tests/unit/policies/schedule/test_schedule.py index 339f43c..a96c7b1 100644 --- a/tests/unit/policies/schedule/test_schedule.py +++ b/tests/unit/policies/schedule/test_schedule.py @@ -60,3 +60,44 @@ def noop(_graph): with pytest.raises(ValueError): policies.once_at(-1, noop) + + with pytest.raises(ValueError): + policies.at([], noop) + + with pytest.raises(ValueError): + policies.until(-1, noop) + + with pytest.raises(ValueError): + policies.repeat(-1, noop) + + with pytest.raises(ValueError): + policies.with_probability(1.5, noop) + + with pytest.raises(ValueError): + policies.jittered_every(0, noop) + + with pytest.raises(ValueError): + policies.cooldown(-1, noop) + + +def test_additional_schedule_wrappers_control_policy_timing(): + graph = AssetGraph("scheduled", node_assets={"cpu": Additive(0, 100)}) + graph.add_node("a", cpu=0) + + def increment(target_graph): + target_graph.nodes["a"]["cpu"] += 1 + + wrappers = [ + policies.at([1, 3], increment), + policies.until(1, increment), + policies.repeat(2, increment), + policies.with_probability(1.0, increment), + policies.jittered_every(2, increment, jitter=0), + policies.cooldown(1, increment), + ] + + for _ in range(4): + for wrapper in wrappers: + wrapper(graph) + + assert graph.nodes["a"]["cpu"] == 14 diff --git a/tests/unit/policies/test_new_families.py b/tests/unit/policies/test_new_families.py new file mode 100644 index 0000000..cdad884 --- /dev/null +++ b/tests/unit/policies/test_new_families.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import pytest + +from eclypse import policies +from tests.unit.policies._helpers import build_graph + + +def test_compose_family_combines_policies(): + graph = build_graph() + + def add_cpu(target_graph): + target_graph.nodes["a"].update(cpu=81) + + def add_ram(target_graph): + target_graph.nodes["a"].update(ram=33) + + policies.compose.chain(add_cpu, add_ram)(graph) + assert graph.nodes["a"]["cpu"] == 81 + assert graph.nodes["a"]["ram"] == 33 + + policies.compose.conditional(lambda _: True, add_cpu)(graph) + assert graph.nodes["a"]["cpu"] == 81 + + policies.compose.one_of(add_cpu)(graph) + policies.compose.weighted_choice([add_ram], [1.0])(graph) + assert graph.nodes["a"]["ram"] == 33 + + with pytest.raises(ValueError): + policies.compose.one_of() + with pytest.raises(ValueError): + policies.compose.weighted_choice([add_cpu], [0.0]) + + +def test_workload_family_updates_load_values(): + graph = build_graph() + graph.nodes["a"]["users"] = 0 + graph.edges["a", "b"]["traffic"] = 0 + + policies.workload.arrival_process(2, node_assets="users")(graph) + assert graph.nodes["a"]["users"] >= 0 + + policies.workload.traffic_matrix({("a", "b"): 12})(graph) + assert graph.edges["a", "b"]["traffic"] == 12 + + policy = policies.workload.diurnal_load( + amplitude=1, + period=4, + node_assets="cpu", + ) + policy(graph) + policy(graph) + assert graph.nodes["a"]["cpu"] >= 80 + + with pytest.raises(ValueError): + policies.workload.arrival_process(-1, node_assets="users") + + +def test_topology_family_mutates_graph_structure(): + graph = build_graph() + + policies.topology.add_node("c", cpu=1, ram=1, availability=1.0)(graph) + assert graph.has_node("c") + + policies.topology.add_edge("b", "c", latency=1, bandwidth=1)(graph) + assert graph.has_edge("b", "c") + + policies.topology.rewire([("a", "b")], probability=1.0)(graph) + assert not graph.has_edge("a", "b") + + policies.topology.churn( + add_probability=1.0, + candidate_nodes={"d": {"cpu": 1, "ram": 1, "availability": 1.0}}, + )(graph) + assert graph.has_node("d") + + policies.topology.remove_node("d")(graph) + assert not graph.has_node("d") + + +def test_constraints_family_enforces_numeric_invariants(): + graph = build_graph() + + policies.constraints.clamp_values(upper=60, node_assets="cpu")(graph) + assert graph.nodes["a"]["cpu"] == 60 + + policies.constraints.round_int(node_assets="availability")(graph) + assert graph.nodes["a"]["availability"] == 1 + + policies.constraints.ensure_capacity_floor(70, edge_assets="bandwidth")(graph) + assert graph.edges["a", "b"]["bandwidth"] == 100 + + policies.constraints.normalise(100, node_assets="cpu")(graph) + assert graph.nodes["a"]["cpu"] + graph.nodes["b"]["cpu"] == 100 diff --git a/tests/unit/remote/communication/test_communication_and_services.py b/tests/unit/remote/communication/test_communication_and_services.py index 6603fb4..1b54717 100644 --- a/tests/unit/remote/communication/test_communication_and_services.py +++ b/tests/unit/remote/communication/test_communication_and_services.py @@ -52,6 +52,11 @@ async def step(self): return "step" +class DummyRESTRuntime(RESTService): + async def step(self): + return None + + class DemoRESTHandlers: def __init__(self): self.id = "svc" @@ -581,7 +586,7 @@ def fanout(self): assert service.logger == {"id": "worker"} assert service.deployed is True - rest_runtime = RESTService("frontend") + rest_runtime = DummyRESTRuntime("frontend") rest_runtime.attach_node(node) with pytest.raises(RuntimeError, match="not mpi"): diff --git a/tests/unit/remote/service/test_service_runtime.py b/tests/unit/remote/service/test_service_runtime.py index 229cff6..da19cc0 100644 --- a/tests/unit/remote/service/test_service_runtime.py +++ b/tests/unit/remote/service/test_service_runtime.py @@ -129,7 +129,7 @@ def _make_node(dummy_logger): def test_service_guard_properties_and_basic_accessors(dummy_logger): with pytest.raises(ValueError, match="Invalid communication interface"): - Service("broken", communication_interface="grpc") # type: ignore[arg-type] + ScriptedService("broken", [1], communication_interface="grpc") # type: ignore[arg-type] service = ScriptedService("worker", [1], store_step=True) @@ -143,8 +143,8 @@ def test_service_guard_properties_and_basic_accessors(dummy_logger): service.node with pytest.raises(ValueError, match="Application ID not set"): service.full_id - with pytest.raises(NotImplementedError, match="must be overridden"): - asyncio.run(Service("base").step()) + with pytest.raises(TypeError, match="abstract"): + Service("base") # type: ignore[abstract] node = _make_node(dummy_logger) service.attach_node(node) @@ -316,7 +316,7 @@ def test_start_loop_handles_forever_cancelled_and_error_paths(monkeypatch): ) monkeypatch.setattr( "eclypse.remote.service.service.print_exception", - lambda exc, label: printed.append((str(exc), label)), + lambda exc, label, _logger: printed.append((str(exc), label)), ) normal_loop = FakeLoop() diff --git a/tests/unit/report/metrics/test_metrics.py b/tests/unit/report/metrics/test_metrics.py index 4c8e68d..0563d0d 100644 --- a/tests/unit/report/metrics/test_metrics.py +++ b/tests/unit/report/metrics/test_metrics.py @@ -26,6 +26,7 @@ seed, step_result, ) +from eclypse.report.metrics import metric from eclypse.utils.constants import ( DRIVING_EVENT, RND_SEED, @@ -92,3 +93,15 @@ def test_simulation_metric_helpers_and_default_metric_list( assert isinstance(time_metric(object()), float) assert step_result(service_with_results) == "first" assert len(get_default_metrics()) >= 10 + + +def test_metric_decorators_share_event_options(): + @metric.link(name="latency_probe", activates_on="step", report="json") + def latency(): + return 1 + + assert latency.name == "latency_probe" + assert latency.type == "link" + assert latency.is_metric + assert latency.report_types == ["json"] + assert latency.triggers diff --git a/tests/unit/report/test_report.py b/tests/unit/report/test_report.py index a3996de..0af8553 100644 --- a/tests/unit/report/test_report.py +++ b/tests/unit/report/test_report.py @@ -107,6 +107,20 @@ def test_report_filter_ignores_unknown_columns( assert report.filter([], service_id=["gateway"]) == [] +def test_report_describe_summarises_available_frames( + csv_report_dir: Path, list_frame_backend +): + report = Report(csv_report_dir, backend=list_frame_backend) + + description = report.describe() + + assert "6 rows x 3 steps x 3 metrics" in description + assert "1 applications" in description + assert "application: 1 rows, 1 metrics" in description + assert "service: 3 rows, 1 metrics" in description + assert "simulation: 2 rows, 1 metrics" in description + + def test_report_config_and_report_format_fallbacks(tmp_path: Path, list_frame_backend): explicit_path = tmp_path / "explicit" (explicit_path / "csv").mkdir(parents=True) diff --git a/tests/unit/simulation/_simulator/test_local.py b/tests/unit/simulation/_simulator/test_local.py index fd95f3d..406c18f 100644 --- a/tests/unit/simulation/_simulator/test_local.py +++ b/tests/unit/simulation/_simulator/test_local.py @@ -188,7 +188,7 @@ async def fake_sleep(_: float): simulator.fire = fake_fire # type: ignore[method-assign] monkeypatch.setattr( "eclypse.simulation._simulator.local.print_exception", - lambda exc, origin: printed.append((str(exc), origin)), + lambda exc, origin, _logger: printed.append((str(exc), origin)), ) monkeypatch.setattr( "eclypse.simulation._simulator.local.asyncio.sleep", diff --git a/tests/unit/simulation/test_config.py b/tests/unit/simulation/test_config.py index bc201f9..703ced3 100644 --- a/tests/unit/simulation/test_config.py +++ b/tests/unit/simulation/test_config.py @@ -27,6 +27,12 @@ def __call__(self, *_args, **_kwargs): def test_simulation_config_normalises_and_serialises(list_frame_backend, tmp_path): + default_config = SimulationConfig( + path=tmp_path / "default-run", + report_backend=list_frame_backend, + ) + assert default_config.step_every_ms == 0.0 + config = SimulationConfig( path=tmp_path / "run", report_backend=list_frame_backend, @@ -53,29 +59,40 @@ def test_simulation_config_rejects_invalid_step_and_duplicate_keys(list_frame_ba def test_require_module_surfaces_install_hint(): - with pytest.raises(ImportError, match="pip install eclypse\\[remote\\]"): - _require_module("module_that_does_not_exist", extras_name="remote") + with pytest.raises(ImportError, match="pip install module_that_does_not_exist"): + _require_module("module_that_does_not_exist") -def test_simulation_config_helper_methods_cover_optional_paths(monkeypatch, tmp_path): - require_calls: list[tuple[str, str | None]] = [] +def test_simulation_config_helper_methods_cover_optional_paths( + monkeypatch, tmp_path, dummy_logger +): + require_calls: list[str] = [] monkeypatch.setattr( "eclypse.simulation.config._require_module", - lambda module, extras_name=None: require_calls.append((module, extras_name)), + lambda module: require_calls.append(module), ) monkeypatch.setattr( "eclypse.simulation.config.strftime", lambda _fmt: "20260407_120000" ) + monkeypatch.setattr("eclypse.simulation.config.logger", dummy_logger) existing_path = tmp_path / "run" existing_path.mkdir() assert SimulationConfig._resolve_step_every_ms(2) == 2 assert SimulationConfig._resolve_step_every_ms(None) is None + assert SimulationConfig._resolve_step_every_ms("manual") is None + assert SimulationConfig._resolve_step_every_ms("auto") == 0.0 + assert SimulationConfig._resolve_step_every_ms("auto", remote=True) is None assert SimulationConfig._resolve_path(existing_path) == Path( f"{existing_path}-20260407_120000" ) + assert any( + "Target path exists; writing to" in args[0] + for level, args in dummy_logger.records + if level == "info" + ) bootstrap = RemoteBootstrap() assert SimulationConfig._resolve_remote(bootstrap) is bootstrap @@ -92,10 +109,10 @@ def test_simulation_config_helper_methods_cover_optional_paths(monkeypatch, tmp_ config._ensure_optional_dependencies() assert require_calls == [ - ("tensorboardX", "tboard"), - ("polars", None), - ("ray", "remote"), - ("pandas", None), + "tensorboardX", + "polars", + "ray", + "pandas", ] require_calls.clear() @@ -103,10 +120,10 @@ def test_simulation_config_helper_methods_cover_optional_paths(monkeypatch, tmp_ config._ensure_optional_dependencies() assert require_calls == [ - ("tensorboardX", "tboard"), - ("polars", None), - ("ray", "remote"), - ("polars", None), + "tensorboardX", + "polars", + "ray", + "polars", ] diff --git a/tests/unit/simulation/test_simulation.py b/tests/unit/simulation/test_simulation.py index 7296eb3..d0dfccb 100644 --- a/tests/unit/simulation/test_simulation.py +++ b/tests/unit/simulation/test_simulation.py @@ -36,6 +36,7 @@ def test_simulation_register_start_step_stop_and_report( sample_application, simulation_config, static_strategy, + dummy_logger, ): event_calls: list[tuple[str, tuple, dict]] = [] @@ -50,6 +51,7 @@ def test_simulation_register_start_step_stop_and_report( ) simulation = Simulation(sample_infrastructure, simulation_config) + monkeypatch.setattr(simulation, "_logger", dummy_logger) simulation.register(sample_application, static_strategy) simulation.start() simulation.step() @@ -58,6 +60,12 @@ def test_simulation_register_start_step_stop_and_report( assert sample_application.id in simulation.applications assert event_calls[0][0] == "start" assert event_calls[1][0] == "trigger" + eclypse_messages = [ + args[0] for level, args in dummy_logger.records if level == "ECLYPSE" + ] + assert any("Simulation configuration | " in msg for msg in eclypse_messages) + assert any("infrastructure=edge-cloud" in msg for msg in eclypse_messages) + assert "Simulation started." in eclypse_messages assert simulation.report == "report-object" assert report_calls assert simulation.path == simulation_config.path @@ -144,6 +152,48 @@ def test_simulation_start_without_path_and_blocking_stop( ] +def test_simulation_run_helpers_and_context_cleanup( + monkeypatch, + sample_infrastructure, + simulation_config, +): + simulation = Simulation(sample_infrastructure, simulation_config) + calls: list[object] = [] + + monkeypatch.setattr(simulation, "start", lambda: calls.append("start")) + monkeypatch.setattr(simulation, "step", lambda: calls.append("step")) + monkeypatch.setattr(simulation, "stop", lambda: calls.append("stop")) + monkeypatch.setattr( + simulation, + "wait", + lambda timeout=None: calls.append(("wait", timeout)), + ) + monkeypatch.setattr( + type(simulation), + "status", + property(lambda _self: SimpleNamespace(name="RUNNING")), + ) + + simulation.run(steps=2) + assert calls == ["start", "step", "step", "stop"] + + calls.clear() + simulation.run(seconds=0.5) + assert calls == ["start", ("wait", 0.5), "stop"] + + with pytest.raises(ValueError, match="Only one"): + simulation.run(steps=1, seconds=1) + with pytest.raises(ValueError, match="steps"): + simulation.run(steps=-1) + with pytest.raises(ValueError, match="seconds"): + simulation.run(seconds=-1) + + calls.clear() + with simulation as managed: + assert managed is simulation + assert calls == ["stop"] + + def test_simulation_remote_paths_and_report_cache( monkeypatch, sample_infrastructure, @@ -200,11 +250,12 @@ def __init__(self): with pytest.raises(ValueError, match="All services must have a logic"): simulation.register( - SimpleNamespace(id="plain-app", has_logic=False), static_strategy + SimpleNamespace(id="plain-app", has_service_implementations=False), + static_strategy, ) -def test_simulation_register_prefers_global_strategy_and_requires_one( +def test_simulation_register_uses_default_strategy_and_requires_one( monkeypatch, sample_infrastructure, sample_application, @@ -214,12 +265,13 @@ def test_simulation_register_prefers_global_strategy_and_requires_one( ): simulation = Simulation(sample_infrastructure, simulation_config) - with pytest.raises(ValueError, match="Must provide a global placement strategy"): + with pytest.raises(ValueError, match="Must provide a default placement strategy"): simulation.register(sample_application) - sample_infrastructure.strategy = static_strategy + simulation_config.default_strategy = static_strategy + simulation = Simulation(sample_infrastructure, simulation_config) monkeypatch.setattr(simulation, "_logger", dummy_logger) - simulation.register(sample_application, static_strategy) + simulation.register(sample_application) assert sample_application.id in simulation.applications - assert any(level == "warning" for level, _ in dummy_logger.records) + assert not any(level == "warning" for level, _ in dummy_logger.records) diff --git a/tests/unit/test_pyproject_scripts.py b/tests/unit/test_pyproject_scripts.py new file mode 100644 index 0000000..73ed0d0 --- /dev/null +++ b/tests/unit/test_pyproject_scripts.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import ast +import tomllib +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] + +EXPECTED_EXAMPLE_SCRIPTS = { + "echo": "examples.echo.main:main", + "grid-analysis": "examples.grid_analysis.main:main", + "image-prediction": "examples.image_prediction.main:main", + "off-the-shelf": "examples.off_the_shelf.main:main", + "sock-shop-mpi": "examples.sock_shop.mpi:main", + "sock-shop-rest": "examples.sock_shop.rest:main", + "user-distribution": "examples.user_distribution.main:main", +} + + +def test_example_poetry_scripts_target_main_functions(): + with (REPO_ROOT / "pyproject.toml").open("rb") as pyproject: + config = tomllib.load(pyproject) + + scripts = config["project"]["scripts"] + for script, target in EXPECTED_EXAMPLE_SCRIPTS.items(): + assert scripts[script] == target + + +def test_example_script_targets_define_main_functions(): + for target in EXPECTED_EXAMPLE_SCRIPTS.values(): + module_name, function_name = target.split(":") + module_path = REPO_ROOT.joinpath(*module_name.split(".")).with_suffix(".py") + + tree = ast.parse(module_path.read_text(), filename=str(module_path)) + assert any( + isinstance(node, ast.FunctionDef) and node.name == function_name + for node in tree.body + ) diff --git a/tests/unit/utils/test_tools_and_logging.py b/tests/unit/utils/test_tools_and_logging.py index efc6a8a..f3a451b 100644 --- a/tests/unit/utils/test_tools_and_logging.py +++ b/tests/unit/utils/test_tools_and_logging.py @@ -4,6 +4,7 @@ from eclypse.utils._logging import ( _is_eclypse, + _is_eclypse_exception, _is_not_eclypse, config_logger, log_assets_violations, @@ -18,7 +19,6 @@ def test_logging_helpers_configure_and_format_messages( monkeypatch, - capsys, tmp_path: Path, dummy_logger, sample_infrastructure, @@ -36,18 +36,26 @@ def test_logging_helpers_configure_and_format_messages( handlers = configured["handlers"] assert isinstance(handlers, list) - assert len(handlers) == 3 + assert len(handlers) == 4 assert _is_eclypse({"level": type("Level", (), {"name": "ECLYPSE"})()}) is True + assert ( + _is_eclypse_exception( + {"level": type("Level", (), {"name": "ECLYPSE_EXCEPTION"})()} + ) + is True + ) assert _is_not_eclypse({"level": type("Level", (), {"name": "INFO"})()}) is True try: raise ValueError("broken") except ValueError as exc: - print_exception(exc, "worker") + print_exception(exc, "worker", dummy_logger) - output = capsys.readouterr().out - assert "Traceback (most recent call last):" in output - assert "ValueError in worker: broken" in output + exception_messages = [ + args[0] for level, args in dummy_logger.records if level == "ECLYPSE_EXCEPTION" + ] + assert "Traceback (most recent call last):" in exception_messages[0] + assert "ValueError in worker: broken" in exception_messages[0] log_placement_violations( dummy_logger, diff --git a/tests/unit/workflow/event/test_dispatch.py b/tests/unit/workflow/event/test_dispatch.py index 04eb315..0e53912 100644 --- a/tests/unit/workflow/event/test_dispatch.py +++ b/tests/unit/workflow/event/test_dispatch.py @@ -8,7 +8,7 @@ from eclypse.workflow.event import ( EclypseEvent, EventRole, - event, + every, ) from eclypse.workflow.event.event import ( _application_fn, @@ -358,8 +358,8 @@ def test_event_dispatch_by_type_and_runtime_logging( assert any(level == "debug" for level, _ in dummy_logger.records) -def test_event_decorator_wraps_callable_classes(): - @event +def test_scheduled_decorator_wraps_callable_classes(): + @every(ms=1) class CustomCallable: def __call__(self): return {"ok": True} diff --git a/tests/unit/workflow/event/test_wrapper.py b/tests/unit/workflow/event/test_wrapper.py index c5a4ca1..1c53a7e 100644 --- a/tests/unit/workflow/event/test_wrapper.py +++ b/tests/unit/workflow/event/test_wrapper.py @@ -8,9 +8,12 @@ from eclypse.workflow.event import ( EclypseEvent, EventRole, - event, + after, + every, get_default_events, + once_at, ) +from eclypse.workflow.event import decorator as decorator_module from eclypse.workflow.event.event import _application_fn from eclypse.workflow.event.wrapper import EventWrapper from eclypse.workflow.trigger import ( @@ -18,6 +21,7 @@ PeriodicCascadeTrigger, PeriodicTrigger, RandomCascadeTrigger, + ScheduledTrigger, ScheduledCascadeTrigger, ) @@ -30,12 +34,12 @@ def __call__(self, *_args, **_kwargs): return {"value": 1} -def test_event_decorator_wrapper_and_defaults( +def test_scheduled_decorator_wrapper_and_defaults( sample_infrastructure, sample_application ): - @event( + @every( + ms=5, activates_on=["start", ("step", 2), ("start", 1.0), ("start", [1])], - trigger_every_ms=5, verbose=True, remote=True, role=EventRole.METRIC, @@ -87,3 +91,30 @@ def test_wrapper_validation_rejects_invalid_activation_shapes(): with pytest.raises(ValueError, match="Invalid activates_on type"): EventWrapper(lambda: None, "bad", [], activates_on={"step"}) # type: ignore[arg-type] + + +def test_scheduling_decorators_create_expected_triggers(): + @every(ms=25, event_type="simulation") + def heartbeat(): + return {"ok": True} + + @after(sim_seconds=2) + def delayed(): + return {"ok": True} + + @once_at(sim_seconds=3, name="single") + def once(): + return {"ok": True} + + assert heartbeat.type == "simulation" + assert heartbeat.trigger_bucket.max_triggers > 1 + assert any(isinstance(trigger, PeriodicTrigger) for trigger in heartbeat.triggers) + assert any(isinstance(trigger, ScheduledTrigger) for trigger in delayed.triggers) + assert delayed.trigger_bucket.max_triggers == 1 + assert once.name == "single" + assert once.trigger_bucket.max_triggers == 1 + assert any(isinstance(trigger, ScheduledTrigger) for trigger in once.triggers) + + +def test_scheduling_decorators_do_not_expose_scheduled_helper(): + assert not hasattr(decorator_module, "_scheduled_event") diff --git a/tests/unit/workflow/test_package_exports.py b/tests/unit/workflow/test_package_exports.py new file mode 100644 index 0000000..e7463b3 --- /dev/null +++ b/tests/unit/workflow/test_package_exports.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from eclypse import workflow +from eclypse.workflow import event as event_package +from eclypse.workflow.event import decorator as decorator_module + + +def test_workflow_root_reexports_public_primitives(): + assert workflow.get_default_events is not None + assert workflow.EclypseEvent is not None + assert workflow.Trigger is not None + assert workflow.TriggerBucket is not None + assert workflow.every is not None + assert workflow.after is not None + assert workflow.once_at is not None + assert "after" in workflow.__all__ + assert "every" in workflow.__all__ + assert "once_at" in workflow.__all__ + assert "TriggerBucket" in workflow.__all__ + assert "event" not in workflow.__all__ + assert not callable(workflow.event) + assert not callable(event_package.event) + assert not hasattr(decorator_module, "event") diff --git a/tests/unit/workflow/trigger/test_triggers.py b/tests/unit/workflow/trigger/test_triggers.py index 576d7d9..e3f8552 100644 --- a/tests/unit/workflow/trigger/test_triggers.py +++ b/tests/unit/workflow/trigger/test_triggers.py @@ -46,7 +46,7 @@ def test_triggers_and_trigger_bucket(monkeypatch): assert repr(periodic) == "PeriodicTrigger(trigger_every_ms=10)" scheduled = ScheduledTrigger(timedelta(seconds=1)) - scheduled.init() + scheduled.prepare() scheduled._scheduled_times = [datetime.now() - timedelta(seconds=1)] # pylint: disable=protected-access assert scheduled.trigger() assert ( @@ -56,7 +56,7 @@ def test_triggers_and_trigger_bucket(monkeypatch): monkeypatch.setenv("ECLYPSE_RND_SEED", "3") random_trigger = RandomTrigger(1.0) - random_trigger.init() + random_trigger.prepare() assert random_trigger.trigger() assert repr(random_trigger) == "RandomTrigger(probability=1.0)" @@ -80,7 +80,7 @@ def test_triggers_and_trigger_bucket(monkeypatch): ) random_cascade = RandomCascadeTrigger("step", probability=1.0, seed=4) - random_cascade.init() + random_cascade.prepare() assert random_cascade.trigger(trigger_event) assert ( repr(random_cascade) @@ -98,7 +98,7 @@ def test_triggers_and_trigger_bucket(monkeypatch): def test_trigger_helpers_cover_error_and_reset_paths(monkeypatch): dummy = DummyTrigger() - assert dummy.init() is None + assert dummy.prepare() is None assert dummy.reset() is None assert repr(dummy) == "DummyTrigger" @@ -108,9 +108,12 @@ def test_trigger_helpers_cover_error_and_reset_paths(monkeypatch): monkeypatch.setenv("ECLYPSE_RND_SEED", "11") seeded_random = RandomTrigger(0.5) - seeded_random.init() + seeded_random.prepare() assert seeded_random.rnd is not None + scheduled.prepare() + assert scheduled.trigger() is False + uninitialised_random = RandomTrigger(0.5) with pytest.raises(RuntimeError, match="Trigger not initialised"): uninitialised_random.trigger()