diff --git a/.github/workflows/upload_coverage.yaml b/.github/workflows/upload_coverage.yaml index 65c6ee6..428c452 100644 --- a/.github/workflows/upload_coverage.yaml +++ b/.github/workflows/upload_coverage.yaml @@ -20,8 +20,12 @@ on: jobs: test: - name: Run tests and collect coverage + name: Run tests on Python ${{ matrix.python-version }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] steps: - name: Checkout Source Code uses: actions/checkout@v6 @@ -29,7 +33,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: 3.12 + python-version: ${{ matrix.python-version }} - name: Install Dependencies run: make setup-test @@ -38,6 +42,7 @@ jobs: run: pytest - name: Upload results to Codecov + if: matrix.python-version == '3.11' uses: codecov/codecov-action@v6 with: files: ./coverage.xml diff --git a/.ruff.toml b/.ruff.toml index 4de8d6d..250dd76 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -target-version = "py312" +target-version = "py311" include = ["eclypse/**/*.py", "tests/**/*.py"] line-length = 88 @@ -40,7 +40,7 @@ select = [ "D", ] -ignore = ["D203", "D213", "D100", "D104", "E501", "PLC0415", "UP008"] +ignore = ["D203", "D213", "D100", "D104", "E501", "PLC0415", "UP008", "UP040"] fixable = ["ALL"] diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst index 82beff6..fb41f79 100644 --- a/docs/source/api/index.rst +++ b/docs/source/api/index.rst @@ -10,6 +10,7 @@ Reference builders graph placement + policies remote report simulation diff --git a/docs/source/overview/concepts/topology.rst b/docs/source/overview/concepts/topology.rst index a47dacd..39f6d19 100644 --- a/docs/source/overview/concepts/topology.rst +++ b/docs/source/overview/concepts/topology.rst @@ -15,12 +15,20 @@ The two classes share many structural similarities, but differ in purpose and in .. code-block:: python + from eclypse import policies from eclypse.graph.infrastructure import Infrastructure infrastructure = Infrastructure( infrastructure_id="infra", - node_update_policy=[...], - edge_update_policy=[...], + update_policies=[ + policies.failure.availability_flap(0.01, up_probability=0.2), + policies.distribution.uniform( + node_assets=["cpu", "ram"], + edge_assets=["latency", "bandwidth"], + node_distribution=(0.95, 1.05), + edge_distribution=(0.95, 1.05), + ), + ], node_assets=[...], edge_assets=[...], resource_init="min", @@ -33,7 +41,7 @@ The two classes share many structural similarities, but differ in purpose and in **Key parameters:** - ``infrastructure_id``: identifier of the infrastructure - - ``node_update_policy`` / ``edge_update_policy``: list of :doc:`update policies ` for infrastructure resources + - ``update_policies``: list of :doc:`update policies ` for infrastructure resources - ``node_assets`` / ``edge_assets``: available capabilities (:doc:`asset ` values) of nodes and links - ``resource_init``: initialisation of resources (*min* or *max*) - ``seed``: random seed for reproducibility @@ -46,12 +54,30 @@ The two classes share many structural similarities, but differ in purpose and in .. code-block:: python + from eclypse import policies from eclypse.graph.application import Application application = Application( application_id="app", - node_update_policy=[...], - edge_update_policy=[...], + update_policies=[ + policies.after( + 50, + policies.degrade.reduce( + factor=0.6, + epochs=200, + node_assets=["cpu", "ram"], + edge_assets=["bandwidth"], + ), + ), + policies.after( + 50, + policies.degrade.increase( + factor=1.6667, + epochs=200, + edge_assets=["latency"], + ), + ), + ], node_assets=[...], edge_assets=[...], requirement_init="min", @@ -62,7 +88,7 @@ The two classes share many structural similarities, but differ in purpose and in **Key parameters:** - ``application_id``: identifier of the application - - ``node_update_policy`` / ``edge_update_policy``: list of :doc:`update policies ` for application requirements + - ``update_policies``: list of :doc:`update policies ` for application requirements - ``node_assets`` / ``edge_assets``: resource requirements (:doc:`asset ` values) of services and links - ``requirement_init``: initialisation of resources (*min* or *max*) - ``seed``: random seed for reproducibility diff --git a/docs/source/overview/concepts/update-policy.rst b/docs/source/overview/concepts/update-policy.rst index ba86680..45aace3 100644 --- a/docs/source/overview/concepts/update-policy.rst +++ b/docs/source/overview/concepts/update-policy.rst @@ -5,53 +5,188 @@ In ECLYPSE, an update policy is a function that defines how the state of the infrastructure or application evolves over time. It enables dynamic simulations by modifying node or edge assets at each simulation step. -Unlike assets, update policies are not classes. Instead, they are simple functions with a fixed signature, depending on whether they operate on nodes or edges. +Unlike assets, update policies are not tied to separate node- or edge-specific +interfaces. They are simple graph-oriented callables that receive the graph +being updated. Function Signature ------------------ -There are two kinds of update policies: +.. code-block:: python + + from eclypse.graph import AssetGraph + + def my_policy(graph: AssetGraph): + ... + +The graph exposes the standard `networkx` views through ``graph.nodes`` and +``graph.edges``. Each node or edge has an associated data dictionary containing +its current asset values. + +Built-in Policies +----------------- + +ECLYPSE also provides a catalogue of off-the-shelf policies in +:mod:`eclypse.policies`. The module groups reusable policies into a few common +families: -- **Node update policies**: +- **failure**: availability flapping, node failures, and latency spikes +- **noise**: bounded random walks, momentum walks, and impulse shocks +- **distribution**: uniform, normal, lognormal, triangular, beta, gamma, + truncated-normal, and categorical multiplicative perturbations +- **degrade**: progressive increase or reduction of selected assets through + explicit ``increase()`` and ``reduce()`` policies +- **replay**: replay of node or edge values from records, dataframes, or parquet files +- **schedule**: wrappers such as ``every()``, ``after()``, ``between()``, and ``once_at()`` - .. code-block:: python +For most simulations, the easiest workflow is to compose a few built-in +policies and only fall back to a custom callable when the behaviour is +scenario-specific. - def my_node_policy(nodes: NodeView): - ... +Using Built-in Policies +----------------------- + +Built-in policies are regular graph callables, so you use them exactly like any +custom update policy. + +.. code-block:: python + :caption: **Example:** Infrastructure policies composed from ``eclypse.policies`` + + from eclypse import policies + from eclypse.graph import Infrastructure + + infrastructure = Infrastructure( + "edge-cloud", + update_policies=[ + policies.failure.availability_flap( + down_probability=0.02, + up_probability=0.5, + node_filter=lambda _, data: data["availability"] > 0, + ), + policies.distribution.uniform( + node_assets=["cpu", "ram", "storage"], + edge_assets=["latency", "bandwidth"], + node_asset_distributions={ + "cpu": (0.95, 1.05), + "ram": (0.9, 1.1), + "storage": (0.98, 1.02), + }, + edge_asset_distributions={ + "latency": (0.95, 1.05), + "bandwidth": (0.98, 1.02), + }, + ), + ], + ) + +Selectors and Assets +-------------------- + +Most built-in policies separate **what** to change from **where** to change it. + +- ``node_assets`` / ``edge_assets`` select which graph assets are updated +- ``node_ids`` / ``edge_ids`` target specific nodes or links +- ``node_filter`` / ``edge_filter`` let you target assets dynamically + +.. code-block:: python + :caption: **Example:** Apply noise only to edge nodes and WAN links -- **Edge update policies**: + from eclypse import policies - .. code-block:: python + policy = policies.distribution.uniform( + node_assets=["cpu", "ram"], + edge_assets=["latency"], + node_filter=lambda node_id, data: data.get("tier") == "edge", + edge_filter=lambda source, target, data: data.get("kind") == "wan", + ) - def my_edge_policy(edges: EdgeView): - ... +Scheduling Policies +------------------- -Both `NodeView` and `EdgeView` are provided by the `networkx` library and behave like dictionaries over the graph structure. Each node or edge has an associated data dictionary containing asset instances. -In particular a node is a tuple of the form ``(node_id, node_data)``, where `node_id` is the node identifier and `node_data` is a dictionary containing the asset instances. -On the other hand, an edge is a tuple of the form ``(source_node_id, target_node_id, edge_data)``, where `source_node_id` and `target_node_id` are the identifiers of the source and target nodes, respectively, and `edge_data` is a dictionary containing the asset instances. +Scheduling wrappers let you activate a policy only during part of the run. + +.. code-block:: python + :caption: **Example:** Start value adjustments after step 100 + + from eclypse import policies + + update_policies = [ + policies.after( + 100, + policies.degrade.reduce( + factor=0.5, + epochs=200, + node_assets=["cpu", "ram", "storage"], + edge_assets=["bandwidth"], + ), + ), + policies.after( + 100, + policies.degrade.increase( + factor=2.0, + epochs=200, + edge_assets=["latency"], + ), + ), + ] + +Replay Policies +--------------- + +Replay helpers are useful when you want the simulation to follow observed +or synthetic measurements over time. + +.. code-block:: python + :caption: **Example:** Replay node load from a parquet trace + + from eclypse import policies + + replay_users = policies.replay.from_parquet( + "examples/user_distribution/dataset.parquet", + target="nodes", + node_id_column="node_id", + time_column="time", + value_columns=["user_count"], + start_step=0, + ) Writing Custom Policies ----------------------- -You can define your own update policies by modifying the relevant asset values within each node or edge. +You can define your own update policies by modifying the relevant asset values +within the graph. .. code-block:: python :caption: **Example:** A node policy that caps CPU to a fixed maximum - def cap_cpu(nodes: NodeView): - for _, data in nodes.items(): + from eclypse.graph import AssetGraph + + def cap_cpu(graph: AssetGraph): + for _, data in graph.nodes.items(): if "cpu" in data: - data["cpu"].value = min(data["cpu"].value, 2.0) + data["cpu"] = min(data["cpu"], 2.0) .. code-block:: python :caption: **Example:** An edge policy that increases latency: - def increase_latency(edges: EdgeView): - for _, _, data in edges: + from eclypse.graph import AssetGraph + + def add_latency(graph: AssetGraph): + for _, _, data in graph.edges.data(): if "latency" in data: - data["latency"].value += 1.0 + data["latency"] += 1.0 + +Custom vs built-in +------------------ + +Built-in policies are ideal for common patterns such as failures, distributions, +explicit value adjustments, and replay from traces. When an example or scenario couples +multiple effects in a very specific way, keeping a custom callable is still the +right choice. Several examples in the repository intentionally do that to +preserve their original behaviour. .. important:: Update policies must always ensure that modified asset values remain consistent. - Use the asset's :py:meth:`~eclypse.graph.assets.asset.Asset.is_consistent()` method if needed. Otherwise, placement and simulation logic may occur on inconsistent data. + Use the asset's :py:meth:`~eclypse.graph.assets.asset.Asset.is_consistent()` method if needed. + Otherwise, placement and simulation logic may occur on inconsistent data. diff --git a/docs/source/overview/examples/echo.rst b/docs/source/overview/examples/echo.rst index 9430169..9067fa8 100644 --- a/docs/source/overview/examples/echo.rst +++ b/docs/source/overview/examples/echo.rst @@ -58,8 +58,9 @@ nodes connected through links with different latency and bandwidth values. .. literalinclude:: ../../../../examples/echo/infrastructure.py :language: python -The infrastructure is also updated at each iteration through random node and -edge update policies, which simulate changing runtime conditions. +The infrastructure is also updated at each iteration through a graph update +policy that mutates both nodes and links to simulate changing runtime +conditions. .. dropdown:: Update policy code diff --git a/docs/source/overview/examples/index.rst b/docs/source/overview/examples/index.rst index 2950ff0..51e4b78 100644 --- a/docs/source/overview/examples/index.rst +++ b/docs/source/overview/examples/index.rst @@ -7,6 +7,7 @@ Examples :hidden: echo + off_the_shelf sock_shop The examples section complements the getting-started guides with runnable @@ -29,6 +30,16 @@ subdirectory of the repository. communication patterns. + .. grid-item:: + + .. card:: :octicon:`tools;1em;info` **Off-the-shelf** + :link-type: doc + :link: off_the_shelf + + A local simulation composed only from built-in builders, policies, + and placement logic. + + .. grid-item:: .. card:: :octicon:`package-dependents;1em;info` **SockShop** diff --git a/docs/source/overview/examples/off_the_shelf.rst b/docs/source/overview/examples/off_the_shelf.rst new file mode 100644 index 0000000..08f8ded --- /dev/null +++ b/docs/source/overview/examples/off_the_shelf.rst @@ -0,0 +1,56 @@ +Off-the-shelf +============= + +This example shows a complete local simulation built only from reusable ECLYPSE +components: + +- the :func:`~eclypse.builders.application.get_sock_shop` application builder +- the :func:`~eclypse.builders.infrastructure.hierarchical` infrastructure builder +- built-in update policies from :mod:`eclypse.policies` +- a built-in placement strategy + +Use it when you want a minimal reference for composing existing building blocks +without writing custom services, topologies, or policy callables. + +The full code lives in the +`examples/off_the_shelf `_ +directory. + +Application +----------- + +The application is the standard Sock Shop graph created through the built-in +builder, with built-in uniform-distribution and degradation policies that +progressively make placement harder. + +.. dropdown:: Application code + + .. literalinclude:: ../../../../examples/off_the_shelf/application.py + :language: python + + +Infrastructure +-------------- + +The infrastructure is a generated hierarchical topology using the default +assets and a built-in policy mix for flapping availability, uniform +perturbations, periodic latency spikes, and scheduled degradation. Together with +``BestFitStrategy``, this makes the example exercise repeated placement under a +changing environment. + +.. dropdown:: Infrastructure code + + .. literalinclude:: ../../../../examples/off_the_shelf/infrastructure.py + :language: python + + +Simulation +---------- + +The simulation registers the built-in application on the generated +infrastructure with a built-in placement strategy and runs it locally. + +.. dropdown:: Simulation code + + .. literalinclude:: ../../../../examples/off_the_shelf/main.py + :language: python diff --git a/eclypse/builders/_helpers.py b/eclypse/builders/_helpers.py new file mode 100644 index 0000000..7e2e1c3 --- /dev/null +++ b/eclypse/builders/_helpers.py @@ -0,0 +1,33 @@ +"""Helper functions shared by ECLYPSE builders.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Any, +) + +if TYPE_CHECKING: + from eclypse.graph.assets import AssetBucket + + +def prune_assets( + assets: AssetBucket, + **requirements, +) -> dict[str, Any]: + """Prune the requirements dictionary. + + Removes all the keys from the requirements dictionary that are not present in the + assets dictionary. + + Args: + assets (AssetBucket): The assets dictionary. + **requirements: The requirements dictionary. + + Returns: + dict[str, Any]: The pruned requirements dictionary. + """ + return {key: value for key, value in requirements.items() if assets.get(key)} + + +__all__ = ["prune_assets"] diff --git a/eclypse/builders/application/sock_shop/application.py b/eclypse/builders/application/sock_shop/application.py index 75c609e..1ca405e 100644 --- a/eclypse/builders/application/sock_shop/application.py +++ b/eclypse/builders/application/sock_shop/application.py @@ -19,33 +19,26 @@ get_args, ) +from eclypse.builders._helpers import prune_assets from eclypse.graph import Application -from eclypse.utils.tools import prune_assets from eclypse.utils.types import CommunicationInterface if TYPE_CHECKING: - from collections.abc import ( - Callable, - ) - - from networkx.classes.reportviews import ( - EdgeView, - NodeView, - ) - from eclypse.graph.assets import Asset - from eclypse.utils.types import InitPolicy + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) -SUPPORTED_COMMUNICATION_INTERFACES = get_args(CommunicationInterface.__value__) +_SUPPORTED_COMMUNICATION_INTERFACES = get_args(CommunicationInterface) """Supported remote communication interfaces for the Sock Shop builders.""" def get_sock_shop( application_id: str = "SockShop", communication_interface: CommunicationInterface | None = None, - node_update_policy: Callable[[NodeView], None] | None = None, - edge_update_policy: Callable[[EdgeView], None] | None = None, + update_policies: UpdatePolicies = None, node_assets: dict[str, Asset] | None = None, edge_assets: dict[str, Asset] | None = None, include_default_assets: bool = False, @@ -59,10 +52,8 @@ def get_sock_shop( application_id (str): The ID of the application. communication_interface (CommunicationInterface | None): The communication interface. - node_update_policy (Callable[[NodeView], None] | None): - A function to update the nodes. - edge_update_policy (Callable[[EdgeView], None] | None): - A function to update the edges. + update_policies (Callable | list[Callable] | None): + Graph update policies executed during ``evolve()``. node_assets (dict[str, Asset] | None): The assets of the nodes. edge_assets (dict[str, Asset] | None): The assets of the edges. include_default_assets (bool): @@ -103,8 +94,7 @@ def get_sock_shop( app = Application( application_id=application_id, - node_update_policy=node_update_policy, - edge_update_policy=edge_update_policy, + update_policies=update_policies, node_assets=node_assets, edge_assets=edge_assets, include_default_assets=include_default_assets, @@ -119,7 +109,7 @@ def get_sock_shop( def id_fn(service): return service - elif communication_interface in SUPPORTED_COMMUNICATION_INTERFACES: + elif communication_interface in _SUPPORTED_COMMUNICATION_INTERFACES: add_fn = app.add_service # type: ignore[assignment] if communication_interface == "mpi": from . import mpi_services as services diff --git a/eclypse/builders/infrastructure/generators/b_cube.py b/eclypse/builders/infrastructure/generators/b_cube.py index 7d00f21..ffe3d72 100644 --- a/eclypse/builders/infrastructure/generators/b_cube.py +++ b/eclypse/builders/infrastructure/generators/b_cube.py @@ -29,27 +29,23 @@ from eclypse.graph import Infrastructure if TYPE_CHECKING: - from collections.abc import ( - Callable, - ) + from collections.abc import Callable import networkx as nx - from networkx.classes.reportviews import ( - EdgeView, - NodeView, - ) from eclypse.graph.assets import Asset from eclypse.placement.strategies import PlacementStrategy - from eclypse.utils.types import InitPolicy + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) def b_cube( k: int, n: int, infrastructure_id: str = "b_cube", - node_update_policy: Callable[[NodeView], None] | None = None, - link_update_policy: Callable[[EdgeView], None] | None = None, + update_policies: UpdatePolicies = None, node_assets: dict[str, Asset] | None = None, link_assets: dict[str, Asset] | None = None, include_default_assets: bool = False, @@ -69,9 +65,7 @@ def b_cube( n (int): Number of ports per switch, and number of switches per level. infrastructure_id (str): Unique ID for the infrastructure instance.\ Defaults to "b_cube". - node_update_policy (Callable[[NodeView], None] | None): Policy to update nodes.\ - Defaults to None. - link_update_policy (Callable[[EdgeView], None] | None): Policy to update links.\ + update_policies (Callable | list[Callable] | None): Graph update policies.\ Defaults to None. node_assets (dict[str, Asset] | None): Default attributes for all nodes.\ Defaults to None. @@ -94,8 +88,7 @@ def b_cube( """ infra = Infrastructure( infrastructure_id=infrastructure_id, - node_update_policy=node_update_policy, - edge_update_policy=link_update_policy, + update_policies=update_policies, node_assets=node_assets, edge_assets=link_assets, include_default_assets=include_default_assets, diff --git a/eclypse/builders/infrastructure/generators/fat_tree.py b/eclypse/builders/infrastructure/generators/fat_tree.py index 6bac05e..c24ca4e 100644 --- a/eclypse/builders/infrastructure/generators/fat_tree.py +++ b/eclypse/builders/infrastructure/generators/fat_tree.py @@ -22,26 +22,22 @@ from eclypse.graph import Infrastructure if TYPE_CHECKING: - from collections.abc import ( - Callable, - ) + from collections.abc import Callable import networkx as nx - from networkx.classes.reportviews import ( - EdgeView, - NodeView, - ) from eclypse.graph.assets import Asset from eclypse.placement.strategies import PlacementStrategy - from eclypse.utils.types import InitPolicy + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) def fat_tree( k: int, infrastructure_id: str = "fat_tree", - node_update_policy: Callable[[NodeView], None] | None = None, - link_update_policy: Callable[[EdgeView], None] | None = None, + update_policies: UpdatePolicies = None, node_assets: dict[str, Asset] | None = None, link_assets: dict[str, Asset] | None = None, include_default_assets: bool = False, @@ -62,9 +58,7 @@ def fat_tree( Determines the size and structure of the Fat-Tree topology. infrastructure_id (str): Unique ID for the infrastructure instance.\ Defaults to "fat_tree". - node_update_policy (Callable[[NodeView], None] | None): Policy to update nodes.\ - Defaults to None. - link_update_policy (Callable[[EdgeView], None] | None): Policy to update links.\ + update_policies (Callable | list[Callable] | None): Graph update policies.\ Defaults to None. node_assets (dict[str, Asset] | None): Default attributes for all nodes.\ Defaults to None. @@ -90,8 +84,7 @@ def fat_tree( infra = Infrastructure( infrastructure_id=infrastructure_id, - node_update_policy=node_update_policy, - edge_update_policy=link_update_policy, + update_policies=update_policies, node_assets=node_assets, edge_assets=link_assets, include_default_assets=include_default_assets, diff --git a/eclypse/builders/infrastructure/generators/hierarchical.py b/eclypse/builders/infrastructure/generators/hierarchical.py index 6eef0b1..1d37ca9 100644 --- a/eclypse/builders/infrastructure/generators/hierarchical.py +++ b/eclypse/builders/infrastructure/generators/hierarchical.py @@ -29,16 +29,13 @@ ) from networkx import nx - from networkx.classes.reportviews import ( - EdgeView, - NodeView, - ) from eclypse.graph.assets import Asset from eclypse.placement.strategies import PlacementStrategy from eclypse.utils.types import ( ConnectivityFn, InitPolicy, + UpdatePolicies, ) DEFAULT_NODE_PARTITIONING = [0.35, 0.3, 0.2, 0.15] @@ -51,8 +48,7 @@ def hierarchical( node_partitioning: list[float] | None = None, connectivity: ConnectivityFn | list[float] | None = None, cross_level_connectivity: ConnectivityFn | list[float] | None = None, - node_update_policy: Callable[[NodeView], None] | None = None, - link_update_policy: Callable[[EdgeView], None] | None = None, + update_policies: UpdatePolicies = None, node_assets: dict[str, Asset] | None = None, link_assets: dict[str, Asset] | None = None, include_default_assets: bool = False, @@ -82,10 +78,8 @@ def hierarchical( cross_level_connectivity (ConnectivityFn | list[float] | None): The connectivity function or list of probabilities for the connections between nodes in the same level. Defaults to None. - node_update_policy (Callable[[NodeView], None] | None): - The policy to update the nodes. Defaults to None. - link_update_policy (Callable[[EdgeView], None] | None): - The policy to update the links. Defaults to None. + update_policies (Callable | list[Callable] | None): + Graph update policies. Defaults to None. node_assets (dict[str, Asset] | None): The assets for the nodes. Defaults to None. link_assets (dict[str, Asset] | None): @@ -140,8 +134,7 @@ def hierarchical( infrastructure = Infrastructure( infrastructure_id=infrastructure_id, - node_update_policy=node_update_policy, - edge_update_policy=link_update_policy, + update_policies=update_policies, node_assets=node_assets, edge_assets=link_assets, include_default_assets=include_default_assets, diff --git a/eclypse/builders/infrastructure/generators/random.py b/eclypse/builders/infrastructure/generators/random.py index 69ac93c..b2008f0 100644 --- a/eclypse/builders/infrastructure/generators/random.py +++ b/eclypse/builders/infrastructure/generators/random.py @@ -22,18 +22,14 @@ from eclypse.graph import Infrastructure if TYPE_CHECKING: - from collections.abc import ( - Callable, - ) - - from networkx.classes.reportviews import ( - EdgeView, - NodeView, - ) + from collections.abc import Callable from eclypse.graph.assets import Asset from eclypse.placement.strategies import PlacementStrategy - from eclypse.utils.types import InitPolicy + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) def random( @@ -41,8 +37,7 @@ def random( infrastructure_id: str = "random", p: float = 0.5, symmetric: bool = False, - node_update_policy: Callable[[NodeView], None] | None = None, - link_update_policy: Callable[[EdgeView], None] | None = None, + update_policies: UpdatePolicies = None, node_assets: dict[str, Asset] | None = None, link_assets: dict[str, Asset] | None = None, include_default_assets: bool = False, @@ -62,10 +57,8 @@ def random( infrastructure_id (str): The ID of the infrastructure. p (float): The probability of connecting two nodes. Defaults to 0.5. symmetric (bool): Whether the links are symmetric. Defaults to False. - node_update_policy (Callable[[NodeView], None] | None): - The policy to update the nodes. Defaults to None. - link_update_policy (Callable[[EdgeView], None] | None): - The policy to update the links. Defaults to None. + update_policies (Callable | list[Callable] | None): + Graph update policies. Defaults to None. node_assets (dict[str, Asset] | None): The assets for the nodes. Defaults to None. link_assets (dict[str, Asset] | None): @@ -88,8 +81,7 @@ def random( """ infrastructure = Infrastructure( infrastructure_id=infrastructure_id, - node_update_policy=node_update_policy, - edge_update_policy=link_update_policy, + update_policies=update_policies, node_assets=node_assets, edge_assets=link_assets, include_default_assets=include_default_assets, diff --git a/eclypse/builders/infrastructure/generators/star.py b/eclypse/builders/infrastructure/generators/star.py index 9ba06a9..b98e542 100644 --- a/eclypse/builders/infrastructure/generators/star.py +++ b/eclypse/builders/infrastructure/generators/star.py @@ -20,27 +20,23 @@ from eclypse.graph import Infrastructure if TYPE_CHECKING: - from collections.abc import ( - Callable, - ) + from collections.abc import Callable import networkx as nx - from networkx.classes.reportviews import ( - EdgeView, - NodeView, - ) from eclypse.graph.assets import Asset from eclypse.placement.strategies import PlacementStrategy - from eclypse.utils.types import InitPolicy + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) def star( n_clients: int, infrastructure_id: str = "star", symmetric: bool = False, - node_update_policy: Callable[[NodeView], None] | None = None, - link_update_policy: Callable[[EdgeView], None] | None = None, + update_policies: UpdatePolicies = None, node_assets: dict[str, Asset] | None = None, link_assets: dict[str, Asset] | None = None, center_assets_values: dict[str, Any] | None = None, @@ -60,10 +56,8 @@ def star( n_clients (int): The number of clients in the infrastructure. infrastructure_id (str): The ID of the infrastructure. symmetric (bool): Whether the links are symmetric. Defaults to False. - node_update_policy (Callable[[NodeView], None] | None): - The policy to update the nodes. Defaults to None. - link_update_policy (Callable[[EdgeView], None] | None): - The policy to update the links. Defaults to None. + update_policies (Callable | list[Callable] | None): + Graph update policies. Defaults to None. node_assets (dict[str, Asset] | None): The assets for the nodes. Defaults to None. link_assets (dict[str, Asset] | None): @@ -91,8 +85,7 @@ def star( """ infrastructure = Infrastructure( infrastructure_id=infrastructure_id, - node_update_policy=node_update_policy, - edge_update_policy=link_update_policy, + update_policies=update_policies, node_assets=node_assets, edge_assets=link_assets, include_default_assets=include_default_assets, diff --git a/eclypse/builders/infrastructure/orion_cev.py b/eclypse/builders/infrastructure/orion_cev.py index 4297cb8..2b62f04 100644 --- a/eclypse/builders/infrastructure/orion_cev.py +++ b/eclypse/builders/infrastructure/orion_cev.py @@ -18,29 +18,25 @@ TYPE_CHECKING, ) +from eclypse.builders._helpers import prune_assets from eclypse.graph import Infrastructure -from eclypse.utils.tools import prune_assets if TYPE_CHECKING: - from collections.abc import ( - Callable, - ) + from collections.abc import Callable import networkx as nx - from networkx.classes.reportviews import ( - EdgeView, - NodeView, - ) from eclypse.graph.assets import Asset from eclypse.placement.strategies import PlacementStrategy - from eclypse.utils.types import InitPolicy + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) def get_orion_cev( infrastructure_id: str = "orion_cev", - node_update_policy: Callable[[NodeView], None] | None = None, - link_update_policy: Callable[[EdgeView], None] | None = None, + update_policies: UpdatePolicies = None, node_assets: dict[str, Asset] | None = None, link_assets: dict[str, Asset] | None = None, include_default_assets: bool = False, @@ -54,10 +50,8 @@ def get_orion_cev( Args: infrastructure_id (str): The ID of the infrastructure. Defaults to "OrionCEV". - node_update_policy (Callable[[NodeView], None] | None): - The policy to update the nodes. Defaults to None. - link_update_policy (Callable[[EdgeView], None] | None): - The policy to update the links. Defaults to None. + update_policies (Callable | list[Callable] | None): + Graph update policies. Defaults to None. node_assets (dict[str, Asset] | None): The assets for the nodes. Defaults to None. link_assets (dict[str, Asset] | None): @@ -78,8 +72,7 @@ def get_orion_cev( """ infra = Infrastructure( infrastructure_id=infrastructure_id, - node_update_policy=node_update_policy, - edge_update_policy=link_update_policy, + update_policies=update_policies, node_assets=node_assets, edge_assets=link_assets, include_default_assets=include_default_assets, diff --git a/eclypse/graph/application.py b/eclypse/graph/application.py index 7d5d6b0..8b29a2a 100644 --- a/eclypse/graph/application.py +++ b/eclypse/graph/application.py @@ -21,17 +21,11 @@ from eclypse.remote.service import Service if TYPE_CHECKING: - from collections.abc import ( - Callable, + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, ) - from networkx.classes.reportviews import ( - EdgeView, - NodeView, - ) - - from eclypse.utils.types import InitPolicy - from .assets import Asset @@ -41,12 +35,7 @@ class Application(AssetGraph): # pylint: disable=too-few-public-methods def __init__( self, application_id: str, - node_update_policy: Callable[[NodeView], None] - | list[Callable[[NodeView], None]] - | None = None, - edge_update_policy: Callable[[EdgeView], None] - | list[Callable[[EdgeView], None]] - | None = None, + update_policies: UpdatePolicies = None, node_assets: dict[str, Asset] | None = None, edge_assets: dict[str, Asset] | None = None, include_default_assets: bool = False, @@ -58,10 +47,8 @@ def __init__( Args: application_id (str): The ID of the application. - node_update_policy (Callable | list[Callable] | None):\ - A function to update the nodes. Defaults to None. - edge_update_policy (Callable | list[Callable] | None):\ - A function to update the edges. Defaults to None. + update_policies (Callable | list[Callable] | None):\ + Graph update policies executed during ``evolve()``. node_assets (dict[str, Asset] | None): The assets of the nodes. edge_assets (dict[str, Asset] | None): The assets of the edges. include_default_assets (bool): Whether to include the default assets. \ @@ -78,8 +65,7 @@ def __init__( super().__init__( graph_id=application_id, - node_update_policy=node_update_policy, - edge_update_policy=edge_update_policy, + update_policies=update_policies, node_assets=_node_assets, edge_assets=_edge_assets, attr_init=requirement_init, diff --git a/eclypse/graph/asset_graph.py b/eclypse/graph/asset_graph.py index 2aa3e61..b76ac9c 100644 --- a/eclypse/graph/asset_graph.py +++ b/eclypse/graph/asset_graph.py @@ -3,7 +3,7 @@ Extensions are: - Initialization of nodes and edges with a given set of assets (asset bucket). -- Definition of an update policy for nodes and edges. +- Definition of graph update policies. - Definition of a seed for the randomicity of the assets. - Binding of the graph id in the logs. """ @@ -12,31 +12,25 @@ import random as rnd from copy import deepcopy -from typing import ( - TYPE_CHECKING, -) +from typing import TYPE_CHECKING import networkx as nx from eclypse.graph.assets import AssetBucket from eclypse.utils._logging import ( + format_log_kv, log_assets_violations, logger, ) if TYPE_CHECKING: - from collections.abc import ( - Callable, - ) - - from networkx.classes.reportviews import ( - EdgeView, - NodeView, - ) - from eclypse.graph.assets import Asset from eclypse.utils._logging import Logger - from eclypse.utils.types import InitPolicy + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + UpdatePolicy, + ) class AssetGraph(nx.DiGraph): @@ -47,12 +41,7 @@ def __init__( graph_id: str, node_assets: dict[str, Asset] | None = None, edge_assets: dict[str, Asset] | None = None, - node_update_policy: Callable[[NodeView], None] - | list[Callable[[NodeView], None]] - | None = None, - edge_update_policy: Callable[[EdgeView], None] - | list[Callable[[EdgeView], None]] - | None = None, + update_policies: UpdatePolicies = None, attr_init: InitPolicy = "min", flip_assets: bool = False, seed: int | None = None, @@ -65,10 +54,9 @@ def __init__( The assets of the nodes. Defaults to None. edge_assets (dict[str, Asset] | None, optional): The assets of the edges. Defaults to None. - node_update_policy (Callable | list[Callable] | None): - The policy to update the nodes. Defaults to None. - edge_update_policy (Callable | list[Callable] | None): - The policy to update the edges. Defaults to None. + update_policies (Callable | list[Callable] | None): + The graph update policies to execute during ``evolve()``. + Defaults to None. attr_init (InitPolicy, optional): The initialization policy for the assets. Defaults to "min". @@ -79,22 +67,7 @@ def __init__( self.rnd = rnd.Random(seed) self.id = graph_id - if node_update_policy is None: - _node_update_policy = [] - elif not isinstance(node_update_policy, list): - _node_update_policy = [node_update_policy] - else: - _node_update_policy = node_update_policy - - if edge_update_policy is None: - _edge_update_policy = [] - elif not isinstance(edge_update_policy, list): - _edge_update_policy = [edge_update_policy] - else: - _edge_update_policy = edge_update_policy - - self.node_update_policy = _node_update_policy - self.edge_update_policy = _edge_update_policy + self.update_policies = _normalize_update_policies(update_policies) _node_assets = node_assets if node_assets is not None else {} _edge_assets = edge_assets if edge_assets is not None else {} @@ -144,8 +117,10 @@ def add_node(self, node_for_adding: str, strict: bool = True, **assets): _assets.update(assets) violations = self.node_assets.is_consistent(_assets, violations=True) - if violations: - msg = f"Node {node_for_adding} has inconsistent assets:" + if isinstance(violations, dict) and violations: + msg = f"Node {node_for_adding} has inconsistent assets | " + format_log_kv( + assets=",".join(sorted(violations)) + ) if strict: raise ValueError(f"{msg}{violations}") self.logger.warning(msg) @@ -191,8 +166,11 @@ def add_edge( _assets.update(assets) violations = self.edge_assets.is_consistent(_assets, violations=True) - if violations: - msg = f"Edge {u_of_edge} -> {v_of_edge} has inconsistent assets:" + if isinstance(violations, dict) and violations: + msg = ( + f"Edge {u_of_edge} -> {v_of_edge} has inconsistent assets | " + + format_log_kv(assets=",".join(sorted(violations))) + ) if strict: raise ValueError(f"{msg}{violations}") self.logger.warning(msg) @@ -205,11 +183,10 @@ def add_edge( def evolve(self): """Updates the graph according to its update policies.""" - for node_update in self.node_update_policy: - node_update(self.nodes) - - for edge_update in self.edge_update_policy: - edge_update(self.edges) + if self.update_policies: + self.logger.debug(f"Applying {len(self.update_policies)} update policies.") + for update_policy in self.update_policies: + update_policy(self) def _get_node_lower_bound(self): """Returns the lower bound of the node assets.""" @@ -234,7 +211,7 @@ def is_dynamic(self) -> bool: Returns: bool: True if the graph is dynamic, False otherwise. """ - return self.node_update_policy != [] or self.edge_update_policy != [] + return self.update_policies != [] @property def logger(self) -> Logger: @@ -244,3 +221,12 @@ def logger(self) -> Logger: Logger: The logger for the graph. """ return logger.bind(id=self.id) + + +def _normalize_update_policies(update_policies: UpdatePolicies) -> list[UpdatePolicy]: + """Normalise a policy declaration to a list of graph policies.""" + if update_policies is None: + return [] + if isinstance(update_policies, list): + return update_policies + return [update_policies] diff --git a/eclypse/graph/infrastructure.py b/eclypse/graph/infrastructure.py index e2e0f3b..7d63ffc 100644 --- a/eclypse/graph/infrastructure.py +++ b/eclypse/graph/infrastructure.py @@ -22,7 +22,10 @@ from networkx.classes.filters import no_filter from eclypse.graph import AssetGraph -from eclypse.utils._logging import log_placement_violations +from eclypse.utils._logging import ( + format_log_kv, + log_placement_violations, +) from eclypse.utils.constants import ( COST_RECOMPUTATION_THRESHOLD, MIN_FLOAT, @@ -36,18 +39,14 @@ ) if TYPE_CHECKING: - from collections.abc import ( - Callable, - ) - - from networkx.classes.reportviews import ( - EdgeView, - NodeView, - ) + from collections.abc import Callable from eclypse.graph.assets.asset import Asset from eclypse.placement.strategies import PlacementStrategy - from eclypse.utils.types import InitPolicy + from eclypse.utils.types import ( + InitPolicy, + UpdatePolicies, + ) class Infrastructure(AssetGraph): # pylint: disable=too-few-public-methods @@ -57,12 +56,7 @@ def __init__( self, infrastructure_id: str = "Infrastructure", placement_strategy: PlacementStrategy | None = None, - node_update_policy: Callable[[NodeView], None] - | list[Callable[[NodeView], None]] - | None = None, - edge_update_policy: Callable[[EdgeView], None] - | list[Callable[[EdgeView], None]] - | None = None, + update_policies: UpdatePolicies = None, node_assets: dict[str, Asset] | None = None, edge_assets: dict[str, Asset] | None = None, include_default_assets: bool = False, @@ -77,10 +71,8 @@ def __init__( infrastructure_id (str): The ID of the infrastructure. placement_strategy (PlacementStrategy | None): The placement \ strategy to use. - node_update_policy (Callable | list[Callable] | None):\ - A function to update the nodes. Defaults to None. - edge_update_policy (Callable | list[Callable] | None):\ - A function to update the edges. Defaults to None. + update_policies (Callable | list[Callable] | None):\ + Graph update policies executed during ``evolve()``. node_assets (dict[str, Asset] | None): The assets of the nodes. edge_assets (dict[str, Asset] | None): The assets of the edges. include_default_assets (bool): Whether to include the default assets. \ @@ -100,8 +92,7 @@ def __init__( super().__init__( graph_id=infrastructure_id, - node_update_policy=node_update_policy, - edge_update_policy=edge_update_policy, + update_policies=update_policies, node_assets=_node_assets, edge_assets=_edge_assets, attr_init=resource_init, @@ -146,6 +137,11 @@ def __init__( self._path_resources: dict[str, dict[str, dict[str, Any]]] = {} self._processing_times: dict[str, dict[str, float]] = {} + def evolve(self): + """Update the infrastructure and invalidate derived path caches.""" + super().evolve() + self._invalidate_cache() + def add_node(self, node_for_adding: str, strict: bool = False, **assets: Any): """Add a node and invalidate the path cache. @@ -208,24 +204,30 @@ def contains(self, other: nx.DiGraph) -> list[str]: other (Infrastructure): The Infrastructure to compare with. Returns: - list[str]: A list of nodes whose requirements are not respected or \ + list[str]: A list of nodes whose requirements are not respected or whose connected links are not respected. """ not_respected = set() for n, req in other.nodes(data=True): res = self.nodes[n] node_violations = self.node_assets.satisfies(res, req, violations=True) - if node_violations: - self.logger.warning(f'Node "{n}" not respected:') - log_placement_violations(self.logger, node_violations) # type: ignore[arg-type] + if isinstance(node_violations, dict) and node_violations: + self.logger.warning( + f'Node "{n}" violated | ' + + format_log_kv(assets=",".join(sorted(node_violations))) + ) + log_placement_violations(self.logger, node_violations) not_respected.add(n) for u, v, req in other.edges(data=True): res = self.path_resources(u, v) edge_violations = self.edge_assets.satisfies(res, req, violations=True) - if edge_violations: - self.logger.warning(f'Link "{u} -> {v}" not respected:') - log_placement_violations(self.logger, edge_violations) # type: ignore[arg-type] + if isinstance(edge_violations, dict) and edge_violations: + self.logger.warning( + f'Link "{u} -> {v}" violated | ' + + format_log_kv(assets=",".join(sorted(edge_violations))) + ) + log_placement_violations(self.logger, edge_violations) not_respected.add(u) not_respected.add(v) diff --git a/eclypse/placement/_manager.py b/eclypse/placement/_manager.py index 0411cca..71b868b 100644 --- a/eclypse/placement/_manager.py +++ b/eclypse/placement/_manager.py @@ -14,7 +14,10 @@ ) from eclypse.placement import Placement -from eclypse.utils._logging import logger +from eclypse.utils._logging import ( + format_log_kv, + logger, +) from .view import PlacementView @@ -53,28 +56,38 @@ def audit(self): If not, it resets the placement of the applications whose requirements are not respected. """ + reset_requests = 0 for _, not_respected in self.mapping_phase(): if not_respected: for n, apps in self.placement_view.nodes_used_by.items(): if n in not_respected: for app in apps: p = self.get(app) + if not p.reset_requested: + reset_requests += 1 p.mark_for_reset() + self.logger.debug( + "Placement audit completed | " + + format_log_kv(applications=len(self.placements), resets=reset_requests) + ) def enact(self): """Manage and apply, or reset, application placements.""" for p in self.placements.values(): if p.reset_requested: - self.logger.warning(f"Resetting placement of {p.application.id}") - self.logger.trace(p) + label = "Resetting placement | " + self.logger.warning(f"{label}" + format_log_kv(app=p.application.id)) + self.logger.trace( + label + format_log_kv(app=p.application.id, mapping=p.mapping) + ) p._reset_mapping() if p.mapping: self.logger.log( "ECLYPSE", - f"Placement of {p.application.id} on {self.infrastructure.id}", + "Placement established | " + + format_log_kv(app=p.application.id, mapping=p.mapping), ) - self.logger.log("ECLYPSE", p) def generate_mapping(self, placement: Placement): """Create application-to-infrastructure mapping from a placement strategy. @@ -110,11 +123,29 @@ def generate_mapping(self, placement: Placement): placement._generate_mapping(self.placements, self.placement_view) if not placement.mapping or all(v is None for v in placement.mapping.values()): + label = "No placement found | " self.logger.log( - "ECLYPSE", f"No placement found for {placement.application.id}" + "ECLYPSE", + label + format_log_kv(app=placement.application.id), + ) + self.logger.trace( + label + + format_log_kv(app=placement.application.id, mapping=placement.mapping) ) placement._reset_mapping() elif not_placed_services := placement.is_partial: + label = "Partial placement | " + self.logger.log( + "ECLYPSE", + label + + format_log_kv( + app=placement.application.id, unplaced=not_placed_services + ), + ) + self.logger.trace( + label + + format_log_kv(app=placement.application.id, mapping=placement.mapping) + ) self.logger.warning(f"Partial placement for {placement.application.id}") self.logger.warning(f"Not placed services: {not_placed_services}") placement._reset_mapping() @@ -145,8 +176,6 @@ def mapping_phase( self.placement_view._update_view(p) not_respected = self.infrastructure.contains(self.placement_view) - if not_respected: - p.mark_for_reset() yield (p, not_respected) def register( diff --git a/eclypse/placement/strategies/static.py b/eclypse/placement/strategies/static.py index 51d4dc1..efc3395 100644 --- a/eclypse/placement/strategies/static.py +++ b/eclypse/placement/strategies/static.py @@ -68,7 +68,7 @@ def is_feasible(self, infrastructure: Infrastructure, _: Application) -> bool: """ for node in self.mapping.values(): if node not in infrastructure.nodes: - infrastructure.logger.error( + infrastructure.logger.warning( f"Node {node} not found or not available in the infrastructure." ) return False diff --git a/eclypse/policies/__init__.py b/eclypse/policies/__init__.py new file mode 100644 index 0000000..baf46d7 --- /dev/null +++ b/eclypse/policies/__init__.py @@ -0,0 +1,49 @@ +"""Utilities and built-in update policies. + +This module hosts the public update-policy interface used by ECLYPSE graphs. +Policies are graph-oriented callables that mutate an +:class:`~eclypse.graph.asset_graph.AssetGraph` during ``evolve()``. +""" + +from __future__ import annotations + +from eclypse.policies import ( + degrade, + distribution, + failure, + noise, + replay, + schedule, +) +from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, +) +from eclypse.policies.schedule import ( + after, + between, + every, + once_at, +) +from eclypse.utils.types import ( + UpdatePolicies, + UpdatePolicy, +) + + +__all__ = [ + "EdgeFilter", + "NodeFilter", + "UpdatePolicies", + "UpdatePolicy", + "after", + "between", + "degrade", + "distribution", + "every", + "failure", + "noise", + "once_at", + "replay", + "schedule", +] diff --git a/eclypse/policies/_filters.py b/eclypse/policies/_filters.py new file mode 100644 index 0000000..d4a44a9 --- /dev/null +++ b/eclypse/policies/_filters.py @@ -0,0 +1,143 @@ +"""Shared helpers for selecting graph items in built-in policies.""" +# ruff: noqa: UP035 + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Any, + Callable, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + +NodeFilter = Callable[[str, dict[str, Any]], bool] +EdgeFilter = Callable[[str, str, dict[str, Any]], bool] + + +def iter_selected_nodes( + graph: AssetGraph, + *, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, +) -> list[tuple[str, dict[str, Any]]]: + """Yield nodes matching the provided selectors.""" + selected_node_ids = set(node_ids) if node_ids is not None else None + selected_nodes: list[tuple[str, dict[str, Any]]] = [] + + for node_id, data in graph.nodes.data(): + if selected_node_ids is not None and node_id not in selected_node_ids: + continue + if node_filter is not None and not node_filter(node_id, data): + continue + selected_nodes.append((node_id, data)) + + return selected_nodes + + +def iter_selected_edges( + graph: AssetGraph, + *, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> list[tuple[str, str, dict[str, Any]]]: + """Yield edges matching the provided selectors.""" + selected_edge_ids = set(edge_ids) if edge_ids is not None else None + selected_edges: list[tuple[str, str, dict[str, Any]]] = [] + + for source, target, data in graph.edges.data(): + if selected_edge_ids is not None and (source, target) not in selected_edge_ids: + continue + if edge_filter is not None and not edge_filter(source, target, data): + continue + selected_edges.append((source, target, data)) + + return selected_edges + + +def iter_selected_keys( + data: dict[str, Any], + keys: str | list[str] | None = None, +) -> list[str]: + """Yield existing keys selected for a policy operation.""" + selected = normalize_selected_keys(keys) + if selected is None: + return list(data.keys()) + + selected_keys: list[str] = [] + for key in selected: + if key in data: + selected_keys.append(key) + + return selected_keys + + +def normalize_selected_keys( + keys: str | list[str] | None, +) -> list[str] | None: + """Normalise a string-or-list selector to a list of keys.""" + if keys is None: + return None + if isinstance(keys, str): + return [keys] + return list(keys) + + +def effective_assets( + assets: str | list[str] | None, + per_asset_values: dict[str, Any] | None = None, +) -> list[str]: + """Resolve selected asset keys from explicit selectors and per-asset maps.""" + selected_assets = list(normalize_selected_keys(assets) or []) + + for key in per_asset_values or {}: + if key not in selected_assets: + selected_assets.append(key) + + return selected_assets + + +def ensure_numeric_value(key: str, value: Any) -> float: + """Return a numeric value or raise a clear error for unsupported assets.""" + if isinstance(value, bool) or not isinstance(value, int | float): + raise TypeError( + f'Policy expected numeric asset "{key}", got {type(value).__name__}.' + ) + return float(value) + + +def clamp( + value: float, + lower: float | None = None, + upper: float | None = None, +) -> float: + """Clamp a numeric value between optional bounds.""" + if lower is not None: + value = max(lower, value) + if upper is not None: + value = min(upper, value) + return value + + +def coerce_numeric_like(original: Any, value: float) -> int | float: + """Cast a computed value back to the original numeric kind when possible.""" + if isinstance(original, bool): + return value + if isinstance(original, int): + return round(value) + return value + + +__all__ = [ + "EdgeFilter", + "NodeFilter", + "clamp", + "coerce_numeric_like", + "effective_assets", + "ensure_numeric_value", + "iter_selected_edges", + "iter_selected_keys", + "iter_selected_nodes", + "normalize_selected_keys", +] diff --git a/eclypse/policies/degrade/__init__.py b/eclypse/policies/degrade/__init__.py new file mode 100644 index 0000000..68eebf0 --- /dev/null +++ b/eclypse/policies/degrade/__init__.py @@ -0,0 +1,11 @@ +"""Built-in deterministic value-adjustment policies.""" + +from __future__ import annotations + +from .increase import increase +from .reduce import reduce + +__all__ = [ + "increase", + "reduce", +] diff --git a/eclypse/policies/degrade/_helpers.py b/eclypse/policies/degrade/_helpers.py new file mode 100644 index 0000000..d0813fd --- /dev/null +++ b/eclypse/policies/degrade/_helpers.py @@ -0,0 +1,356 @@ +"""Shared helpers for degradation policies.""" + +from __future__ import annotations + +from dataclasses import ( + dataclass, + field, +) +from typing import ( + TYPE_CHECKING, +) + +from eclypse.policies._filters import ( + coerce_numeric_like, + effective_assets, + ensure_numeric_value, + iter_selected_edges, + iter_selected_keys, + iter_selected_nodes, + normalize_selected_keys, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import ( + UpdatePolicy, + ValueAdjustmentDirection, + ValueAdjustmentOverride, + ValueAdjustmentOverrides, + ) + + +@dataclass(slots=True) +class ValueAdjustmentPolicy: + """Adjust selected asset values over a fixed number of epochs.""" + + direction: ValueAdjustmentDirection + factor: float | None = None + target: float | None = None + epochs: int = 1 + node_assets: str | list[str] | None = None + edge_assets: str | list[str] | None = None + node_ids: list[str] | None = None + node_filter: NodeFilter | None = None + edge_ids: list[tuple[str, str]] | None = None + edge_filter: EdgeFilter | None = None + step: int = 0 + initial_values: dict[tuple[str, ...], float] = field(default_factory=dict) + + def __post_init__(self): + """Validate the value-adjustment configuration.""" + validate_adjustment_parameters( + self.direction, + factor=self.factor, + target=self.target, + epochs=self.epochs, + ) + + def __call__(self, graph: AssetGraph): + """Apply the value adjustment to the selected assets.""" + if self.step >= self.epochs: + return + + selected_node_assets = normalize_selected_keys(self.node_assets) + selected_edge_assets = normalize_selected_keys(self.edge_assets) + + if selected_node_assets is not None: + for node_id, data in iter_selected_nodes( + graph, + node_ids=self.node_ids, + node_filter=self.node_filter, + ): + for key in iter_selected_keys(data, selected_node_assets): + data[key] = _adjust_value( + original=data[key], + current=ensure_numeric_value(key, data[key]), + state_key=("node", node_id, key), + policy=self, + ) + + if selected_edge_assets is not None: + for source, target_node, data in iter_selected_edges( + graph, + edge_ids=self.edge_ids, + edge_filter=self.edge_filter, + ): + for key in iter_selected_keys(data, selected_edge_assets): + data[key] = _adjust_value( + original=data[key], + current=ensure_numeric_value(key, data[key]), + state_key=("edge", source, target_node, key), + policy=self, + ) + + self.step += 1 + + +def build_value_adjustment_policy( + direction: ValueAdjustmentDirection, + *, + factor: float | None = None, + target: float | None = None, + epochs: int, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Build a stateful value-adjustment policy.""" + if node_assets is None and edge_assets is None: + raise ValueError("At least one of node_assets or edge_assets must be provided.") + + return ValueAdjustmentPolicy( + direction=direction, + factor=factor, + target=target, + epochs=epochs, + node_assets=node_assets, + edge_assets=edge_assets, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) + + +def build_configured_value_adjustment_policy( + direction: ValueAdjustmentDirection, + *, + factor: float | None = None, + target: float | None = None, + epochs: int | None = None, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_asset_overrides: ValueAdjustmentOverrides | None = None, + edge_asset_overrides: ValueAdjustmentOverrides | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Build a value-adjustment policy with defaults and per-asset overrides.""" + effective_node_assets = effective_assets(node_assets, node_asset_overrides) + effective_edge_assets = effective_assets(edge_assets, edge_asset_overrides) + + if not effective_node_assets and not effective_edge_assets: + raise ValueError( + "At least one of node_assets, edge_assets, " + "node_asset_overrides, or edge_asset_overrides must be provided." + ) + + validate_overrides( + direction, + { + **normalize_overrides("node_asset_overrides", node_asset_overrides), + **normalize_overrides("edge_asset_overrides", edge_asset_overrides), + }, + ) + + child_policies: list[UpdatePolicy] = [] + + for asset in effective_node_assets: + adjustment = resolve_adjustment( + direction, + asset_name=asset, + factor=factor, + target=target, + epochs=epochs, + per_asset_overrides=node_asset_overrides, + ) + child_policies.append( + build_value_adjustment_policy( + direction, + factor=adjustment.get("factor"), + target=adjustment.get("target"), + epochs=adjustment["epochs"], + node_assets=asset, + node_ids=node_ids, + node_filter=node_filter, + ) + ) + + for asset in effective_edge_assets: + adjustment = resolve_adjustment( + direction, + asset_name=asset, + factor=factor, + target=target, + epochs=epochs, + per_asset_overrides=edge_asset_overrides, + ) + child_policies.append( + build_value_adjustment_policy( + direction, + factor=adjustment.get("factor"), + target=adjustment.get("target"), + epochs=adjustment["epochs"], + edge_assets=asset, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) + ) + + def policy(graph: AssetGraph): + for child_policy in child_policies: + child_policy(graph) + + graph.logger.trace(f"Applied {direction} value policy.") + + return policy + + +def validate_adjustment_parameters( + direction: ValueAdjustmentDirection, + *, + factor: float | None, + target: float | None, + epochs: int | None, +) -> None: + """Validate a value-adjustment policy configuration.""" + if epochs is None: + raise ValueError("epochs must be provided.") + if epochs <= 0: + raise ValueError("epochs must be strictly positive.") + if (factor is None) == (target is None): + raise ValueError("Exactly one of factor or target must be provided.") + if direction not in {"increase", "reduce"}: + raise ValueError("direction must be either 'increase' or 'reduce'.") + + if factor is not None: + if direction == "increase" and factor < 1: + raise ValueError("factor must be greater than or equal to 1.") + if direction == "reduce" and not 0 <= factor <= 1: + raise ValueError("factor must be between 0 and 1.") + + if target is not None and target < 0: + raise ValueError("target must be non-negative.") + + +def normalize_overrides( + name: str, + overrides: ValueAdjustmentOverrides | None, +) -> dict[str, ValueAdjustmentOverride]: + """Normalise one or more named overrides into a flat mapping.""" + if overrides is None: + return {} + + return { + f"{name}[{asset_name!r}]": adjustment + for asset_name, adjustment in overrides.items() + } + + +def validate_overrides( + direction: ValueAdjustmentDirection, + overrides: dict[str, ValueAdjustmentOverride], +) -> None: + """Validate one or more named value-adjustment overrides.""" + for name, adjustment in overrides.items(): + _ensure_only_supported_adjustment_fields(name, adjustment) + validate_adjustment_parameters( + direction, + factor=adjustment.get("factor"), + target=adjustment.get("target"), + epochs=adjustment.get("epochs"), + ) + + +def resolve_adjustment( + direction: ValueAdjustmentDirection, + *, + asset_name: str, + factor: float | None, + target: float | None, + epochs: int | None, + per_asset_overrides: ValueAdjustmentOverrides | None, +) -> ValueAdjustmentOverride: + """Merge default and per-asset override settings for a selected asset.""" + adjustment: ValueAdjustmentOverride = {} + + if factor is not None: + adjustment["factor"] = factor + if target is not None: + adjustment["target"] = target + if epochs is not None: + adjustment["epochs"] = epochs + + override = (per_asset_overrides or {}).get(asset_name, {}) + if "factor" in override: + adjustment.pop("target", None) + if "target" in override: + adjustment.pop("factor", None) + adjustment.update(override) + validate_overrides(direction, {asset_name: adjustment}) + return adjustment + + +def _ensure_only_supported_adjustment_fields( + name: str, + adjustment: ValueAdjustmentOverride, +) -> None: + invalid_fields = sorted(set(adjustment) - {"factor", "target", "epochs"}) + if invalid_fields: + raise ValueError( + f"{name} uses unsupported fields: {', '.join(invalid_fields)}." + ) + + +def _adjust_value( + *, + original: object, + current: float, + state_key: tuple[str, ...], + policy: ValueAdjustmentPolicy, +) -> int | float: + if policy.factor is not None: + step_factor = policy.factor ** (1 / policy.epochs) + return coerce_numeric_like(original, current * step_factor) + + target_value = policy.target + assert target_value is not None + initial_value = policy.initial_values.setdefault(state_key, current) + _validate_target_direction(policy.direction, initial_value, target_value) + progress = min(policy.step + 1, policy.epochs) / policy.epochs + return coerce_numeric_like( + original, + interpolate_value(initial_value, target_value, progress), + ) + + +def _validate_target_direction( + direction: ValueAdjustmentDirection, + initial_value: float, + target_value: float, +) -> None: + if direction == "increase" and target_value < initial_value: + raise ValueError("target must be greater than or equal to the initial value.") + if direction == "reduce" and target_value > initial_value: + raise ValueError("target must be less than or equal to the initial value.") + + +def interpolate_value( + initial_value: float, + target_value: float, + progress: float, +) -> float: + """Interpolate smoothly between an initial value and a target.""" + if initial_value > 0 and target_value > 0: + return initial_value * ((target_value / initial_value) ** progress) + return initial_value + ((target_value - initial_value) * progress) diff --git a/eclypse/policies/degrade/increase.py b/eclypse/policies/degrade/increase.py new file mode 100644 index 0000000..030ed83 --- /dev/null +++ b/eclypse/policies/degrade/increase.py @@ -0,0 +1,78 @@ +"""Generic value-increase policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.degrade._helpers import ( + build_configured_value_adjustment_policy, +) + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import ( + UpdatePolicy, + ValueAdjustmentOverrides, + ) + + +def increase( + *, + factor: float | None = None, + target: float | None = None, + epochs: int | None = None, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_asset_overrides: ValueAdjustmentOverrides | None = None, + edge_asset_overrides: ValueAdjustmentOverrides | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Increase selected asset values over a fixed number of epochs. + + The policy applies either a relative ``factor`` or an absolute ``target`` + to the selected node and edge assets. Default parameters can be provided + once and then refined with ``node_asset_overrides`` or + ``edge_asset_overrides`` for specific assets. + + Args: + factor: Relative multiplicative factor applied to each selected asset. + Provide either ``factor`` or ``target``. + target: Absolute value reached by each selected asset at the end of the + adjustment horizon. Provide either ``factor`` or ``target``. + epochs: Number of evolution steps over which the increase is applied. + node_assets: Node asset names using the default adjustment + configuration. + edge_assets: Edge asset names using the default adjustment + configuration. + node_asset_overrides: Per-node-asset overrides for ``factor``, + ``target``, or ``epochs``. + edge_asset_overrides: Per-edge-asset overrides for ``factor``, + ``target``, or ``epochs``. + node_ids: Optional subset of node identifiers to update. + node_filter: Optional predicate used to select nodes dynamically. + edge_ids: Optional subset of edge identifiers to update. + edge_filter: Optional predicate used to select edges dynamically. + + Returns: + A graph update policy that increases the selected asset values. + """ + return build_configured_value_adjustment_policy( + "increase", + factor=factor, + target=target, + epochs=epochs, + node_assets=node_assets, + edge_assets=edge_assets, + node_asset_overrides=node_asset_overrides, + edge_asset_overrides=edge_asset_overrides, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/degrade/reduce.py b/eclypse/policies/degrade/reduce.py new file mode 100644 index 0000000..faee0af --- /dev/null +++ b/eclypse/policies/degrade/reduce.py @@ -0,0 +1,78 @@ +"""Generic value-reduction policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.degrade._helpers import ( + build_configured_value_adjustment_policy, +) + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import ( + UpdatePolicy, + ValueAdjustmentOverrides, + ) + + +def reduce( + *, + factor: float | None = None, + target: float | None = None, + epochs: int | None = None, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_asset_overrides: ValueAdjustmentOverrides | None = None, + edge_asset_overrides: ValueAdjustmentOverrides | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Reduce selected asset values over a fixed number of epochs. + + The policy applies either a relative ``factor`` or an absolute ``target`` + to the selected node and edge assets. Default parameters can be provided + once and then refined with ``node_asset_overrides`` or + ``edge_asset_overrides`` for specific assets. + + Args: + factor: Relative multiplicative factor applied to each selected asset. + Provide either ``factor`` or ``target``. + target: Absolute value reached by each selected asset at the end of the + adjustment horizon. Provide either ``factor`` or ``target``. + epochs: Number of evolution steps over which the reduction is applied. + node_assets: Node asset names using the default adjustment + configuration. + edge_assets: Edge asset names using the default adjustment + configuration. + node_asset_overrides: Per-node-asset overrides for ``factor``, + ``target``, or ``epochs``. + edge_asset_overrides: Per-edge-asset overrides for ``factor``, + ``target``, or ``epochs``. + node_ids: Optional subset of node identifiers to update. + node_filter: Optional predicate used to select nodes dynamically. + edge_ids: Optional subset of edge identifiers to update. + edge_filter: Optional predicate used to select edges dynamically. + + Returns: + A graph update policy that reduces the selected asset values. + """ + return build_configured_value_adjustment_policy( + "reduce", + factor=factor, + target=target, + epochs=epochs, + node_assets=node_assets, + edge_assets=edge_assets, + node_asset_overrides=node_asset_overrides, + edge_asset_overrides=edge_asset_overrides, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/distribution/__init__.py b/eclypse/policies/distribution/__init__.py new file mode 100644 index 0000000..51a40b4 --- /dev/null +++ b/eclypse/policies/distribution/__init__.py @@ -0,0 +1,21 @@ +"""Distribution-based built-in policies.""" + +from .beta import beta +from .categorical import categorical +from .gamma import gamma +from .lognormal import lognormal +from .normal import normal +from .triangular import triangular +from .truncated_normal import truncated_normal +from .uniform import uniform + +__all__ = [ + "beta", + "categorical", + "gamma", + "lognormal", + "normal", + "triangular", + "truncated_normal", + "uniform", +] diff --git a/eclypse/policies/distribution/_helpers.py b/eclypse/policies/distribution/_helpers.py new file mode 100644 index 0000000..d74823e --- /dev/null +++ b/eclypse/policies/distribution/_helpers.py @@ -0,0 +1,310 @@ +"""Shared helpers for distribution-based policies.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Any, +) + +from eclypse.policies._filters import ( + clamp, + coerce_numeric_like, + effective_assets, + ensure_numeric_value, + iter_selected_edges, + iter_selected_keys, + iter_selected_nodes, +) + +if TYPE_CHECKING: + from random import Random + + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import ( + Distribution, + UpdatePolicy, + ) + + +_BUILTIN_DISTRIBUTION_CHECKS = { + "beta": [ + ( + lambda distribution: distribution[0] > 0 and distribution[1] > 0, + "must use strictly positive parameters.", + ), + ], + "gamma": [ + ( + lambda distribution: distribution[0] > 0 and distribution[1] > 0, + "must use strictly positive parameters.", + ), + ], + "lognormal": [ + ( + lambda distribution: distribution[1] >= 0, + "must use a non-negative sigma.", + ), + ], + "normal": [ + ( + lambda distribution: distribution[1] >= 0, + "must use a non-negative standard deviation.", + ), + ], + "uniform": [ + ( + lambda distribution: distribution[0] <= distribution[1], + "must be ordered as (low, high).", + ), + ], +} + + +def build_distribution_policy( + kind: Distribution, + *, + node_assets: str | list[str] | None, + edge_assets: str | list[str] | None, + node_distribution: tuple[float, float], + edge_distribution: tuple[float, float] | None, + node_asset_distributions: dict[str, tuple[float, float]] | None, + edge_asset_distributions: dict[str, tuple[float, float]] | None, + minimum: float, + node_ids: list[str] | None, + node_filter: NodeFilter | None, + edge_ids: list[tuple[str, str]] | None, + edge_filter: EdgeFilter | None, +) -> UpdatePolicy: + """Build a distribution-based multiplicative update policy.""" + effective_edge_distribution = ( + node_distribution if edge_distribution is None else edge_distribution + ) + checks = _BUILTIN_DISTRIBUTION_CHECKS[kind] + validate_distributions( + { + **normalize_distributions( + "node_distribution", + node_distribution, + ), + **normalize_distributions( + "edge_distribution", + effective_edge_distribution, + ), + **normalize_distributions( + "node_asset_distributions", + node_asset_distributions, + ), + **normalize_distributions( + "edge_asset_distributions", + edge_asset_distributions, + ), + }, + checks=checks, + ) + + return build_sampled_distribution_policy( + kind=kind, + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=effective_edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + sampler=lambda rnd, distribution: sample_distribution(rnd, kind, distribution), + ) + + +def build_sampled_distribution_policy( + *, + kind: str, + node_assets: str | list[str] | None, + edge_assets: str | list[str] | None, + node_distribution: Any, + edge_distribution: Any, + node_asset_distributions: dict[str, Any] | None, + edge_asset_distributions: dict[str, Any] | None, + minimum: float, + node_ids: list[str] | None, + node_filter: NodeFilter | None, + edge_ids: list[tuple[str, str]] | None, + edge_filter: EdgeFilter | None, + sampler: Any, +) -> UpdatePolicy: + """Build a multiplicative update policy from a custom distribution sampler.""" + effective_node_assets = effective_assets(node_assets, node_asset_distributions) + effective_edge_assets = effective_assets(edge_assets, edge_asset_distributions) + + if not effective_node_assets and not effective_edge_assets: + raise ValueError( + "At least one of node_assets, edge_assets, " + "node_asset_distributions, or edge_asset_distributions must be provided." + ) + + log_message = build_distribution_log_message( + kind, + node_distribution=node_distribution, + edge_distribution=edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + ) + + def policy(graph: AssetGraph): + for _, data in iter_selected_nodes( + graph, + node_ids=node_ids, + node_filter=node_filter, + ): + for key in iter_selected_keys(data, effective_node_assets): + distribution = ( + node_asset_distributions.get(key, node_distribution) + if node_asset_distributions is not None + else node_distribution + ) + current = ensure_numeric_value(key, data[key]) + new_value = current * sampler(graph.rnd, distribution) + data[key] = coerce_numeric_like( + data[key], + clamp(new_value, lower=minimum), + ) + + for _, _, data in iter_selected_edges( + graph, + edge_ids=edge_ids, + edge_filter=edge_filter, + ): + for key in iter_selected_keys(data, effective_edge_assets): + distribution = ( + edge_asset_distributions.get(key, edge_distribution) + if edge_asset_distributions is not None + else edge_distribution + ) + current = ensure_numeric_value(key, data[key]) + new_value = current * sampler(graph.rnd, distribution) + data[key] = coerce_numeric_like( + data[key], + clamp(new_value, lower=minimum), + ) + + graph.logger.trace(log_message) + + return policy + + +def build_distribution_log_message( + kind: str, + *, + node_distribution: Any, + edge_distribution: Any, + node_asset_distributions: dict[str, Any] | None, + edge_asset_distributions: dict[str, Any] | None, +) -> str: + """Build a compact trace message describing a distribution policy.""" + has_overrides = bool(node_asset_distributions or edge_asset_distributions) + return ( + f"Applied {kind} distribution policy " + f"[node=({describe_distribution(kind, node_distribution)}), " + f"edge=({describe_distribution(kind, edge_distribution)}), " + f"overrides={'yes' if has_overrides else 'no'}]." + ) + + +def describe_distribution(kind: str, distribution: Any) -> str: + """Describe a distribution with kind-appropriate parameter names.""" + description: str + + if kind == "uniform": + low, high = distribution + description = f"low={low}, high={high}" + elif kind in {"normal", "truncated_normal"}: + mean, std = distribution + description = f"mean={mean}, std={std}" + elif kind == "lognormal": + mu, sigma = distribution + description = f"mu={mu}, sigma={sigma}" + elif kind == "beta": + alpha, beta_param = distribution + description = f"alpha={alpha}, beta={beta_param}" + elif kind == "gamma": + shape, scale = distribution + description = f"shape={shape}, scale={scale}" + elif kind == "triangular": + low, high, mode = distribution + description = f"low={low}, high={high}, mode={mode}" + elif kind == "categorical": + description = f"choices={len(distribution[0])}" + else: + description = str(distribution) + + return description + + +def normalize_distributions( + name: str, + distributions: Any | dict[str, Any] | None, +) -> dict[str, Any]: + """Normalise one or more named distributions into a flat mapping.""" + if distributions is None: + return {} + + if isinstance(distributions, dict): + return { + f"{name}[{distribution_name!r}]": distribution + for distribution_name, distribution in distributions.items() + } + + return {name: distributions} + + +def validate_distributions( + distributions: dict[str, Any], + *, + checks: list[tuple[Any, str]], +) -> None: + """Validate one or more named distributions against predicate-based checks.""" + for name, distribution in distributions.items(): + for predicate, message in checks: + if not predicate(distribution): + raise ValueError(f"{name} {message}") + + +def sample_distribution( + rnd: Random, + kind: Distribution, + distribution: tuple[float, float], +) -> float: + """Sample a multiplier from the requested distribution.""" + first, second = distribution + + if kind == "normal": + return rnd.gauss(first, second) + + if kind == "lognormal": + return rnd.lognormvariate(first, second) + + if kind == "beta": + return rnd.betavariate(first, second) + + if kind == "gamma": + return rnd.gammavariate(first, second) + + return rnd.uniform(first, second) + + +__all__ = [ + "build_distribution_log_message", + "build_distribution_policy", + "build_sampled_distribution_policy", + "describe_distribution", + "normalize_distributions", + "validate_distributions", +] diff --git a/eclypse/policies/distribution/beta.py b/eclypse/policies/distribution/beta.py new file mode 100644 index 0000000..e6311d4 --- /dev/null +++ b/eclypse/policies/distribution/beta.py @@ -0,0 +1,68 @@ +"""Beta-distribution resource policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import build_distribution_policy +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def beta( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: tuple[float, float] = (2.0, 2.0), + edge_distribution: tuple[float, float] | None = None, + node_asset_distributions: dict[str, tuple[float, float]] | None = None, + edge_asset_distributions: dict[str, tuple[float, float]] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply multiplicative beta noise to selected node and edge assets. + + Args: + node_assets (str | list[str] | None): Node assets to perturb. + edge_assets (str | list[str] | None): Edge assets to perturb. + node_distribution (tuple[float, float]): Default ``(alpha, beta)`` pair + used for node multipliers. + edge_distribution (tuple[float, float] | None): Default ``(alpha, beta)`` + pair used for edge multipliers. Defaults to ``node_distribution``. + node_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-node-asset overrides for ``node_distribution``. + edge_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-edge-asset overrides for ``edge_distribution``. + minimum (float): Lower clamp applied after perturbation. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy applying beta multiplicative noise. + """ + return build_distribution_policy( + "beta", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/distribution/categorical.py b/eclypse/policies/distribution/categorical.py new file mode 100644 index 0000000..74fab4b --- /dev/null +++ b/eclypse/policies/distribution/categorical.py @@ -0,0 +1,202 @@ +"""Categorical-distribution resource policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import ( + build_sampled_distribution_policy, + normalize_distributions, + validate_distributions, +) +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def categorical( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: list[float] | None = None, + edge_distribution: list[float] | None = None, + node_asset_distributions: dict[str, list[float]] | None = None, + edge_asset_distributions: dict[str, list[float]] | None = None, + node_weights: list[float] | None = None, + edge_weights: list[float] | None = None, + node_asset_weights: dict[str, list[float]] | None = None, + edge_asset_weights: dict[str, list[float]] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply multiplicative categorical noise to selected node and edge assets. + + Args: + node_assets (str | list[str] | None): Node assets to perturb. + edge_assets (str | list[str] | None): Edge assets to perturb. + node_distribution (list[float] | None): Default node multipliers to sample + from. Defaults to ``[0.95, 1.0, 1.05]``. + edge_distribution (list[float] | None): Default edge multipliers to sample + from. Defaults to ``node_distribution``. + node_asset_distributions (dict[str, list[float]] | None): Optional + per-node-asset overrides for ``node_distribution``. + edge_asset_distributions (dict[str, list[float]] | None): Optional + per-edge-asset overrides for ``edge_distribution``. + node_weights (list[float] | None): Optional default weights for + ``node_distribution``. + edge_weights (list[float] | None): Optional default weights for + ``edge_distribution``. Defaults to ``node_weights``. + node_asset_weights (dict[str, list[float]] | None): Optional per-node-asset + weight overrides. + edge_asset_weights (dict[str, list[float]] | None): Optional per-edge-asset + weight overrides. + minimum (float): Lower clamp applied after perturbation. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy applying categorical multiplicative + noise. + """ + effective_node_distribution = ( + [0.95, 1.0, 1.05] if node_distribution is None else node_distribution + ) + effective_edge_distribution = ( + effective_node_distribution if edge_distribution is None else edge_distribution + ) + effective_edge_weights = node_weights if edge_weights is None else edge_weights + + checks = [(bool, "must not be empty.")] + validate_distributions( + { + **normalize_distributions( + "node_distribution", + effective_node_distribution, + ), + **normalize_distributions( + "edge_distribution", + effective_edge_distribution, + ), + **normalize_distributions( + "node_asset_distributions", + node_asset_distributions, + ), + **normalize_distributions( + "edge_asset_distributions", + edge_asset_distributions, + ), + }, + checks=checks, + ) + validate_weights( + normalize_weight_sets( + "node_weights", + effective_node_distribution, + node_weights, + "node_asset_weights", + node_asset_distributions, + node_asset_weights, + ) + | normalize_weight_sets( + "edge_weights", + effective_edge_distribution, + effective_edge_weights, + "edge_asset_weights", + edge_asset_distributions, + edge_asset_weights, + ) + ) + + return build_sampled_distribution_policy( + kind="categorical", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=(effective_node_distribution, node_weights), + edge_distribution=(effective_edge_distribution, effective_edge_weights), + node_asset_distributions=_merge_distributions_and_weights( + node_asset_distributions, + node_asset_weights, + ), + edge_asset_distributions=_merge_distributions_and_weights( + edge_asset_distributions, + edge_asset_weights, + ), + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + sampler=lambda rnd, distribution: rnd.choices( + distribution[0], + weights=distribution[1], + k=1, + )[0], + ) + + +def normalize_weight_sets( + default_name: str, + default_distribution: list[float], + default_weights: list[float] | None, + asset_name: str, + asset_distributions: dict[str, list[float]] | None, + asset_weights: dict[str, list[float]] | None, +) -> dict[str, tuple[list[float], list[float] | None]]: + """Normalise default and per-asset categorical weights into one mapping.""" + normalized_weights = { + default_name: (default_distribution, default_weights), + } + + if asset_weights is None: + return normalized_weights + + if asset_distributions is None: + raise ValueError(f"{asset_name} requires matching per-asset distributions.") + + for key, weights in asset_weights.items(): + distribution = asset_distributions.get(key) + if distribution is None: + raise ValueError(f"{asset_name}[{key!r}] requires a matching distribution.") + normalized_weights[f"{asset_name}[{key!r}]"] = (distribution, weights) + + return normalized_weights + + +def validate_weights( + weight_sets: dict[str, tuple[list[float], list[float] | None]], +) -> None: + """Validate one or more named categorical weight sets.""" + for name, (distribution, weights) in weight_sets.items(): + if weights is None: + continue + if len(distribution) != len(weights): + raise ValueError(f"{name} must match the distribution length.") + if any(weight < 0 for weight in weights): + raise ValueError(f"{name} must use non-negative weights.") + if all(weight == 0 for weight in weights): + raise ValueError(f"{name} must contain at least one positive weight.") + + +def _merge_distributions_and_weights( + asset_distributions: dict[str, list[float]] | None, + asset_weights: dict[str, list[float]] | None, +) -> dict[str, tuple[list[float], list[float] | None]] | None: + """Combine per-asset categorical choices and weights for the generic helper.""" + if asset_distributions is None: + return None + + return { + key: (distribution, None if asset_weights is None else asset_weights.get(key)) + for key, distribution in asset_distributions.items() + } diff --git a/eclypse/policies/distribution/gamma.py b/eclypse/policies/distribution/gamma.py new file mode 100644 index 0000000..b218892 --- /dev/null +++ b/eclypse/policies/distribution/gamma.py @@ -0,0 +1,68 @@ +"""Gamma-distribution resource policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import build_distribution_policy +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def gamma( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: tuple[float, float] = (2.0, 0.5), + edge_distribution: tuple[float, float] | None = None, + node_asset_distributions: dict[str, tuple[float, float]] | None = None, + edge_asset_distributions: dict[str, tuple[float, float]] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply multiplicative gamma noise to selected node and edge assets. + + Args: + node_assets (str | list[str] | None): Node assets to perturb. + edge_assets (str | list[str] | None): Edge assets to perturb. + node_distribution (tuple[float, float]): Default ``(shape, scale)`` pair + used for node multipliers. + edge_distribution (tuple[float, float] | None): Default ``(shape, scale)`` + pair used for edge multipliers. Defaults to ``node_distribution``. + node_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-node-asset overrides for ``node_distribution``. + edge_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-edge-asset overrides for ``edge_distribution``. + minimum (float): Lower clamp applied after perturbation. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy applying gamma multiplicative noise. + """ + return build_distribution_policy( + "gamma", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/distribution/lognormal.py b/eclypse/policies/distribution/lognormal.py new file mode 100644 index 0000000..6d1f16f --- /dev/null +++ b/eclypse/policies/distribution/lognormal.py @@ -0,0 +1,69 @@ +"""Lognormal-distribution resource policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import build_distribution_policy +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def lognormal( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: tuple[float, float] = (0.0, 0.05), + edge_distribution: tuple[float, float] | None = None, + node_asset_distributions: dict[str, tuple[float, float]] | None = None, + edge_asset_distributions: dict[str, tuple[float, float]] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply multiplicative lognormal noise to selected node and edge assets. + + Args: + node_assets (str | list[str] | None): Node assets to perturb. + edge_assets (str | list[str] | None): Edge assets to perturb. + node_distribution (tuple[float, float]): Default ``(mu, sigma)`` pair + used for node multipliers. + edge_distribution (tuple[float, float] | None): Default ``(mu, sigma)`` + pair used for edge multipliers. Defaults to ``node_distribution``. + node_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-node-asset overrides for ``node_distribution``. + edge_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-edge-asset overrides for ``edge_distribution``. + minimum (float): Lower clamp applied after perturbation. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy applying lognormal multiplicative + noise. + """ + return build_distribution_policy( + "lognormal", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/distribution/normal.py b/eclypse/policies/distribution/normal.py new file mode 100644 index 0000000..0489226 --- /dev/null +++ b/eclypse/policies/distribution/normal.py @@ -0,0 +1,68 @@ +"""Normal-distribution resource policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import build_distribution_policy +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def normal( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: tuple[float, float] = (1.0, 0.05), + edge_distribution: tuple[float, float] | None = None, + node_asset_distributions: dict[str, tuple[float, float]] | None = None, + edge_asset_distributions: dict[str, tuple[float, float]] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply multiplicative Gaussian noise to selected node and edge assets. + + Args: + node_assets (str | list[str] | None): Node assets to perturb. + edge_assets (str | list[str] | None): Edge assets to perturb. + node_distribution (tuple[float, float]): Default ``(mean, std)`` pair used + for node multipliers. + edge_distribution (tuple[float, float] | None): Default ``(mean, std)`` + pair used for edge multipliers. Defaults to ``node_distribution``. + node_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-node-asset overrides for ``node_distribution``. + edge_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-edge-asset overrides for ``edge_distribution``. + minimum (float): Lower clamp applied after perturbation. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy applying Gaussian multiplicative noise. + """ + return build_distribution_policy( + "normal", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/distribution/triangular.py b/eclypse/policies/distribution/triangular.py new file mode 100644 index 0000000..e17bde9 --- /dev/null +++ b/eclypse/policies/distribution/triangular.py @@ -0,0 +1,110 @@ +"""Triangular-distribution resource policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import ( + build_sampled_distribution_policy, + normalize_distributions, + validate_distributions, +) +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def triangular( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: tuple[float, float, float] = (0.95, 1.05, 1.0), + edge_distribution: tuple[float, float, float] | None = None, + node_asset_distributions: dict[str, tuple[float, float, float]] | None = None, + edge_asset_distributions: dict[str, tuple[float, float, float]] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply multiplicative triangular noise to selected node and edge assets. + + Args: + node_assets (str | list[str] | None): Node assets to perturb. + edge_assets (str | list[str] | None): Edge assets to perturb. + node_distribution (tuple[float, float, float]): Default + ``(low, high, mode)`` triple used for node multipliers. + edge_distribution (tuple[float, float, float] | None): Default + ``(low, high, mode)`` triple used for edge multipliers. Defaults to + ``node_distribution``. + node_asset_distributions (dict[str, tuple[float, float, float]] | None): + Optional per-node-asset overrides for ``node_distribution``. + edge_asset_distributions (dict[str, tuple[float, float, float]] | None): + Optional per-edge-asset overrides for ``edge_distribution``. + minimum (float): Lower clamp applied after perturbation. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy applying triangular multiplicative + noise. + """ + effective_edge_distribution = ( + node_distribution if edge_distribution is None else edge_distribution + ) + checks = [ + ( + lambda distribution: distribution[0] <= distribution[1], + "must be ordered as (low, high, mode).", + ), + ( + lambda distribution: distribution[0] <= distribution[2] <= distribution[1], + "must use a mode contained in [low, high].", + ), + ] + validate_distributions( + { + **normalize_distributions( + "node_distribution", + node_distribution, + ), + **normalize_distributions( + "edge_distribution", + effective_edge_distribution, + ), + **normalize_distributions( + "node_asset_distributions", + node_asset_distributions, + ), + **normalize_distributions( + "edge_asset_distributions", + edge_asset_distributions, + ), + }, + checks=checks, + ) + + return build_sampled_distribution_policy( + kind="triangular", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=effective_edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + sampler=lambda rnd, distribution: rnd.triangular(*distribution), + ) diff --git a/eclypse/policies/distribution/truncated_normal.py b/eclypse/policies/distribution/truncated_normal.py new file mode 100644 index 0000000..57f98b9 --- /dev/null +++ b/eclypse/policies/distribution/truncated_normal.py @@ -0,0 +1,141 @@ +"""Truncated-normal resource policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import clamp +from eclypse.policies.distribution._helpers import ( + build_sampled_distribution_policy, + normalize_distributions, + validate_distributions, +) +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def truncated_normal( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: tuple[float, float] = (1.0, 0.05), + edge_distribution: tuple[float, float] | None = None, + node_asset_distributions: dict[str, tuple[float, float]] | None = None, + edge_asset_distributions: dict[str, tuple[float, float]] | None = None, + lower: float = 0.0, + upper: float | None = None, + max_attempts: int = 100, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply multiplicative truncated-normal noise to selected assets. + + Args: + node_assets (str | list[str] | None): Node assets to perturb. + edge_assets (str | list[str] | None): Edge assets to perturb. + node_distribution (tuple[float, float]): Default ``(mean, std)`` pair + used for node multipliers. + edge_distribution (tuple[float, float] | None): Default ``(mean, std)`` + pair used for edge multipliers. Defaults to ``node_distribution``. + node_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-node-asset overrides for ``node_distribution``. + edge_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-edge-asset overrides for ``edge_distribution``. + lower (float): Lower bound for sampled multipliers. + upper (float | None): Optional upper bound for sampled multipliers. + max_attempts (int): Maximum rejection-sampling attempts before clamping. + minimum (float): Lower clamp applied after perturbation. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy applying truncated-normal + multiplicative noise. + """ + effective_edge_distribution = ( + node_distribution if edge_distribution is None else edge_distribution + ) + checks = [ + ( + lambda distribution: distribution[1] >= 0, + "must use a non-negative standard deviation.", + ), + ] + validate_distributions( + { + **normalize_distributions( + "node_distribution", + node_distribution, + ), + **normalize_distributions( + "edge_distribution", + effective_edge_distribution, + ), + **normalize_distributions( + "node_asset_distributions", + node_asset_distributions, + ), + **normalize_distributions( + "edge_asset_distributions", + edge_asset_distributions, + ), + }, + checks=checks, + ) + + if upper is not None and lower > upper: + raise ValueError("lower must not be greater than upper.") + if max_attempts <= 0: + raise ValueError("max_attempts must be strictly positive.") + + return build_sampled_distribution_policy( + kind="truncated_normal", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=effective_edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + sampler=lambda rnd, distribution: _sample_truncated_normal( + rnd, + distribution, + lower=lower, + upper=upper, + max_attempts=max_attempts, + ), + ) + + +def _sample_truncated_normal( + rnd, + distribution: tuple[float, float], + *, + lower: float, + upper: float | None, + max_attempts: int, +) -> float: + """Sample a truncated normal value with bounded rejection sampling.""" + mean, std = distribution + for _ in range(max_attempts): + value = rnd.gauss(mean, std) + if value >= lower and (upper is None or value <= upper): + return value + + return clamp(value, lower=lower, upper=upper) diff --git a/eclypse/policies/distribution/uniform.py b/eclypse/policies/distribution/uniform.py new file mode 100644 index 0000000..e79d9c7 --- /dev/null +++ b/eclypse/policies/distribution/uniform.py @@ -0,0 +1,68 @@ +"""Uniform-distribution resource policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.distribution._helpers import build_distribution_policy +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def uniform( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + node_distribution: tuple[float, float] = (0.95, 1.05), + edge_distribution: tuple[float, float] | None = None, + node_asset_distributions: dict[str, tuple[float, float]] | None = None, + edge_asset_distributions: dict[str, tuple[float, float]] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply multiplicative uniform noise to selected node and edge assets. + + Args: + node_assets (str | list[str] | None): Node assets to perturb. + edge_assets (str | list[str] | None): Edge assets to perturb. + node_distribution (tuple[float, float]): Default ``(low, high)`` pair used + for node multipliers. + edge_distribution (tuple[float, float] | None): Default ``(low, high)`` + pair used for edge multipliers. Defaults to ``node_distribution``. + node_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-node-asset overrides for ``node_distribution``. + edge_asset_distributions (dict[str, tuple[float, float]] | None): Optional + per-edge-asset overrides for ``edge_distribution``. + minimum (float): Lower clamp applied after perturbation. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy applying uniform multiplicative noise. + """ + return build_distribution_policy( + "uniform", + node_assets=node_assets, + edge_assets=edge_assets, + node_distribution=node_distribution, + edge_distribution=edge_distribution, + node_asset_distributions=node_asset_distributions, + edge_asset_distributions=edge_asset_distributions, + minimum=minimum, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + ) diff --git a/eclypse/policies/failure/__init__.py b/eclypse/policies/failure/__init__.py new file mode 100644 index 0000000..b165911 --- /dev/null +++ b/eclypse/policies/failure/__init__.py @@ -0,0 +1,15 @@ +"""Built-in failure-oriented update policies.""" + +from __future__ import annotations + +from .availability_flap import availability_flap +from .kill_nodes import kill_nodes +from .latency_spike import latency_spike +from .revive_nodes import revive_nodes + +__all__ = [ + "availability_flap", + "kill_nodes", + "latency_spike", + "revive_nodes", +] diff --git a/eclypse/policies/failure/_helpers.py b/eclypse/policies/failure/_helpers.py new file mode 100644 index 0000000..2c780f8 --- /dev/null +++ b/eclypse/policies/failure/_helpers.py @@ -0,0 +1,11 @@ +"""Shared helpers for failure-oriented update policies.""" + +from __future__ import annotations + + +def validate_probability(name: str, value: float | None): + """Validate an optional probability value.""" + if value is None: + return + if not 0 <= value <= 1: + raise ValueError(f"{name} must be between 0 and 1.") diff --git a/eclypse/policies/failure/availability_flap.py b/eclypse/policies/failure/availability_flap.py new file mode 100644 index 0000000..adfd0c3 --- /dev/null +++ b/eclypse/policies/failure/availability_flap.py @@ -0,0 +1,72 @@ +"""Availability flapping policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + ensure_numeric_value, + iter_selected_nodes, +) +from eclypse.policies.failure._helpers import validate_probability +from eclypse.utils.constants import ( + MAX_AVAILABILITY, + MIN_AVAILABILITY, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import NodeFilter + from eclypse.utils.types import UpdatePolicy + + +def availability_flap( + down_probability: float, + *, + up_probability: float | None = None, + down_availability: float = MIN_AVAILABILITY, + up_availability: float = MAX_AVAILABILITY, + availability_key: str = "availability", + unavailable_at_or_below: float = MIN_AVAILABILITY, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, +) -> UpdatePolicy: + """Toggle node availability up and down according to separate probabilities. + + Args: + down_probability (float): Probability of taking an available node down. + up_probability (float | None): Probability of restoring an unavailable node. + Defaults to ``down_probability`` when omitted. + down_availability (float): Availability value assigned to failed nodes. + up_availability (float): Availability value assigned to restored nodes. + availability_key (str): Node asset storing availability. + unavailable_at_or_below (float): Threshold below which a node is considered + unavailable. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + + Returns: + UpdatePolicy: A graph update policy implementing flapping behaviour. + """ + validate_probability("down_probability", down_probability) + validate_probability("up_probability", up_probability) + effective_up_probability = ( + down_probability if up_probability is None else up_probability + ) + + def policy(graph: AssetGraph): + for _, data in iter_selected_nodes( + graph, + node_ids=node_ids, + node_filter=node_filter, + ): + current = ensure_numeric_value(availability_key, data[availability_key]) + if current <= unavailable_at_or_below: + if graph.rnd.random() < effective_up_probability: + data[availability_key] = up_availability + elif graph.rnd.random() < down_probability: + data[availability_key] = down_availability + + graph.logger.trace("Applied availability_flap policy.") + + return policy diff --git a/eclypse/policies/failure/kill_nodes.py b/eclypse/policies/failure/kill_nodes.py new file mode 100644 index 0000000..d876a9e --- /dev/null +++ b/eclypse/policies/failure/kill_nodes.py @@ -0,0 +1,69 @@ +"""Random node failure policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + ensure_numeric_value, + iter_selected_nodes, +) +from eclypse.policies.failure._helpers import validate_probability +from eclypse.utils.constants import MIN_AVAILABILITY + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import NodeFilter + from eclypse.utils.types import UpdatePolicy + + +def kill_nodes( + probability: float, + *, + revive_probability: float | None = None, + down_availability: float = MIN_AVAILABILITY, + revived_availability: float = 0.99, + availability_key: str = "availability", + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, +) -> UpdatePolicy: + """Randomly mark selected nodes as unavailable, with optional revival. + + Args: + probability (float): Probability of marking a selected node as unavailable. + revive_probability (float | None): Optional probability of reviving an + unavailable selected node. + down_availability (float): Availability value assigned to failed nodes. + revived_availability (float): Availability value assigned to revived nodes. + availability_key (str): Node asset storing availability. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + + Returns: + UpdatePolicy: A graph update policy implementing node failures. + """ + validate_probability("probability", probability) + validate_probability("revive_probability", revive_probability) + + def policy(graph: AssetGraph): + for _, data in iter_selected_nodes( + graph, + node_ids=node_ids, + node_filter=node_filter, + ): + availability = ensure_numeric_value( + availability_key, + data[availability_key], + ) + if graph.rnd.random() < probability: + data[availability_key] = down_availability + elif ( + revive_probability is not None + and availability <= down_availability + and graph.rnd.random() < revive_probability + ): + data[availability_key] = revived_availability + + graph.logger.trace("Applied kill_nodes policy.") + + return policy diff --git a/eclypse/policies/failure/latency_spike.py b/eclypse/policies/failure/latency_spike.py new file mode 100644 index 0000000..d1b06f7 --- /dev/null +++ b/eclypse/policies/failure/latency_spike.py @@ -0,0 +1,81 @@ +"""Latency spike policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + clamp, + coerce_numeric_like, + ensure_numeric_value, + iter_selected_edges, +) +from eclypse.policies.failure._helpers import validate_probability +from eclypse.utils.constants import MIN_LATENCY + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import EdgeFilter + from eclypse.utils.types import UpdatePolicy + + +def latency_spike( + probability: float, + *, + min_increase: float = 1.0, + max_increase: float | None = None, + factor: float | None = None, + latency_key: str = "latency", + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Inject random latency spikes on selected edges. + + Args: + probability (float): Probability of applying a spike to each selected edge. + min_increase (float): Minimum additive spike size when using additive mode. + max_increase (float | None): Maximum additive spike size when using additive + mode. Defaults to ``min_increase``. + factor (float | None): Optional multiplicative spike factor. When provided, + additive spike parameters are ignored. + latency_key (str): Edge asset storing latency. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy implementing latency spikes. + """ + validate_probability("probability", probability) + if factor is not None and factor < 0: + raise ValueError("factor must be non-negative.") + if min_increase < 0: + raise ValueError("min_increase must be non-negative.") + + spike_ceiling = min_increase if max_increase is None else max_increase + if spike_ceiling < min_increase: + raise ValueError("max_increase must be greater than or equal to min_increase.") + + def policy(graph: AssetGraph): + for _, _, data in iter_selected_edges( + graph, + edge_ids=edge_ids, + edge_filter=edge_filter, + ): + if graph.rnd.random() >= probability: + continue + + current = ensure_numeric_value(latency_key, data[latency_key]) + if factor is not None: + new_value = current * factor + else: + new_value = current + graph.rnd.uniform(min_increase, spike_ceiling) + + data[latency_key] = coerce_numeric_like( + data[latency_key], + clamp(new_value, lower=MIN_LATENCY), + ) + + graph.logger.trace("Applied latency_spike policy.") + + return policy diff --git a/eclypse/policies/failure/revive_nodes.py b/eclypse/policies/failure/revive_nodes.py new file mode 100644 index 0000000..46cc159 --- /dev/null +++ b/eclypse/policies/failure/revive_nodes.py @@ -0,0 +1,57 @@ +"""Random node revival policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + ensure_numeric_value, + iter_selected_nodes, +) +from eclypse.policies.failure._helpers import validate_probability +from eclypse.utils.constants import MIN_AVAILABILITY + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import NodeFilter + from eclypse.utils.types import UpdatePolicy + + +def revive_nodes( + probability: float, + *, + availability: float = 0.99, + availability_key: str = "availability", + unavailable_at_or_below: float = MIN_AVAILABILITY, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, +) -> UpdatePolicy: + """Randomly restore selected unavailable nodes. + + Args: + probability (float): Probability of reviving each selected unavailable node. + availability (float): Availability value assigned to revived nodes. + availability_key (str): Node asset storing availability. + unavailable_at_or_below (float): Threshold below which a node is considered + unavailable. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + + Returns: + UpdatePolicy: A graph update policy implementing node revival. + """ + validate_probability("probability", probability) + + def policy(graph: AssetGraph): + for _, data in iter_selected_nodes( + graph, + node_ids=node_ids, + node_filter=node_filter, + ): + current = ensure_numeric_value(availability_key, data[availability_key]) + if current <= unavailable_at_or_below and graph.rnd.random() < probability: + data[availability_key] = availability + + graph.logger.trace("Applied revive_nodes policy.") + + return policy diff --git a/eclypse/policies/noise/__init__.py b/eclypse/policies/noise/__init__.py new file mode 100644 index 0000000..a5908d8 --- /dev/null +++ b/eclypse/policies/noise/__init__.py @@ -0,0 +1,14 @@ +"""Built-in stochastic drift and noise policies.""" + +from __future__ import annotations + + +from .bounded_random_walk import bounded_random_walk +from .impulse import impulse +from .momentum_walk import momentum_walk + +__all__ = [ + "bounded_random_walk", + "impulse", + "momentum_walk", +] diff --git a/eclypse/policies/noise/_helpers.py b/eclypse/policies/noise/_helpers.py new file mode 100644 index 0000000..4a234f8 --- /dev/null +++ b/eclypse/policies/noise/_helpers.py @@ -0,0 +1,97 @@ +"""Shared helpers for noise policies.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Any, +) + +from eclypse.policies._filters import ( + clamp, + coerce_numeric_like, + ensure_numeric_value, + iter_selected_keys, +) +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from random import Random + + +def validate_steps( + *, + node_steps: dict[str, float] | None, + edge_steps: dict[str, float] | None, +) -> None: + """Validate additive walk step sizes.""" + if not node_steps and not edge_steps: + raise ValueError("At least one of node_steps or edge_steps must be provided.") + + for key, step in (node_steps or {}).items(): + if step < 0: + raise ValueError(f'node step for "{key}" must be non-negative.') + + for key, step in (edge_steps or {}).items(): + if step < 0: + raise ValueError(f'edge step for "{key}" must be non-negative.') + + +def apply_additive_walk( + values: dict[str, object], + steps: dict[str, float], + bounds: dict[str, tuple[float | None, float | None]] | None, + *, + delta_sampler: Any, +) -> None: + """Apply additive updates sampled independently per configured asset.""" + for key, step in steps.items(): + if key not in values: + continue + + current = ensure_numeric_value(key, values[key]) + lower, upper = (bounds or {}).get(key, (MIN_FLOAT, None)) + delta = delta_sampler(key, step) + values[key] = coerce_numeric_like( + values[key], + clamp(current + delta, lower=lower, upper=upper), + ) + + +def validate_probability(name: str, probability: float) -> None: + """Validate a probability parameter.""" + if probability < 0 or probability > 1: + raise ValueError(f"{name} must be between 0 and 1.") + + +def validate_factor_range(name: str, factor_range: tuple[float, float]) -> None: + """Validate a multiplicative factor range.""" + lower, upper = factor_range + if lower < 0: + raise ValueError(f"{name} must use non-negative factors.") + if lower > upper: + raise ValueError(f"{name} must be ordered as (low, high).") + + +def apply_impulses( + values: dict[str, object], + assets: str | list[str] | None, + *, + probability: float, + factor_range: tuple[float, float], + minimum: float, + random: Random, +) -> None: + """Apply rare multiplicative shocks to selected numeric assets.""" + lower_factor, upper_factor = factor_range + + for key in iter_selected_keys(values, assets): + if random.random() >= probability: + continue + + current = ensure_numeric_value(key, values[key]) + factor = random.uniform(lower_factor, upper_factor) + values[key] = coerce_numeric_like( + values[key], + clamp(current * factor, lower=minimum), + ) diff --git a/eclypse/policies/noise/bounded_random_walk.py b/eclypse/policies/noise/bounded_random_walk.py new file mode 100644 index 0000000..6a4beb5 --- /dev/null +++ b/eclypse/policies/noise/bounded_random_walk.py @@ -0,0 +1,83 @@ +"""Bounded random walk policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + iter_selected_edges, + iter_selected_nodes, +) +from eclypse.policies.noise._helpers import ( + apply_additive_walk, + validate_steps, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def bounded_random_walk( + *, + node_steps: dict[str, float] | None = None, + edge_steps: dict[str, float] | None = None, + node_bounds: dict[str, tuple[float | None, float | None]] | None = None, + edge_bounds: dict[str, tuple[float | None, float | None]] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply additive random walks while clamping values within bounds. + + Args: + node_steps (dict[str, float] | None): Maximum additive step per node asset. + edge_steps (dict[str, float] | None): Maximum additive step per edge asset. + node_bounds (dict[str, tuple[float | None, float | None]] | None): Optional + lower/upper bounds for node assets. + edge_bounds (dict[str, tuple[float | None, float | None]] | None): Optional + lower/upper bounds for edge assets. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy applying bounded random walks. + """ + validate_steps(node_steps=node_steps, edge_steps=edge_steps) + + def policy(graph: AssetGraph): + for _, data in iter_selected_nodes( + graph, + node_ids=node_ids, + node_filter=node_filter, + ): + apply_additive_walk( + data, + node_steps or {}, + node_bounds, + delta_sampler=lambda _, step: graph.rnd.uniform(-step, step), + ) + + for _, _, data in iter_selected_edges( + graph, + edge_ids=edge_ids, + edge_filter=edge_filter, + ): + apply_additive_walk( + data, + edge_steps or {}, + edge_bounds, + delta_sampler=lambda _, step: graph.rnd.uniform(-step, step), + ) + + graph.logger.trace("Applied bounded_random_walk policy.") + + return policy diff --git a/eclypse/policies/noise/impulse.py b/eclypse/policies/noise/impulse.py new file mode 100644 index 0000000..809bf4a --- /dev/null +++ b/eclypse/policies/noise/impulse.py @@ -0,0 +1,102 @@ +"""Impulse noise policy.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies._filters import ( + iter_selected_edges, + iter_selected_nodes, +) +from eclypse.policies.noise._helpers import ( + apply_impulses, + validate_factor_range, + validate_probability, +) +from eclypse.utils.constants import MIN_FLOAT + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +def impulse( + *, + node_assets: str | list[str] | None = None, + edge_assets: str | list[str] | None = None, + probability: float = 0.05, + node_factor_range: tuple[float, float] = (0.5, 1.5), + edge_factor_range: tuple[float, float] | None = None, + minimum: float = MIN_FLOAT, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply rare multiplicative shocks to selected node and edge assets. + + Args: + node_assets (str | list[str] | None): Node assets eligible for impulses. + edge_assets (str | list[str] | None): Edge assets eligible for impulses. + probability (float): Per-asset probability of an impulse at each epoch. + node_factor_range (tuple[float, float]): Multiplicative factor range used + for shocked node assets. + edge_factor_range (tuple[float, float] | None): Multiplicative factor + range used for shocked edge assets. Defaults to ``node_factor_range``. + minimum (float): Lower clamp applied after a shock. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy applying rare multiplicative shocks. + """ + if node_assets is None and edge_assets is None: + raise ValueError("At least one of node_assets or edge_assets must be provided.") + + validate_probability("probability", probability) + validate_factor_range("node_factor_range", node_factor_range) + + effective_edge_factor_range = ( + node_factor_range if edge_factor_range is None else edge_factor_range + ) + validate_factor_range("edge_factor_range", effective_edge_factor_range) + + def policy(graph: AssetGraph): + for _, data in iter_selected_nodes( + graph, + node_ids=node_ids, + node_filter=node_filter, + ): + apply_impulses( + data, + node_assets, + probability=probability, + factor_range=node_factor_range, + minimum=minimum, + random=graph.rnd, + ) + + for _, _, data in iter_selected_edges( + graph, + edge_ids=edge_ids, + edge_filter=edge_filter, + ): + apply_impulses( + data, + edge_assets, + probability=probability, + factor_range=effective_edge_factor_range, + minimum=minimum, + random=graph.rnd, + ) + + graph.logger.trace("Applied impulse policy.") + + return policy diff --git a/eclypse/policies/noise/momentum_walk.py b/eclypse/policies/noise/momentum_walk.py new file mode 100644 index 0000000..ef796d3 --- /dev/null +++ b/eclypse/policies/noise/momentum_walk.py @@ -0,0 +1,129 @@ +"""Momentum random walk policy.""" + +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + TypeVar, +) + +from eclypse.policies._filters import ( + clamp, + iter_selected_edges, + iter_selected_nodes, +) +from eclypse.policies.noise._helpers import ( + apply_additive_walk, + validate_steps, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import UpdatePolicy + + +StateKeyT = TypeVar("StateKeyT", tuple[str, str], tuple[str, str, str]) + + +def momentum_walk( + *, + node_steps: dict[str, float] | None = None, + edge_steps: dict[str, float] | None = None, + node_bounds: dict[str, tuple[float | None, float | None]] | None = None, + edge_bounds: dict[str, tuple[float | None, float | None]] | None = None, + momentum: float = 0.75, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, +) -> UpdatePolicy: + """Apply additive random walks with directional persistence. + + Args: + node_steps (dict[str, float] | None): Maximum additive step per node asset. + edge_steps (dict[str, float] | None): Maximum additive step per edge asset. + node_bounds (dict[str, tuple[float | None, float | None]] | None): Optional + lower/upper bounds for node assets. + edge_bounds (dict[str, tuple[float | None, float | None]] | None): Optional + lower/upper bounds for edge assets. + momentum (float): Fraction of the previous additive step reused at the + next epoch. Must be between 0 and 1. + node_ids (list[str] | None): Optional explicit list of node ids to target. + node_filter (NodeFilter | None): Optional predicate to filter target nodes. + edge_ids (list[tuple[str, str]] | None): Optional explicit list of target + edges. + edge_filter (EdgeFilter | None): Optional predicate to filter target edges. + + Returns: + UpdatePolicy: A graph update policy applying momentum random walks. + """ + validate_steps(node_steps=node_steps, edge_steps=edge_steps) + + if momentum < 0 or momentum > 1: + raise ValueError("momentum must be between 0 and 1.") + + previous_node_deltas: dict[tuple[str, str], float] = {} + previous_edge_deltas: dict[tuple[str, str, str], float] = {} + + def policy(graph: AssetGraph): + for node_id, data in iter_selected_nodes( + graph, + node_ids=node_ids, + node_filter=node_filter, + ): + apply_additive_walk( + data, + node_steps or {}, + node_bounds, + delta_sampler=lambda key, step, node_id=node_id: _sample_momentum_delta( + previous_node_deltas, + (node_id, key), + step, + momentum=momentum, + random=graph.rnd, + ), + ) + + for source, target, data in iter_selected_edges( + graph, + edge_ids=edge_ids, + edge_filter=edge_filter, + ): + apply_additive_walk( + data, + edge_steps or {}, + edge_bounds, + delta_sampler=lambda key, step, source=source, target=target: ( + _sample_momentum_delta( + previous_edge_deltas, + (source, target, key), + step, + momentum=momentum, + random=graph.rnd, + ) + ), + ) + + graph.logger.trace("Applied momentum_walk policy.") + + return policy + + +def _sample_momentum_delta( + previous_deltas: dict[StateKeyT, float], + state_key: StateKeyT, + step: float, + *, + momentum: float, + random, +) -> float: + """Sample a bounded additive delta with momentum from the previous epoch.""" + previous_delta = previous_deltas.get(state_key, 0.0) + candidate = momentum * previous_delta + random.uniform(-step, step) + delta = clamp(candidate, lower=-step, upper=step) + previous_deltas[state_key] = delta + return delta diff --git a/eclypse/policies/replay/__init__.py b/eclypse/policies/replay/__init__.py new file mode 100644 index 0000000..59f4178 --- /dev/null +++ b/eclypse/policies/replay/__init__.py @@ -0,0 +1,25 @@ +"""Built-in replay update policies.""" + +from __future__ import annotations + +from .from_dataframe import from_dataframe +from .from_parquet import from_parquet +from .from_records import from_records +from .replay_edges import ( + ReplayEdgesPolicy, + replay_edges, +) +from .replay_nodes import ( + ReplayNodesPolicy, + replay_nodes, +) + +__all__ = [ + "ReplayEdgesPolicy", + "ReplayNodesPolicy", + "from_dataframe", + "from_parquet", + "from_records", + "replay_edges", + "replay_nodes", +] diff --git a/eclypse/policies/replay/_helpers.py b/eclypse/policies/replay/_helpers.py new file mode 100644 index 0000000..3277974 --- /dev/null +++ b/eclypse/policies/replay/_helpers.py @@ -0,0 +1,79 @@ +"""Shared helpers for replay update policies.""" + +from __future__ import annotations + +from collections import defaultdict +from typing import ( + TYPE_CHECKING, + Any, +) + +if TYPE_CHECKING: + from eclypse.utils.types import MissingPolicyBehaviour + + +def normalise_records( + record_source: Any, +) -> list[dict[str, Any]]: + """Convert dataframe-like or iterable sources into plain dictionaries.""" + if hasattr(record_source, "to_dict"): + try: + records = record_source.to_dict(orient="records") + return [dict(record) for record in records] + except TypeError: + pass + + if hasattr(record_source, "iterrows"): + records = [] + for _, row in record_source.iterrows(): + if hasattr(row, "to_dict"): + records.append(dict(row.to_dict())) + else: + records.append(dict(row)) + return records + + return [dict(record) for record in record_source] + + +def infer_value_columns( + records: list[dict[str, Any]], + reserved_columns: list[str], + value_columns: list[str] | tuple[str, ...] | None, +) -> list[str]: + """Determine which record columns should be applied to the graph.""" + if value_columns is not None: + return list(value_columns) + if not records: + return [] + reserved = set(reserved_columns) + return [column for column in records[0] if column not in reserved] + + +def validate_missing_behaviour(missing: MissingPolicyBehaviour): + """Validate the behaviour used for missing graph items.""" + if missing not in {"ignore", "error"}: + raise ValueError('missing must be either "ignore" or "error".') + + +def group_records_by_step( + records: list[dict[str, Any]], + *, + time_column: str, +) -> dict[int, list[dict[str, Any]]]: + """Group records by simulation step.""" + records_by_step: dict[int, list[dict[str, Any]]] = defaultdict(list) + for record in records: + records_by_step[int(record[time_column])].append(record) + return records_by_step + + +def initial_step( + records_by_step: dict[int, list[dict[str, Any]]], + start_step: int | None, +) -> int: + """Resolve the step from which the replay should start.""" + if start_step is not None: + return start_step + if records_by_step: + return min(records_by_step) + return 0 diff --git a/eclypse/policies/replay/from_dataframe.py b/eclypse/policies/replay/from_dataframe.py new file mode 100644 index 0000000..ec13664 --- /dev/null +++ b/eclypse/policies/replay/from_dataframe.py @@ -0,0 +1,53 @@ +"""Replay policy builders from dataframe-like objects.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.replay._helpers import normalise_records +from eclypse.policies.replay.from_records import from_records + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import ( + MissingPolicyBehaviour, + ReplayTarget, + UpdatePolicy, + ) + + +def from_dataframe( + dataframe, + *, + target: ReplayTarget, + node_id_column: str = "node_id", + source_column: str = "source", + target_column: str = "target", + time_column: str = "time", + value_columns: list[str] | tuple[str, ...] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, + missing: MissingPolicyBehaviour = "ignore", + start_step: int | None = None, +) -> UpdatePolicy: + """Build a replay policy from a dataframe-like object.""" + return from_records( + normalise_records(dataframe), + target=target, + node_id_column=node_id_column, + source_column=source_column, + target_column=target_column, + time_column=time_column, + value_columns=value_columns, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + missing=missing, + start_step=start_step, + ) diff --git a/eclypse/policies/replay/from_parquet.py b/eclypse/policies/replay/from_parquet.py new file mode 100644 index 0000000..049d113 --- /dev/null +++ b/eclypse/policies/replay/from_parquet.py @@ -0,0 +1,59 @@ +"""Replay policy builders from parquet files.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.replay.from_dataframe import from_dataframe + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import ( + MissingPolicyBehaviour, + ReplayTarget, + UpdatePolicy, + ) + + +def from_parquet( + path: str, + *, + target: ReplayTarget, + node_id_column: str = "node_id", + source_column: str = "source", + target_column: str = "target", + time_column: str = "time", + value_columns: list[str] | tuple[str, ...] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, + missing: MissingPolicyBehaviour = "ignore", + start_step: int | None = None, +) -> UpdatePolicy: + """Build a replay policy from a parquet file using pandas when available.""" + try: + import pandas as pd + except ImportError as exc: # pragma: no cover - optional dependency + raise ImportError( + "from_parquet requires pandas with parquet support installed." + ) from exc + + return from_dataframe( + pd.read_parquet(path), + target=target, + node_id_column=node_id_column, + source_column=source_column, + target_column=target_column, + time_column=time_column, + value_columns=value_columns, + node_ids=node_ids, + node_filter=node_filter, + edge_ids=edge_ids, + edge_filter=edge_filter, + missing=missing, + start_step=start_step, + ) diff --git a/eclypse/policies/replay/from_records.py b/eclypse/policies/replay/from_records.py new file mode 100644 index 0000000..7eeb8b5 --- /dev/null +++ b/eclypse/policies/replay/from_records.py @@ -0,0 +1,62 @@ +"""Replay policy builders from plain records.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from eclypse.policies.replay.replay_edges import replay_edges +from eclypse.policies.replay.replay_nodes import replay_nodes + +if TYPE_CHECKING: + from eclypse.policies._filters import ( + EdgeFilter, + NodeFilter, + ) + from eclypse.utils.types import ( + MissingPolicyBehaviour, + ReplayTarget, + UpdatePolicy, + ) + + +def from_records( + record_source, + *, + target: ReplayTarget, + node_id_column: str = "node_id", + source_column: str = "source", + target_column: str = "target", + time_column: str = "time", + value_columns: list[str] | tuple[str, ...] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, + missing: MissingPolicyBehaviour = "ignore", + start_step: int | None = None, +) -> UpdatePolicy: + """Build a replay policy from plain Python records.""" + if target == "nodes": + return replay_nodes( + record_source, + node_id_column=node_id_column, + time_column=time_column, + value_columns=value_columns, + node_ids=node_ids, + node_filter=node_filter, + missing=missing, + start_step=start_step, + ) + if target == "edges": + return replay_edges( + record_source, + source_column=source_column, + target_column=target_column, + time_column=time_column, + value_columns=value_columns, + edge_ids=edge_ids, + edge_filter=edge_filter, + missing=missing, + start_step=start_step, + ) + raise ValueError('target must be either "nodes" or "edges".') diff --git a/eclypse/policies/replay/replay_edges.py b/eclypse/policies/replay/replay_edges.py new file mode 100644 index 0000000..2363e38 --- /dev/null +++ b/eclypse/policies/replay/replay_edges.py @@ -0,0 +1,122 @@ +"""Replay edge attributes from records.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + Any, +) + +from eclypse.policies.replay._helpers import ( + group_records_by_step, + infer_value_columns, + initial_step, + normalise_records, + validate_missing_behaviour, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import EdgeFilter + from eclypse.utils.types import ( + MissingPolicyBehaviour, + UpdatePolicy, + ) + + +@dataclass(slots=True) +class ReplayEdgesPolicy: + """Replay edge attributes from time-indexed records.""" + + records_by_step: dict[int, list[dict[str, Any]]] + columns: list[str] + source_column: str = "source" + target_column: str = "target" + selected_edge_ids: set[tuple[str, str]] | None = None + edge_filter: EdgeFilter | None = None + missing: MissingPolicyBehaviour = "ignore" + current_step: int = 0 + + def __call__(self, graph: AssetGraph): + """Apply the replay records for the current step to matching edges.""" + for record in self.records_by_step.get(self.current_step, []): + _update_edge_from_record( + graph, + record, + columns=self.columns, + source_column=self.source_column, + target_column=self.target_column, + selected_edge_ids=self.selected_edge_ids, + edge_filter=self.edge_filter, + missing=self.missing, + ) + + graph.logger.trace(f"Applied replay_edges policy for step {self.current_step}.") + self.current_step += 1 + + +def replay_edges( + record_source, + *, + source_column: str = "source", + target_column: str = "target", + time_column: str = "time", + value_columns: list[str] | tuple[str, ...] | None = None, + edge_ids: list[tuple[str, str]] | None = None, + edge_filter: EdgeFilter | None = None, + missing: MissingPolicyBehaviour = "ignore", + start_step: int | None = None, +) -> UpdatePolicy: + """Replay edge attributes from time-indexed records.""" + validate_missing_behaviour(missing) + records = normalise_records(record_source) + columns = infer_value_columns( + records, + reserved_columns=[source_column, target_column, time_column], + value_columns=value_columns, + ) + records_by_step = group_records_by_step(records, time_column=time_column) + + selected_edge_ids = set(edge_ids) if edge_ids is not None else None + current_step = initial_step(records_by_step, start_step) + return ReplayEdgesPolicy( + records_by_step=records_by_step, + columns=columns, + source_column=source_column, + target_column=target_column, + selected_edge_ids=selected_edge_ids, + edge_filter=edge_filter, + missing=missing, + current_step=current_step, + ) + + +def _update_edge_from_record( + graph: AssetGraph, + record, + *, + columns: list[str], + source_column: str, + target_column: str, + selected_edge_ids: set[tuple[str, str]] | None, + edge_filter, + missing: MissingPolicyBehaviour, +): + edge_id = (record[source_column], record[target_column]) + if selected_edge_ids is not None and edge_id not in selected_edge_ids: + return + if not graph.has_edge(*edge_id): + if missing == "error": + raise KeyError( + f'Edge "{edge_id[0]} -> {edge_id[1]}" not found in the graph.' + ) + return + + data = graph.edges[edge_id] + if edge_filter is not None and not edge_filter(*edge_id, data): + return + + for column in columns: + if column in record: + data[column] = record[column] diff --git a/eclypse/policies/replay/replay_nodes.py b/eclypse/policies/replay/replay_nodes.py new file mode 100644 index 0000000..69efa35 --- /dev/null +++ b/eclypse/policies/replay/replay_nodes.py @@ -0,0 +1,115 @@ +"""Replay node attributes from records.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + Any, +) + +from eclypse.policies.replay._helpers import ( + group_records_by_step, + infer_value_columns, + initial_step, + normalise_records, + validate_missing_behaviour, +) + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.policies._filters import NodeFilter + from eclypse.utils.types import ( + MissingPolicyBehaviour, + UpdatePolicy, + ) + + +@dataclass(slots=True) +class ReplayNodesPolicy: + """Replay node attributes from time-indexed records.""" + + records_by_step: dict[int, list[dict[str, Any]]] + columns: list[str] + node_id_column: str = "node_id" + selected_node_ids: set[str] | None = None + node_filter: NodeFilter | None = None + missing: MissingPolicyBehaviour = "ignore" + current_step: int = 0 + + def __call__(self, graph: AssetGraph): + """Apply the replay records for the current step to matching nodes.""" + for record in self.records_by_step.get(self.current_step, []): + _update_node_from_record( + graph, + record, + columns=self.columns, + node_id_column=self.node_id_column, + selected_node_ids=self.selected_node_ids, + node_filter=self.node_filter, + missing=self.missing, + ) + + graph.logger.trace(f"Applied replay_nodes policy for step {self.current_step}.") + self.current_step += 1 + + +def replay_nodes( + record_source, + *, + node_id_column: str = "node_id", + time_column: str = "time", + value_columns: list[str] | tuple[str, ...] | None = None, + node_ids: list[str] | None = None, + node_filter: NodeFilter | None = None, + missing: MissingPolicyBehaviour = "ignore", + start_step: int | None = None, +) -> UpdatePolicy: + """Replay node attributes from time-indexed records.""" + validate_missing_behaviour(missing) + records = normalise_records(record_source) + columns = infer_value_columns( + records, + reserved_columns=[node_id_column, time_column], + value_columns=value_columns, + ) + records_by_step = group_records_by_step(records, time_column=time_column) + + selected_node_ids = set(node_ids) if node_ids is not None else None + current_step = initial_step(records_by_step, start_step) + return ReplayNodesPolicy( + records_by_step=records_by_step, + columns=columns, + node_id_column=node_id_column, + selected_node_ids=selected_node_ids, + node_filter=node_filter, + missing=missing, + current_step=current_step, + ) + + +def _update_node_from_record( + graph: AssetGraph, + record, + *, + columns: list[str], + node_id_column: str, + selected_node_ids: set[str] | None, + node_filter, + missing: MissingPolicyBehaviour, +): + node_id = record[node_id_column] + if selected_node_ids is not None and node_id not in selected_node_ids: + return + if not graph.has_node(node_id): + if missing == "error": + raise KeyError(f'Node "{node_id}" not found in the graph.') + return + + data = graph.nodes[node_id] + if node_filter is not None and not node_filter(node_id, data): + return + + for column in columns: + if column in record: + data[column] = record[column] diff --git a/eclypse/policies/schedule/__init__.py b/eclypse/policies/schedule/__init__.py new file mode 100644 index 0000000..97296ff --- /dev/null +++ b/eclypse/policies/schedule/__init__.py @@ -0,0 +1,31 @@ +"""Scheduling wrappers for graph update policies.""" + +from __future__ import annotations + +from .after import ( + AfterPolicy, + after, +) +from .between import ( + BetweenPolicy, + between, +) +from .every import ( + EveryPolicy, + every, +) +from .once_at import ( + OnceAtPolicy, + once_at, +) + +__all__ = [ + "AfterPolicy", + "BetweenPolicy", + "EveryPolicy", + "OnceAtPolicy", + "after", + "between", + "every", + "once_at", +] diff --git a/eclypse/policies/schedule/after.py b/eclypse/policies/schedule/after.py new file mode 100644 index 0000000..3264a41 --- /dev/null +++ b/eclypse/policies/schedule/after.py @@ -0,0 +1,47 @@ +"""Run a policy from a given step onward.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class AfterPolicy: + """Run a policy from ``start`` onward.""" + + start: int + policy: UpdatePolicy + step: int = 0 + + def __post_init__(self): + """Validate the schedule configuration.""" + if self.start < 0: + raise ValueError("start must be non-negative.") + + def __call__(self, graph: AssetGraph): + """Apply the wrapped policy from the configured step onward.""" + if self.step >= self.start: + self.policy(graph) + graph.logger.trace(f"Triggered after policy at step {self.step}.") + self.step += 1 + + +def after( + start: int, + policy: UpdatePolicy, +) -> UpdatePolicy: + """Run a policy from ``start`` onward. + + Args: + start (int): First step at which the policy should run. + policy (UpdatePolicy): The wrapped policy. + + Returns: + UpdatePolicy: A scheduled wrapper around ``policy``. + """ + return AfterPolicy(start=start, policy=policy) diff --git a/eclypse/policies/schedule/between.py b/eclypse/policies/schedule/between.py new file mode 100644 index 0000000..cd43529 --- /dev/null +++ b/eclypse/policies/schedule/between.py @@ -0,0 +1,52 @@ +"""Run a policy between two step bounds.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class BetweenPolicy: + """Run a policy between two inclusive step bounds.""" + + start: int + end: int + policy: UpdatePolicy + step: int = 0 + + def __post_init__(self): + """Validate the schedule configuration.""" + if self.start < 0: + raise ValueError("start must be non-negative.") + if self.end < self.start: + raise ValueError("end must be greater than or equal to start.") + + def __call__(self, graph: AssetGraph): + """Apply the wrapped policy while the current step is within bounds.""" + if self.start <= self.step <= self.end: + self.policy(graph) + graph.logger.trace(f"Triggered between policy at step {self.step}.") + self.step += 1 + + +def between( + start: int, + end: int, + policy: UpdatePolicy, +) -> UpdatePolicy: + """Run a policy between two inclusive step bounds. + + Args: + start (int): First step at which the policy should run. + end (int): Last step at which the policy should run. + policy (UpdatePolicy): The wrapped policy. + + Returns: + UpdatePolicy: A scheduled wrapper around ``policy``. + """ + return BetweenPolicy(start=start, end=end, policy=policy) diff --git a/eclypse/policies/schedule/every.py b/eclypse/policies/schedule/every.py new file mode 100644 index 0000000..5f7b454 --- /dev/null +++ b/eclypse/policies/schedule/every.py @@ -0,0 +1,53 @@ +"""Run a policy at a fixed interval.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class EveryPolicy: + """Run a policy every ``interval`` steps starting from ``start``.""" + + interval: int + policy: UpdatePolicy + start: int = 0 + step: int = 0 + + def __post_init__(self): + """Validate the schedule configuration.""" + if self.interval <= 0: + raise ValueError("interval must be strictly positive.") + if self.start < 0: + raise ValueError("start must be non-negative.") + + def __call__(self, graph: AssetGraph): + """Apply the wrapped policy when the current step matches the interval.""" + if self.step >= self.start and (self.step - self.start) % self.interval == 0: + self.policy(graph) + graph.logger.trace(f"Triggered every policy at step {self.step}.") + self.step += 1 + + +def every( + interval: int, + policy: UpdatePolicy, + *, + start: int = 0, +) -> UpdatePolicy: + """Run a policy every ``interval`` steps starting from ``start``. + + Args: + interval (int): Number of steps between policy applications. + policy (UpdatePolicy): The wrapped policy. + start (int): First step at which the policy becomes eligible. + + Returns: + UpdatePolicy: A scheduled wrapper around ``policy``. + """ + return EveryPolicy(interval=interval, policy=policy, start=start) diff --git a/eclypse/policies/schedule/once_at.py b/eclypse/policies/schedule/once_at.py new file mode 100644 index 0000000..5889787 --- /dev/null +++ b/eclypse/policies/schedule/once_at.py @@ -0,0 +1,47 @@ +"""Run a policy only once.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from eclypse.graph.asset_graph import AssetGraph + from eclypse.utils.types import UpdatePolicy + + +@dataclass(slots=True) +class OnceAtPolicy: + """Run a policy only once at the specified step.""" + + step_at: int + policy: UpdatePolicy + step: int = 0 + + def __post_init__(self): + """Validate the schedule configuration.""" + if self.step_at < 0: + raise ValueError("step_at must be non-negative.") + + def __call__(self, graph: AssetGraph): + """Apply the wrapped policy when the configured step is reached.""" + if self.step == self.step_at: + self.policy(graph) + graph.logger.trace(f"Triggered once_at policy at step {self.step}.") + self.step += 1 + + +def once_at( + step_at: int, + policy: UpdatePolicy, +) -> UpdatePolicy: + """Run a policy only once at the specified step. + + Args: + step_at (int): Step at which the policy should run. + policy (UpdatePolicy): The wrapped policy. + + Returns: + UpdatePolicy: A scheduled wrapper around ``policy``. + """ + return OnceAtPolicy(step_at=step_at, policy=policy) diff --git a/eclypse/remote/_node/node.py b/eclypse/remote/_node/node.py index 29519f5..d3f2b13 100644 --- a/eclypse/remote/_node/node.py +++ b/eclypse/remote/_node/node.py @@ -23,6 +23,7 @@ from eclypse.remote.communication.rest import EclypseREST from eclypse.utils._logging import ( config_logger, + format_log_kv, logger, ) from eclypse.utils.constants import RND_SEED @@ -109,7 +110,10 @@ async def ops_entrypoint(self, engine_op: RemoteOps, **op_args) -> Any: engine_op (RemoteOps): The operation to be executed. **op_args: The arguments of the operation to be invoked. """ - self.logger.trace(f"Executing operation: {engine_op}, {op_args}") + self.logger.trace( + "Executing operation | " + + format_log_kv(operation=engine_op, arg_keys=sorted(op_args)) + ) return await self._engine_ops_thread.submit(engine_op, op_args) async def entrypoint( diff --git a/eclypse/remote/communication/route.py b/eclypse/remote/communication/route.py index f06280d..697e29e 100644 --- a/eclypse/remote/communication/route.py +++ b/eclypse/remote/communication/route.py @@ -10,12 +10,12 @@ dataclass, field, ) +from sys import getsizeof from typing import ( Any, ) from eclypse.utils.constants import MIN_LATENCY -from eclypse.utils.tools import get_bytes_size MILLISECONDS_PER_SECOND = 1000 """Number of milliseconds in one second.""" @@ -70,7 +70,7 @@ def cost(self, msg: Any) -> float: Returns: float: The function that computes the cost of the route. """ - msg_size = get_bytes_size(msg) + msg_size = _get_bytes_size(msg) return self.processing_time / MILLISECONDS_PER_SECOND + sum( (msg_size * BYTES_TO_MEGABITS / link.get("bandwidth", float("inf"))) + (link.get("latency", MIN_LATENCY) / MILLISECONDS_PER_SECOND) @@ -108,3 +108,36 @@ def __str__(self) -> str: result += f"to {self.recipient_id} ({self.recipient_node_id}):\n" result += " -> ".join(f"{s} -- {t} ({link})" for s, t, link in self.hops) return result + + +def _get_bytes_size(data: Any) -> int: + """Return the size of an object in bytes. + + The size is computed according to the following rules: + + - int, float, str, bool: the size is the size of the object itself. + - list, tuple, set: the size is the sum of the sizes of the + elements in the collection. + - dict: the size is the sum of the sizes of the keys and values in the dictionary. + - objects with a ``__dict__`` attribute: the size is the size of the + ``__dict__`` attribute. + - other objects: the size is the size of the object itself, computed using + ``sys.getsizeof``. + + Args: + data (Any): The object to be measured. + + Returns: + int: The size of the object in bytes. + """ + if isinstance(data, (int, float, str, bool)): + return getsizeof(data) + if isinstance(data, (list, tuple, set)): + return sum(_get_bytes_size(element) for element in data) + if isinstance(data, dict): + return sum( + _get_bytes_size(key) + _get_bytes_size(value) for key, value in data.items() + ) + if hasattr(data, "__dict__"): + return _get_bytes_size(data.__dict__) + return getsizeof(data) diff --git a/eclypse/remote/service/service.py b/eclypse/remote/service/service.py index c7800b3..0b4f724 100644 --- a/eclypse/remote/service/service.py +++ b/eclypse/remote/service/service.py @@ -46,7 +46,7 @@ from eclypse.utils._logging import Logger -SUPPORTED_COMMUNICATION_INTERFACES = get_args(CommunicationInterface.__value__) +_SUPPORTED_COMMUNICATION_INTERFACES = get_args(CommunicationInterface) """Supported runtime communication interfaces for remote services.""" @@ -68,7 +68,7 @@ def __init__( store_step (bool, optional): Whether to store the results of each step. Defaults to False. """ - if communication_interface not in SUPPORTED_COMMUNICATION_INTERFACES: + if communication_interface not in _SUPPORTED_COMMUNICATION_INTERFACES: raise ValueError("Invalid communication interface.") self._service_id: str = service_id @@ -99,7 +99,7 @@ async def run(self): try: step_result = await self.step() except RouteNotFoundError as error: - self.logger.error( + self.logger.warning( "Skipping service step " f"{self.step_count} because route to {error.recipient_id} " "was not found." diff --git a/eclypse/simulation/_simulator/local.py b/eclypse/simulation/_simulator/local.py index d029a99..d402d55 100644 --- a/eclypse/simulation/_simulator/local.py +++ b/eclypse/simulation/_simulator/local.py @@ -24,6 +24,7 @@ from eclypse.placement import PlacementManager from eclypse.utils._logging import ( + format_log_kv, logger, print_exception, ) @@ -169,8 +170,12 @@ async def enqueue_event(self, event_name: str, triggered_by: str | None = None): c_type = self._events[event_name].type self.logger.trace( (f"[{c_type}]" if c_type else "[event]") - + f" Enqueued '{event_name}'" - + (f", triggered by '{triggered_by}'" if triggered_by else "") + + " Enqueued event | " + + format_log_kv( + name=event_name, + triggered_by=triggered_by, + queue=self._events_queue.qsize(), + ) ) if event_name == STOP_EVENT: diff --git a/eclypse/simulation/_simulator/reporter.py b/eclypse/simulation/_simulator/reporter.py index 5616b59..190bb61 100644 --- a/eclypse/simulation/_simulator/reporter.py +++ b/eclypse/simulation/_simulator/reporter.py @@ -13,7 +13,10 @@ cast, ) -from eclypse.utils._logging import logger +from eclypse.utils._logging import ( + format_log_kv, + logger, +) from eclypse.utils.defaults import DEFAULT_REPORT_CHUNK_SIZE if TYPE_CHECKING: @@ -52,7 +55,10 @@ def __init__( def add_reporter(self, rtype: str, reporter: type[Reporter]): """Add a new reporter type dynamically.""" if rtype in self.reporters: - self.logger.warning(f"[{rtype}] Reporter already exists, skipping.") + self.logger.warning( + "Reporter already exists, skipping | " + + format_log_kv(type=rtype, path=self.report_path) + ) return self.reporters[rtype] = reporter(self.report_path) @@ -99,7 +105,14 @@ async def report(self, event_name: str, event_idx: int, callback: EclypseEvent): return for rtype in callback.report_types: if rtype not in self.reporters: - self.logger.warning(f"[{rtype}] No reporter registered, skipping.") + self.logger.warning( + "No reporter registered, skipping | " + + format_log_kv( + type=rtype, + callback_type=callback.type, + event=event_name, + ) + ) continue data = self.reporters[rtype].report(event_name, event_idx, callback) diff --git a/eclypse/simulation/simulation.py b/eclypse/simulation/simulation.py index 4d144aa..eaefc1b 100644 --- a/eclypse/simulation/simulation.py +++ b/eclypse/simulation/simulation.py @@ -13,7 +13,10 @@ from eclypse.report import Report from eclypse.simulation._simulator.local import Simulator from eclypse.simulation.config import SimulationConfig -from eclypse.utils._logging import logger +from eclypse.utils._logging import ( + format_log_kv, + logger, +) from eclypse.utils.constants import ( DRIVING_EVENT, START_EVENT, @@ -67,6 +70,7 @@ def __init__( ) self.simulator: Simulator | RemoteSimulator = _simulator self._report: Report | None = None + self._finished_logged = False def prepare_runtime(self): """Prepare the process environment required by the simulation runtime.""" @@ -85,6 +89,17 @@ def start(self): json.dump(self._sim_config.to_dict(), handle, indent=4) _local_remote_event_call(self.simulator, self.remote, START_EVENT) + self._finished_logged = False + self.logger.log( + "ECLYPSE", + "Simulation started | " + + format_log_kv( + infrastructure=self.infrastructure.id, + # apps=[app.id for app in self.applications.values()], + # path=self._sim_config.path, + # remote=self.remote is not None, + ), + ) def trigger(self, event_name: str): """Fire an event in the simulation.""" @@ -114,14 +129,19 @@ def wait(self, timeout: float | None = None): ray_backend.get(self.simulator.wait.remote(timeout=timeout)) # type: ignore[union-attr] else: self.simulator.wait(timeout=timeout) + if not self._finished_logged: + self.logger.log("ECLYPSE", "Simulation finished.") + self._finished_logged = True return except KeyboardInterrupt: if interrupted: raise interrupted = True - self.logger.warning("SIMULATION INTERRUPTED. Requesting graceful stop.") - self.logger.warning("Press Ctrl+C again to stop the simulation.") + self.logger.log( + "ECLYPSE", + "Simulation stop requested. Press Ctrl+C again to stop the simulation.", + ) self.stop(blocking=False) timeout = None @@ -158,6 +178,10 @@ def register( ) else: self.simulator.register(application, placement_strategy) + self.logger.debug( + "Registered application | " + + format_log_kv(app=application.id, remote=self.remote is not None) + ) @property def applications(self) -> dict[str, Application]: diff --git a/eclypse/utils/__init__.py b/eclypse/utils/__init__.py index 02b3446..88e530a 100644 --- a/eclypse/utils/__init__.py +++ b/eclypse/utils/__init__.py @@ -1,5 +1,4 @@ """Package for utility functions. -It comprises logging, constants used in ECLYPSE, and helper -functions for the other modules. +It comprises logging, constants and default values used in ECLYPSE. """ diff --git a/eclypse/utils/_logging.py b/eclypse/utils/_logging.py index de377ed..dd18cdd 100644 --- a/eclypse/utils/_logging.py +++ b/eclypse/utils/_logging.py @@ -87,6 +87,18 @@ def print_exception(e: Exception, raised_by: str): print(f"{e.__class__.__name__} in {raised_by}: {e}") +def format_log_kv(separator: str = " | ", **values: Any) -> str: + """Format keyword details as compact ``key=value`` pairs. + + Args: + separator (str): The separator used between key-value pairs. + **values: The values to format. + """ + return separator.join( + f"{key}={value}" for key, value in values.items() if value is not None + ) + + def log_placement_violations(vlogger: Logger, violations: dict[str, dict[str, Any]]): """Logs each placement violation with aligned formatting using the provided logger. @@ -96,13 +108,17 @@ def log_placement_violations(vlogger: Logger, violations: dict[str, dict[str, An A dictionary of violations, where each key maps to a dict with 'asset' and 'constraint' values. """ - total_pad = max(len(key) for key in violations) + 3 # +2 accounts for [ and ] + total_pad = max(len(key) for key in violations) + 3 # space +2 accounts for [ and ] for key, v in violations.items(): label = f" [{key}]" padded_label = label.rjust(total_pad) - vlogger.warning( - f"{padded_label} featured {v['featured']} | required {v['required']}" + vlogger.trace( + f"{padded_label} " + + format_log_kv( + available=v["featured"], + required=v["required"], + ) ) @@ -120,16 +136,24 @@ def log_assets_violations( A dictionary of violations, where each key maps to a dict with 'asset' and 'constraint' values. """ - total_pad = max(len(key) for key in violations) + 3 # +2 accounts for [ and ] + total_pad = max(len(key) for key in violations) + 3 # space +2 accounts for [ and ] for key, v in violations.items(): label = f" [{key}]" padded_label = label.rjust(total_pad) - vlogger.warning( - f"{padded_label} | " - f"{v} upper_bound {bucket[key].upper_bound} | " - f"lower_bound {bucket[key].lower_bound}" + vlogger.trace( + f"{padded_label} " + + format_log_kv( + value=v, + lower_bound=bucket[key].lower_bound, + upper_bound=bucket[key].upper_bound, + ) ) -__all__ = ["Logger", "log_placement_violations", "print_exception"] +__all__ = [ + "Logger", + "format_log_kv", + "log_placement_violations", + "print_exception", +] diff --git a/eclypse/utils/constants.py b/eclypse/utils/constants.py index 204b3f6..677411d 100644 --- a/eclypse/utils/constants.py +++ b/eclypse/utils/constants.py @@ -4,6 +4,8 @@ import sys +# Numeric domains + MIN_FLOAT = 0.0 """Smallest domain value accepted by numeric assets.""" @@ -13,6 +15,8 @@ FLOAT_EPSILON = sys.float_info.min """Smallest positive representable float.""" +# Environment + RND_SEED = "ECLYPSE_RND_SEED" """Environment variable used to seed deterministic randomness.""" @@ -22,6 +26,8 @@ LOG_FILE = "ECLYPSE_LOG_FILE" """Environment variable used to configure the log file path.""" +# Assets + MIN_BANDWIDTH = MIN_FLOAT """Lower bound used by bandwidth assets.""" @@ -40,9 +46,13 @@ MAX_AVAILABILITY = 1.0 """Upper bound used by availability assets.""" +# Infrastructure + COST_RECOMPUTATION_THRESHOLD = 0.05 """Relative threshold above which cached path costs are recomputed.""" +# Workflow + START_EVENT = "start" """Name of the default simulation start event.""" diff --git a/eclypse/utils/defaults.py b/eclypse/utils/defaults.py index 06b1258..98b9fd2 100644 --- a/eclypse/utils/defaults.py +++ b/eclypse/utils/defaults.py @@ -6,6 +6,8 @@ from eclypse.utils.constants import MAX_FLOAT +# Reporting + DEFAULT_REPORT_TYPE = "csv" """Default on-disk format used to write simulation reports.""" @@ -27,12 +29,16 @@ DEFAULT_REPORT_STEP = 1 """Default sampling step used by report queries.""" +# Simulation + SIMULATION_CONFIG_FILENAME = "config.json" """Filename used to persist the simulation configuration.""" SIMULATION_LOG_FILENAME = "simulation.log" """Filename used for simulation runtime logs.""" +# Reporters + CSV_REPORT_DIR = "csv" """Directory name used by the CSV reporter.""" @@ -54,6 +60,8 @@ } """Default Ray environment variables applied to simulation runtimes.""" +# Paths + def get_default_sim_path() -> Path: """Return the default path where simulation outputs are stored.""" diff --git a/eclypse/utils/tools.py b/eclypse/utils/tools.py deleted file mode 100644 index 956fec3..0000000 --- a/eclypse/utils/tools.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Module containing utility functions used throughout the ECLYPSE package.""" - -from __future__ import annotations - -import re -from sys import getsizeof -from typing import ( - TYPE_CHECKING, - Any, -) - -if TYPE_CHECKING: - from eclypse.graph.assets import AssetBucket - - -def get_bytes_size(d: Any) -> int: - """Returns the size of an object in bytes. - - The size is computed according to the following rules: - - - int, float, str, bool: the size is the size of the object itself. - - list, tuple, set: the size is the sum of the sizes of the - elements in the collection. - - dict: the size is the sum of the sizes of the keys and values in the dictionary. - - objects with a __dict__ attribute: the size is the size of the __dict__ attribute. - - other objects: the size is the size of the object itself, - computed using `sys.getsizeof`. - - Args: - d (Any): The object to be measured. - - Returns: - int: The size of the object in bytes. - """ - if isinstance(d, (int, float, str, bool)): - return getsizeof(d) - if isinstance(d, (list, tuple, set)): - return sum(get_bytes_size(e) for e in d) - if isinstance(d, dict): - return sum(get_bytes_size(k) + get_bytes_size(v) for k, v in d.items()) - if hasattr(d, "__dict__"): - return get_bytes_size(d.__dict__) - return getsizeof(d) - - -def camel_to_snake(s: str) -> str: - """Convert a CamelCase string to a snake_case string. - - .. code-block:: python - - s = "MyCamelCaseSentence" - print(camel_to_snake(s)) # my_camel_case_sentence - - Args: - s (str): The CamelCase string to convert. - - Returns: - str: The snake_case string. - """ - s = re.sub(r"(? Callable: "The decorator must be applied to a function or a class" + "that implements the __call__ method.", ) - _name = camel_to_snake(name if name else decoratee.__name__) + _name = _camel_to_snake(name if name else decoratee.__name__) _triggers = ( triggers if isinstance(triggers, list) else [triggers] if triggers else [] @@ -126,3 +126,20 @@ def __new__(cls, *args, **kwargs): if fn_or_class: return decorator(fn_or_class) return decorator + + +def _camel_to_snake(name: str) -> str: + """Convert a CamelCase string to a snake_case string. + + .. code-block:: python + + name = "MyCamelCaseSentence" + print(_camel_to_snake(name)) # my_camel_case_sentence + + Args: + name (str): The CamelCase string to convert. + + Returns: + str: The snake_case string. + """ + return re.sub(r"(? bool: condition = self.trigger_bucket.trigger(trigger_event=trigger_event) if self._verbose and condition: self.simulator.logger.debug( - f"Event {self.name.title()}-{self.n_triggers} triggered." + "Event triggered | " + + format_log_kv( + name=self.name, + type=self.type, + triggers=self.n_triggers, + ) ) return condition @@ -235,13 +246,23 @@ def _fire(self, trigger_event: EclypseEvent | None = None) -> Any: raise ValueError("The event must be associated to a simulator to be fired.") if self._verbose: - self.simulator.logger.log( - "ECLYPSE", f"Event {self.name.title()}-{self.n_calls} fired." + self.simulator.logger.debug( + "Event fired | " + + format_log_kv( + name=self.name, + type=self.type, + calls=self.n_calls, + ) ) event_data = self._call_by_type(trigger_event) self._data = event_data if event_data is not None else {} self.trigger_bucket.reset() + if self.name == DRIVING_EVENT: + self.simulator.logger.log( + "ECLYPSE", + "Simulation step | " + format_log_kv(step=self.n_calls), + ) @property def name(self) -> str: diff --git a/eclypse/workflow/trigger/bucket.py b/eclypse/workflow/trigger/bucket.py index b16d4ec..0c1d208 100644 --- a/eclypse/workflow/trigger/bucket.py +++ b/eclypse/workflow/trigger/bucket.py @@ -84,8 +84,6 @@ def trigger(self, trigger_event: EclypseEvent | None = None) -> bool: for trigger in _triggers: c = trigger.trigger(trigger_event) t_conditions.append(c) - if c: - self.logger.trace(f"{trigger}") triggerable = ( (any(t_conditions) if self.condition == "any" else all(t_conditions)) @@ -107,7 +105,6 @@ def __repr__(self) -> str: """Return a string representation of the trigger.""" return f"{self.__class__.__name__}" - @property def logger(self) -> Logger: """Get the logger for the event. diff --git a/eclypse/workflow/trigger/cascade.py b/eclypse/workflow/trigger/cascade.py index 38feaf6..74ab662 100644 --- a/eclypse/workflow/trigger/cascade.py +++ b/eclypse/workflow/trigger/cascade.py @@ -156,3 +156,10 @@ def trigger(self, trigger_event: EclypseEvent | None = None) -> bool: if self.rnd is None: raise RuntimeError("Trigger not initialised. Call init() before trigger().") return super().trigger(trigger_event) and self.rnd.random() < self.probability + + def __repr__(self) -> str: + """Return a string representation of the random cascade trigger.""" + return ( + f"RandomCascadeTrigger(trigger_event={self.trigger_event}, " + f"probability={self.probability})" + ) diff --git a/eclypse/workflow/trigger/trigger.py b/eclypse/workflow/trigger/trigger.py index 66e3fe8..6b7d3f6 100644 --- a/eclypse/workflow/trigger/trigger.py +++ b/eclypse/workflow/trigger/trigger.py @@ -101,10 +101,7 @@ def reset(self): def __repr__(self) -> str: """Return a string representation of the periodic trigger.""" ms = self.trigger_every_ms.microseconds // 1000 - return ( - f"PeriodicTrigger(trigger_every_ms={ms}, " - f"last_exec_time={self.last_exec_time})" - ) + return f"PeriodicTrigger(trigger_every_ms={ms})" class ScheduledTrigger(Trigger): @@ -153,6 +150,10 @@ def trigger(self, _: EclypseEvent | None = None) -> bool: return True return False + def __repr__(self) -> str: + """Return a string representation of the scheduled trigger.""" + return f"ScheduledTrigger(scheduled_times={self._scheduled_timedelta})" + class RandomTrigger(Trigger): """A trigger that fires randomly.""" diff --git a/examples/echo/infrastructure.py b/examples/echo/infrastructure.py index d4dcbaa..ddfae0f 100755 --- a/examples/echo/infrastructure.py +++ b/examples/echo/infrastructure.py @@ -1,7 +1,4 @@ -from update_policy import ( - edge_random_update, - node_random_update, -) +from update_policy import random_update from eclypse.graph import Infrastructure @@ -10,8 +7,7 @@ def get_infrastructure(seed: int = 2) -> Infrastructure: echo_infra = Infrastructure( "EchoInfrastructure", - node_update_policy=node_random_update, - edge_update_policy=edge_random_update, + update_policies=random_update, include_default_assets=True, seed=seed, ) diff --git a/examples/echo/notebook.ipynb b/examples/echo/notebook.ipynb index 7fd5660..eba73d4 100644 --- a/examples/echo/notebook.ipynb +++ b/examples/echo/notebook.ipynb @@ -39,7 +39,7 @@ "To define an infrastructure, you need to specify its nodes and links, together with the update policy that will be applied to the infrastructure.\n", "\n", "First of all, define an update policy that randomly changes the nodes and links resources. \n", - "To do so we just have to define two functions, taking as argument a `networkx.NodeView` and a `networkx.EdgeView`, respectively, and returning the updated set of nodes and edges." + "We can keep the node and edge update logic separate, then compose both in a single graph-level policy." ] }, { @@ -84,7 +84,12 @@ " )\n", " resources[\"bandwidth\"] = round(\n", " max(0, resources[\"bandwidth\"] * rnd.uniform(0.95, 1.05))\n", - " )" + " )\n", + "\n", + "\n", + "def random_update(graph):\n", + " node_random_update(graph.nodes)\n", + " edge_random_update(graph.edges)\n" ] }, { @@ -92,7 +97,7 @@ "metadata": {}, "source": [ "Now you can define the topology, by adding ndoes and links to an extend `networx.Graph`, that is our `eclypse.graph.Infrastructure` class.\n", - "The previsouly defined update policies will be parameters of the Infrastructure builder." + "The previously defined update policy will be a parameter of the Infrastructure builder." ] }, { @@ -105,8 +110,7 @@ "\n", "echo_infra = Infrastructure(\n", " \"EchoInfrastructure\",\n", - " node_update_policy=node_random_update,\n", - " edge_update_policy=edge_random_update,\n", + " update_policies=random_update,\n", " include_default_assets=True,\n", " seed=SEED,\n", ")\n", diff --git a/examples/echo/update_policy.py b/examples/echo/update_policy.py index 80040e8..7a9c158 100755 --- a/examples/echo/update_policy.py +++ b/examples/echo/update_policy.py @@ -1,13 +1,10 @@ import random as rnd -from networkx.classes.reportviews import ( - EdgeView, - NodeView, -) +from eclypse.graph import AssetGraph -def node_random_update(nodes: NodeView): - for _, resources in nodes.data(): +def random_update(graph: AssetGraph): + for _, resources in graph.nodes.data(): if rnd.random() < 0.02: resources["availability"] = 0 elif rnd.random() < 0.5 and resources["availability"] == 0: @@ -23,9 +20,7 @@ def node_random_update(nodes: NodeView): 1, max(0, resources["availability"] * rnd.uniform(0.995, 1.005)) ) - -def edge_random_update(edges: EdgeView): - for _, _, resources in edges.data(): + for _, _, resources in graph.edges.data(): resources["latency"] = round( max(0, resources["latency"] * rnd.uniform(0.9, 1.1)) ) diff --git a/examples/grid_analysis/applications/assembly_platform.py b/examples/grid_analysis/applications/assembly_platform.py index 48fed0e..fd2abef 100644 --- a/examples/grid_analysis/applications/assembly_platform.py +++ b/examples/grid_analysis/applications/assembly_platform.py @@ -1,22 +1,16 @@ from typing import ( - Callable, Dict, Optional, ) -from networkx.classes.reportviews import ( - EdgeView, - NodeView, -) - from eclypse.graph import Application from eclypse.graph.assets import Asset +from eclypse.utils.types import UpdatePolicies def get_assembly_platform( application_id: str = "AssemblyPlatform", - node_update_policy: Optional[Callable[[NodeView], None]] = None, - edge_update_policy: Optional[Callable[[EdgeView], None]] = None, + update_policies: UpdatePolicies = None, node_assets: Optional[Dict[str, Asset]] = None, edge_assets: Optional[Dict[str, Asset]] = None, include_default_assets: bool = True, @@ -49,8 +43,7 @@ def get_assembly_platform( app = Application( application_id=application_id, - node_update_policy=node_update_policy, - edge_update_policy=edge_update_policy, + update_policies=update_policies, node_assets=node_assets, edge_assets=edge_assets, include_default_assets=include_default_assets, diff --git a/examples/grid_analysis/applications/health_guardian.py b/examples/grid_analysis/applications/health_guardian.py index 84e02b3..f6eb0da 100644 --- a/examples/grid_analysis/applications/health_guardian.py +++ b/examples/grid_analysis/applications/health_guardian.py @@ -1,22 +1,16 @@ from typing import ( - Callable, Dict, Optional, ) -from networkx.classes.reportviews import ( - EdgeView, - NodeView, -) - from eclypse.graph import Application from eclypse.graph.assets import Asset +from eclypse.utils.types import UpdatePolicies def get_health_guardian( application_id: str = "HealthGuardian", - node_update_policy: Optional[Callable[[NodeView], None]] = None, - edge_update_policy: Optional[Callable[[EdgeView], None]] = None, + update_policies: UpdatePolicies = None, node_assets: Optional[Dict[str, Asset]] = None, edge_assets: Optional[Dict[str, Asset]] = None, include_default_assets: bool = True, @@ -47,8 +41,7 @@ def get_health_guardian( app = Application( application_id=application_id, - node_update_policy=node_update_policy, - edge_update_policy=edge_update_policy, + update_policies=update_policies, node_assets=node_assets, edge_assets=edge_assets, include_default_assets=include_default_assets, diff --git a/examples/grid_analysis/applications/vas.py b/examples/grid_analysis/applications/vas.py index 72bc4db..d6ea89d 100644 --- a/examples/grid_analysis/applications/vas.py +++ b/examples/grid_analysis/applications/vas.py @@ -1,22 +1,16 @@ from typing import ( - Callable, Dict, Optional, ) -from networkx.classes.reportviews import ( - EdgeView, - NodeView, -) - from eclypse.graph import Application from eclypse.graph.assets import Asset +from eclypse.utils.types import UpdatePolicies def get_vas( application_id: str = "VAS", - node_update_policy: Optional[Callable[[NodeView], None]] = None, - edge_update_policy: Optional[Callable[[EdgeView], None]] = None, + update_policies: UpdatePolicies = None, node_assets: Optional[Dict[str, Asset]] = None, edge_assets: Optional[Dict[str, Asset]] = None, include_default_assets: bool = True, @@ -43,8 +37,7 @@ def get_vas( app = Application( application_id=application_id, - node_update_policy=node_update_policy, - edge_update_policy=edge_update_policy, + update_policies=update_policies, node_assets=node_assets, edge_assets=edge_assets, include_default_assets=include_default_assets, diff --git a/examples/grid_analysis/infrastructure.py b/examples/grid_analysis/infrastructure.py index 2a344cd..a963a0a 100644 --- a/examples/grid_analysis/infrastructure.py +++ b/examples/grid_analysis/infrastructure.py @@ -31,10 +31,9 @@ def get_infrastructure(config) -> Infrastructure: - node_update_policy, link_update_policy = get_policy(config) + update_policies = get_policy(config) common_config = { - "node_update_policy": node_update_policy, - "link_update_policy": link_update_policy, + "update_policies": update_policies, "resource_init": "max", "symmetric": True, "seed": config["seed"], diff --git a/examples/grid_analysis/policies/degrade.py b/examples/grid_analysis/policies/degrade.py index e15eed3..f1e7204 100644 --- a/examples/grid_analysis/policies/degrade.py +++ b/examples/grid_analysis/policies/degrade.py @@ -1,7 +1,4 @@ -from networkx.classes.reportviews import ( - EdgeView, - NodeView, -) +from eclypse.graph import AssetGraph def degrade_value(value, degradation_rate): @@ -11,14 +8,13 @@ def degrade_value(value, degradation_rate): def degrade_policy(target_degradation: float, epochs: int): degradation_rate = 1 - (target_degradation ** (1 / epochs)) - def node_update_wrapper(nodes: NodeView): - for _, resources in nodes.data(): + def update_wrapper(graph: AssetGraph): + for _, resources in graph.nodes.data(): for key in resources: if key in ["cpu", "gpu", "ram", "storage", "availability"]: resources[key] = degrade_value(resources[key], degradation_rate) - def edge_update_wrapper(edges: EdgeView): - for _, _, resources in edges.data(): + for _, _, resources in graph.edges.data(): for key in resources: resources[key] = degrade_value(resources[key], degradation_rate) @@ -28,4 +24,4 @@ def edge_update_wrapper(edges: EdgeView): resources["bandwidth"], degradation_rate ) - return node_update_wrapper, edge_update_wrapper + return update_wrapper diff --git a/examples/grid_analysis/policies/random_kill.py b/examples/grid_analysis/policies/random_kill.py index c74d37f..0030efa 100644 --- a/examples/grid_analysis/policies/random_kill.py +++ b/examples/grid_analysis/policies/random_kill.py @@ -1,22 +1,16 @@ import random as rnd -from networkx.classes.reportviews import ( - EdgeView, - NodeView, -) +from eclypse.graph import AssetGraph def kill_policy(kill_probability: float): revive_probability = kill_probability / 2 - def node_update_wrapper(nodes: NodeView): - for _, resources in nodes.data(): + def update_wrapper(graph: AssetGraph): + for _, resources in graph.nodes.data(): if rnd.random() < kill_probability: resources["availability"] = 0 elif rnd.random() < revive_probability: resources["availability"] = 0.99 - def edge_update_wrapper(_: EdgeView): - pass - - return node_update_wrapper, edge_update_wrapper + return update_wrapper diff --git a/examples/image_prediction/main.py b/examples/image_prediction/main.py index 4621dae..e682a91 100644 --- a/examples/image_prediction/main.py +++ b/examples/image_prediction/main.py @@ -39,7 +39,7 @@ infrastructure_id="IPInfr", n_clients=5, seed=seed, - link_update_policy=DegradePolicy(epochs=STEPS), + update_policies=DegradePolicy(epochs=STEPS), include_default_assets=True, resource_init="max", symmetric=True, diff --git a/examples/image_prediction/update_policy.py b/examples/image_prediction/update_policy.py index 851dd01..66b4267 100644 --- a/examples/image_prediction/update_policy.py +++ b/examples/image_prediction/update_policy.py @@ -1,7 +1,8 @@ from collections import defaultdict import numpy as np -from networkx.classes.reportviews import EdgeView + +from eclypse.graph import AssetGraph def exponential_decay(init, target, N, decay_rate=None): @@ -27,9 +28,9 @@ def __init__(self, epochs: int): self.i = 0 self.target_latency = 1000 - def __call__(self, edges: EdgeView): + def __call__(self, graph: AssetGraph): if self.i > self.starting_decay_epoch: - for s, d, resources in edges.data(): + for s, d, resources in graph.edges.data(): if self.init_latency[(s, d)] is None: self.init_latency[(s, d)] = resources["latency"] self.values[(s, d)] = exponential_decay( diff --git a/examples/off_the_shelf/application.py b/examples/off_the_shelf/application.py new file mode 100644 index 0000000..9f75afb --- /dev/null +++ b/examples/off_the_shelf/application.py @@ -0,0 +1,44 @@ +"""Application built entirely from off-the-shelf ECLYPSE components.""" + +from __future__ import annotations + +from eclypse import policies +from eclypse.builders.application import get_sock_shop + + +def get_application(seed: int = 7): + """Create a Sock Shop application using built-in policies only.""" + return get_sock_shop( + application_id="SockShopBuiltins", + include_default_assets=True, + seed=seed, + update_policies=[ + policies.every( + 2, + policies.distribution.uniform( + node_assets=["cpu", "ram"], + edge_assets=["latency", "bandwidth"], + node_distribution=(1.02, 1.18), + edge_distribution=(0.98, 1.08), + ), + start=2, + ), + policies.after( + 6, + policies.degrade.reduce( + factor=0.8, + epochs=14, + node_assets=["cpu", "ram"], + edge_assets=["bandwidth"], + ), + ), + policies.after( + 6, + policies.degrade.increase( + factor=1.25, + epochs=14, + edge_assets=["latency"], + ), + ) + ], + ) diff --git a/examples/off_the_shelf/infrastructure.py b/examples/off_the_shelf/infrastructure.py new file mode 100644 index 0000000..7343abc --- /dev/null +++ b/examples/off_the_shelf/infrastructure.py @@ -0,0 +1,61 @@ +"""Infrastructure built entirely from off-the-shelf ECLYPSE components.""" + +from __future__ import annotations + +from eclypse import policies +from eclypse.builders.infrastructure import hierarchical + + +def get_infrastructure(seed: int = 7): + """Create a generated infrastructure with built-in update policies.""" + return hierarchical( + n=64, + infrastructure_id="BuiltinsInfrastructure", + symmetric=True, + update_policies=[ + policies.failure.availability_flap( + down_probability=0.04, + up_probability=0.15, + ), + policies.distribution.uniform( + node_asset_distributions={ + "cpu": (0.85, 1.12), + "ram": (0.8, 1.15), + "storage": (0.92, 1.08), + }, + edge_asset_distributions={ + "latency": (0.95, 1.2), + "bandwidth": (0.82, 1.08), + }, + ), + policies.every( + 2, + policies.failure.latency_spike( + probability=0.35, + min_increase=122.0, + max_increase=126.0, + ), + start=2, + ), + policies.after( + 5, + policies.degrade.reduce( + factor=0.82, + epochs=12, + node_assets=["cpu", "ram", "storage"], + edge_assets=["bandwidth"], + ), + ), + policies.after( + 5, + policies.degrade.increase( + factor=1.22, + epochs=12, + edge_assets=["latency"], + ), + ), + ], + include_default_assets=True, + resource_init="max", + seed=seed, + ) diff --git a/examples/off_the_shelf/main.py b/examples/off_the_shelf/main.py new file mode 100644 index 0000000..074e9d7 --- /dev/null +++ b/examples/off_the_shelf/main.py @@ -0,0 +1,35 @@ +"""Local simulation showcasing off-the-shelf ECLYPSE building blocks.""" + +from __future__ import annotations + +from application import get_application +from infrastructure import get_infrastructure + +from eclypse.placement.strategies import BestFitStrategy +from eclypse.simulation import ( + Simulation, + SimulationConfig, +) +from eclypse.utils.defaults import get_default_sim_path + +if __name__ == "__main__": + + SEED = 42 + MAX_STEPS = 50 + simulation = Simulation( + get_infrastructure(seed=SEED), + simulation_config=SimulationConfig( + seed=SEED, + max_steps=MAX_STEPS, + step_every_ms="auto", + include_default_metrics=True, + log_to_file=True, + path=get_default_sim_path() / "OffTheShelf", + log_level="TRACE", + ), + ) + + simulation.register(get_application(seed=SEED), BestFitStrategy()) + simulation.start() + simulation.wait() + print(simulation.report.application()) diff --git a/examples/sock_shop/mpi/main.py b/examples/sock_shop/mpi/main.py index e995f9f..1ad00e0 100644 --- a/examples/sock_shop/mpi/main.py +++ b/examples/sock_shop/mpi/main.py @@ -1,7 +1,4 @@ -from update_policy import ( - edge_random_update, - node_random_update, -) +from update_policy import random_update from eclypse.builders.application import get_sock_shop from eclypse.builders.infrastructure import hierarchical @@ -15,8 +12,7 @@ infrastructure = hierarchical( n=30, node_partitioning=[0.6, 0.1, 0.15, 0.15], - node_update_policy=node_random_update, - link_update_policy=edge_random_update, + update_policies=random_update, include_default_assets=True, symmetric=True, seed=seed, diff --git a/examples/sock_shop/mpi/update_policy.py b/examples/sock_shop/mpi/update_policy.py index 3217ba9..1e3e482 100644 --- a/examples/sock_shop/mpi/update_policy.py +++ b/examples/sock_shop/mpi/update_policy.py @@ -1,14 +1,10 @@ import random as rnd -from networkx.classes.reportviews import ( - EdgeView, - NodeView, -) +from eclypse.graph import AssetGraph -# update edges -def node_random_update(nodes: NodeView): - for _, resources in nodes.data(): +def random_update(graph: AssetGraph): + for _, resources in graph.nodes.data(): if rnd.random() < 0.02: resources["availability"] = 0 elif rnd.random() < 0.5 and resources["availability"] == 0: @@ -25,9 +21,7 @@ def node_random_update(nodes: NodeView): 1, max(0, resources["availability"] * rnd.uniform(0.995, 1.005)) ) - -def edge_random_update(edges: EdgeView): - for _, _, resources in edges.data(): + for _, _, resources in graph.edges.data(): # Randomly update resources with different ranges resources["latency"] = round( max(0, resources["latency"] * rnd.uniform(0.9, 1.1)) diff --git a/examples/sock_shop/rest/main.py b/examples/sock_shop/rest/main.py index 7ecd61a..7f7d691 100644 --- a/examples/sock_shop/rest/main.py +++ b/examples/sock_shop/rest/main.py @@ -1,7 +1,4 @@ -from update_policy import ( - edge_random_update, - node_random_update, -) +from update_policy import random_update from eclypse.builders.application import get_sock_shop from eclypse.builders.infrastructure import hierarchical @@ -16,8 +13,7 @@ infrastructure = hierarchical( n=30, node_partitioning=[0.6, 0.2, 0.1, 0.1], - node_update_policy=node_random_update, - link_update_policy=edge_random_update, + update_policies=random_update, include_default_assets=True, symmetric=True, seed=seed, diff --git a/examples/sock_shop/rest/update_policy.py b/examples/sock_shop/rest/update_policy.py index 3217ba9..1e3e482 100644 --- a/examples/sock_shop/rest/update_policy.py +++ b/examples/sock_shop/rest/update_policy.py @@ -1,14 +1,10 @@ import random as rnd -from networkx.classes.reportviews import ( - EdgeView, - NodeView, -) +from eclypse.graph import AssetGraph -# update edges -def node_random_update(nodes: NodeView): - for _, resources in nodes.data(): +def random_update(graph: AssetGraph): + for _, resources in graph.nodes.data(): if rnd.random() < 0.02: resources["availability"] = 0 elif rnd.random() < 0.5 and resources["availability"] == 0: @@ -25,9 +21,7 @@ def node_random_update(nodes: NodeView): 1, max(0, resources["availability"] * rnd.uniform(0.995, 1.005)) ) - -def edge_random_update(edges: EdgeView): - for _, _, resources in edges.data(): + for _, _, resources in graph.edges.data(): # Randomly update resources with different ranges resources["latency"] = round( max(0, resources["latency"] * rnd.uniform(0.9, 1.1)) diff --git a/examples/user_distribution/infrastructure.py b/examples/user_distribution/infrastructure.py index 263b107..2e78ca2 100644 --- a/examples/user_distribution/infrastructure.py +++ b/examples/user_distribution/infrastructure.py @@ -1,7 +1,7 @@ import networkx as nx from metric import user_count_asset from update_policy import ( - EdgeUpdatePolicy, + LatencyUpdatePolicy, UserDistributionPolicy, kill_policy, ) @@ -11,16 +11,16 @@ def get_infrastructure(seed: int): kill_probability = 0.1 - node_policy = kill_policy(kill_probability=kill_probability) - edge_policy = EdgeUpdatePolicy(kill_probability=kill_probability) - i = hierarchical( node_assets={"user_count": user_count_asset()}, infrastructure_id="hierarchical", n=187, - node_update_policy=[node_policy, UserDistributionPolicy()], + update_policies=[ + kill_policy(kill_probability=kill_probability), + LatencyUpdatePolicy(kill_probability=kill_probability), + UserDistributionPolicy(), + ], include_default_assets=True, - link_update_policy=edge_policy, symmetric=True, seed=seed, ) diff --git a/examples/user_distribution/update_policy.py b/examples/user_distribution/update_policy.py index 2cd47fe..39a4200 100644 --- a/examples/user_distribution/update_policy.py +++ b/examples/user_distribution/update_policy.py @@ -2,38 +2,36 @@ from pathlib import Path import pandas as pd -from networkx.classes.reportviews import ( - EdgeView, - NodeView, -) + +from eclypse.graph import AssetGraph def kill_policy(kill_probability: float): revive_probability = kill_probability / 2 - def node_update_wrapper(nodes: NodeView): - for _, resources in nodes.data(): + def update_wrapper(graph: AssetGraph): + for _, resources in graph.nodes.data(): if rnd.random() < kill_probability: resources["availability"] = 0 elif rnd.random() < revive_probability: resources["availability"] = 0.99 - return node_update_wrapper + return update_wrapper -class EdgeUpdatePolicy: +class LatencyUpdatePolicy: def __init__(self, kill_probability: float): self.initial_latencies = None self.kill_probability = kill_probability self.revive_probability = kill_probability / 2 - def __call__(self, edges: EdgeView): + def __call__(self, graph: AssetGraph): if self.initial_latencies is None: self.initial_latencies = { - (u, v): data["latency"] for u, v, data in edges.data() + (u, v): data["latency"] for u, v, data in graph.edges.data() } - for u, v, data in edges.data(): + for u, v, data in graph.edges.data(): if rnd.random() < self.kill_probability: data["latency"] += rnd.randint(1, 5) elif rnd.random() < self.revive_probability: @@ -48,7 +46,7 @@ def __init__(self): self.step = self.df["time"].min() self.factor = 1 - def __call__(self, nodes: NodeView): + def __call__(self, graph: AssetGraph): if self.step == 1000 or self.step == 3000: self.factor += 2 @@ -58,6 +56,6 @@ def __call__(self, nodes: NodeView): current_data = self.df[self.df["time"] == self.step] for _, row in current_data.iterrows(): user_count = int(row["user_count"]) * self.factor - nodes[row["node_id"]]["user_count"] = user_count + graph.nodes[row["node_id"]]["user_count"] = user_count self.step += 1 diff --git a/tests/integration/test_simulation_runtime.py b/tests/integration/test_simulation_runtime.py index b9b677d..379878d 100644 --- a/tests/integration/test_simulation_runtime.py +++ b/tests/integration/test_simulation_runtime.py @@ -1,10 +1,16 @@ from __future__ import annotations import csv +import time from pathlib import Path import pytest +from eclypse import policies +from eclypse.placement.strategies import ( + BestFitStrategy, + StaticStrategy, +) from eclypse.report.report import Report from eclypse.simulation._simulator.local import SimulationState from eclypse.simulation.config import SimulationConfig @@ -15,6 +21,15 @@ ) +def _wait_until(predicate, timeout: float = 2.0): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + assert predicate() + + @pytest.mark.integration def test_manual_simulation_runtime_generates_reports_and_config( tmp_path: Path, @@ -122,3 +137,242 @@ def wrapped_runtime_metric(*_args): assert simulation_csv.exists() assert any(row["event_id"] == "start" for row in rows) assert any(row["callback_id"] == "wrapped_runtime_metric" for row in rows) + + +@pytest.mark.integration +def test_manual_simulation_runtime_applies_replay_policies_across_steps( + tmp_path: Path, + sample_infrastructure, + sample_application, + static_strategy, +): + sample_application.update_policies = [ + policies.replay.replay_nodes( + [ + {"time": 0, "node_id": "gateway", "cpu": 2}, + {"time": 1, "node_id": "gateway", "cpu": 3}, + ], + value_columns=["cpu"], + ) + ] + sample_infrastructure.update_policies = [ + policies.replay.replay_edges( + [ + {"time": 0, "source": "edge-a", "target": "edge-b", "bandwidth": 8}, + {"time": 1, "source": "edge-a", "target": "edge-b", "bandwidth": 6}, + ], + value_columns=["bandwidth"], + ) + ] + + config = SimulationConfig( + path=tmp_path / "replay-simulation", + report_backend="pandas", + report_format="csv", + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, static_strategy) + + simulation.start() + simulation.step() + simulation.step() + simulation.stop() + + assert simulation.status is SimulationState.IDLE + assert sample_application.nodes["gateway"]["cpu"] == 3 + assert sample_infrastructure.edges["edge-a", "edge-b"]["bandwidth"] == 6 + + +@pytest.mark.integration +def test_manual_simulation_runtime_resets_and_then_fails_placement_after_degradation( + tmp_path: Path, + sample_infrastructure, + sample_application, +): + sample_infrastructure.update_policies = [ + policies.degrade.reduce( + target=0, + epochs=1, + node_assets="cpu", + ) + ] + + config = SimulationConfig( + path=tmp_path / "placement-reset-simulation", + report_backend="pandas", + report_format="csv", + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, BestFitStrategy()) + + simulation.start() + simulation.step() + + placement = simulation.simulator.placements[sample_application.id] + _wait_until( + lambda: ( + bool(placement.mapping) + and sample_infrastructure.nodes["edge-a"]["cpu"] == 0 + and sample_infrastructure.nodes["edge-b"]["cpu"] == 0 + ) + ) + + simulation.step() + _wait_until(lambda: placement.mapping == {}) + + simulation.step() + _wait_until(lambda: placement.mapping == {}) + simulation.stop() + assert simulation.status is SimulationState.IDLE + + +@pytest.mark.integration +def test_manual_simulation_runtime_uses_global_strategy_when_application_has_none( + tmp_path: Path, + sample_infrastructure, + sample_application, +): + sample_infrastructure.strategy = StaticStrategy( + {"gateway": "edge-a", "worker": "edge-b"} + ) + + config = SimulationConfig( + path=tmp_path / "global-strategy-simulation", + report_backend="pandas", + report_format="csv", + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application) + + simulation.start() + simulation.step() + + placement = simulation.simulator.placements[sample_application.id] + _wait_until(lambda: placement.mapping == {"gateway": "edge-a", "worker": "edge-b"}) + + simulation.stop() + assert simulation.status is SimulationState.IDLE + + +@pytest.mark.integration +def test_manual_simulation_runtime_handles_partial_placement( + tmp_path: Path, + sample_infrastructure, + sample_application, +): + sample_application.nodes["worker"]["cpu"] = 100 + + config = SimulationConfig( + path=tmp_path / "partial-placement-simulation", + report_backend="pandas", + report_format="csv", + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, BestFitStrategy()) + + simulation.start() + simulation.step() + + placement = simulation.simulator.placements[sample_application.id] + _wait_until(lambda: placement.mapping == {}) + + simulation.stop() + assert simulation.status is SimulationState.IDLE + + +@pytest.mark.integration +def test_manual_simulation_runtime_handles_no_placement( + tmp_path: Path, + sample_infrastructure, + sample_application, +): + sample_application.nodes["gateway"]["cpu"] = 100 + sample_application.nodes["worker"]["cpu"] = 100 + + config = SimulationConfig( + path=tmp_path / "no-placement-simulation", + report_backend="pandas", + report_format="csv", + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, BestFitStrategy()) + + simulation.start() + simulation.step() + + placement = simulation.simulator.placements[sample_application.id] + _wait_until(lambda: placement.mapping == {}) + + simulation.stop() + assert simulation.status is SimulationState.IDLE + + +@pytest.mark.integration +def test_manual_simulation_runtime_resets_when_service_path_disappears( + tmp_path: Path, + sample_infrastructure, + sample_application, + static_strategy, +): + def remove_forward_path(graph): + if graph.has_edge("edge-a", "edge-b"): + graph.remove_edge("edge-a", "edge-b") + + sample_infrastructure.update_policies = [remove_forward_path] + + config = SimulationConfig( + path=tmp_path / "path-loss-simulation", + report_backend="pandas", + report_format="csv", + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, static_strategy) + + simulation.start() + simulation.step() + + placement = simulation.simulator.placements[sample_application.id] + _wait_until( + lambda: ( + placement.mapping == {"gateway": "edge-a", "worker": "edge-b"} + and not sample_infrastructure.has_edge("edge-a", "edge-b") + ) + ) + + simulation.step() + _wait_until(lambda: placement.mapping == {}) + + simulation.stop() + assert simulation.status is SimulationState.IDLE + + +@pytest.mark.integration +def test_auto_simulation_runtime_stops_after_event_failure( + tmp_path: Path, + sample_infrastructure, + sample_application, + static_strategy, +): + @event( + event_type="simulation", + activates_on=["start"], + verbose=True, + ) + def failing_runtime_event(*_args): + raise RuntimeError("boom") + + config = SimulationConfig( + path=tmp_path / "failing-event-simulation", + report_backend="pandas", + report_format="csv", + events=[failing_runtime_event], + step_every_ms="auto", + max_steps=5, + ) + simulation = Simulation(sample_infrastructure, config) + simulation.register(sample_application, static_strategy) + + simulation.start() + simulation.wait(timeout=10) + + assert simulation.status is SimulationState.IDLE diff --git a/tests/unit/builders/application/test_sock_shop.py b/tests/unit/builders/application/test_sock_shop.py index 52b6d83..d72a902 100644 --- a/tests/unit/builders/application/test_sock_shop.py +++ b/tests/unit/builders/application/test_sock_shop.py @@ -2,6 +2,7 @@ import pytest +from eclypse.builders._helpers import prune_assets from eclypse.builders.application import get_sock_shop from eclypse.remote.service.service import Service @@ -34,3 +35,9 @@ def test_sock_shop_builder_configures_supported_interfaces_and_flows(): with pytest.raises(ValueError, match="Unknown communication interface"): get_sock_shop(communication_interface="grpc") # type: ignore[arg-type] + + +def test_prune_assets_discards_unknown_builder_requirements(sample_infrastructure): + assert prune_assets(sample_infrastructure.node_assets, cpu=1, missing=2) == { + "cpu": 1 + } diff --git a/tests/unit/graph/test_asset_graph.py b/tests/unit/graph/test_asset_graph.py index 75e3396..0525f1d 100644 --- a/tests/unit/graph/test_asset_graph.py +++ b/tests/unit/graph/test_asset_graph.py @@ -30,10 +30,12 @@ def test_asset_graph_validates_nodes_edges_and_dynamic_flags(): def test_asset_graph_evolve_runs_registered_policies(): graph = AssetGraph( "dynamic", - node_update_policy=lambda nodes: nodes["a"].update(cpu=nodes["a"]["cpu"] + 1), - edge_update_policy=lambda edges: edges["a", "b"].update( - bandwidth=edges["a", "b"]["bandwidth"] + 1 - ), + update_policies=[ + lambda graph: graph.nodes["a"].update(cpu=graph.nodes["a"]["cpu"] + 1), + lambda graph: graph.edges["a", "b"].update( + bandwidth=graph.edges["a", "b"]["bandwidth"] + 1 + ), + ], node_assets={"cpu": Additive(0, 10)}, edge_assets={"bandwidth": Additive(0, 10)}, ) diff --git a/tests/unit/graph/test_infrastructure.py b/tests/unit/graph/test_infrastructure.py index c356211..9b6068e 100644 --- a/tests/unit/graph/test_infrastructure.py +++ b/tests/unit/graph/test_infrastructure.py @@ -32,6 +32,18 @@ def test_infrastructure_path_resources_and_cache_behaviour(sample_infrastructure assert sample_infrastructure.path("edge-a", "edge-b") is None +def test_infrastructure_evolve_invalidates_cached_path_resources(sample_infrastructure): + assert sample_infrastructure.path_resources("edge-a", "edge-b")["bandwidth"] == 10 + + sample_infrastructure.update_policies = [ + lambda graph: graph.edges["edge-a", "edge-b"].update(bandwidth=5) + ] + + sample_infrastructure.evolve() + + assert sample_infrastructure.path_resources("edge-a", "edge-b")["bandwidth"] == 5 + + def test_infrastructure_requires_path_aggregators_for_custom_edge_assets(): with pytest.raises(ValueError, match='path asset aggregator for "bandwidth"'): Infrastructure( diff --git a/tests/unit/policies/_helpers.py b/tests/unit/policies/_helpers.py new file mode 100644 index 0000000..9a5a062 --- /dev/null +++ b/tests/unit/policies/_helpers.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from eclypse.graph.asset_graph import AssetGraph +from eclypse.graph.assets import Additive + + +class FakeDataFrame: + def __init__(self, records): + self._records = records + + def to_dict(self, orient: str): + assert orient == "records" + return self._records + + +class IterRowsFrame: + def __init__(self, records): + self._records = records + + def iterrows(self): + yield from enumerate(self._records) + + +def build_graph(seed: int = 7) -> AssetGraph: + graph = AssetGraph( + "dynamic", + seed=seed, + node_assets={ + "cpu": Additive(0, 1000), + "ram": Additive(0, 1000), + "availability": Additive(0, 1), + }, + edge_assets={ + "latency": Additive(0, 10_000), + "bandwidth": Additive(0, 10_000), + }, + ) + graph.add_node("a", cpu=80, ram=32, availability=1.0) + graph.add_node("b", cpu=50, ram=16, availability=1.0) + graph.add_edge("a", "b", latency=10, bandwidth=100) + return graph diff --git a/tests/unit/policies/common/test_common.py b/tests/unit/policies/common/test_common.py new file mode 100644 index 0000000..db09b8d --- /dev/null +++ b/tests/unit/policies/common/test_common.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import pytest + +from eclypse.graph.asset_graph import _normalize_update_policies +from eclypse.policies._filters import ( + clamp, + coerce_numeric_like, + ensure_numeric_value, + iter_selected_edges, + iter_selected_keys, + iter_selected_nodes, +) +from tests.unit.policies._helpers import build_graph + + +def test_normalize_update_policies_and_filter_helpers_cover_edge_cases(): + def policy(_graph): + return None + + assert _normalize_update_policies(None) == [] + assert _normalize_update_policies(policy) == [policy] + assert _normalize_update_policies([policy]) == [policy] + + graph = build_graph() + + assert iter_selected_nodes( + graph, node_filter=lambda node_id, _: node_id == "a" + ) == [("a", graph.nodes["a"])] + assert iter_selected_edges( + graph, + edge_filter=lambda source, target, _: (source, target) == ("a", "b"), + ) == [("a", "b", graph.edges["a", "b"])] + assert iter_selected_keys(graph.nodes["a"], ["cpu", "missing"]) == ["cpu"] + + with pytest.raises(TypeError): + ensure_numeric_value("availability", True) + + with pytest.raises(TypeError): + ensure_numeric_value("cpu", "busy") + + assert clamp(5, upper=3) == 3 + assert coerce_numeric_like(True, 1.5) == 1.5 diff --git a/tests/unit/policies/degrade/test_degrade.py b/tests/unit/policies/degrade/test_degrade.py new file mode 100644 index 0000000..8b895c6 --- /dev/null +++ b/tests/unit/policies/degrade/test_degrade.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import pytest + +from eclypse import policies +from tests.unit.policies._helpers import build_graph + + +def test_degrade_policies_stop_at_the_requested_epoch(): + graph = build_graph() + + reduce = policies.degrade.reduce( + factor=0.25, + epochs=2, + node_assets="cpu", + edge_assets="bandwidth", + ) + increase = policies.degrade.increase( + target=40, + epochs=2, + edge_assets="latency", + ) + + reduce(graph) + increase(graph) + assert graph.nodes["a"]["cpu"] == 40 + assert graph.edges["a", "b"]["bandwidth"] == 50 + assert graph.edges["a", "b"]["latency"] == 20 + + reduce(graph) + increase(graph) + assert graph.nodes["a"]["cpu"] == 20 + assert graph.edges["a", "b"]["bandwidth"] == 25 + assert graph.edges["a", "b"]["latency"] == 40 + + +def test_degrade_validation_and_factor_mode(): + with pytest.raises(ValueError): + policies.degrade.reduce(factor=0.5, epochs=0) + + with pytest.raises(ValueError): + policies.degrade.increase(epochs=2) + + with pytest.raises(ValueError): + policies.degrade.increase(factor=1.1, target=20, epochs=2) + + with pytest.raises(ValueError): + policies.degrade.increase(factor=0.5, epochs=2) + + with pytest.raises(ValueError): + policies.degrade.increase(target=-1, epochs=2) + + with pytest.raises(ValueError): + policies.degrade.reduce(factor=0.5, epochs=2) + + with pytest.raises(ValueError): + policies.degrade.increase(factor=2.0, epochs=2) + + graph = build_graph() + policy = policies.degrade.increase( + factor=2.25, + epochs=2, + edge_assets="latency", + ) + policy(graph) + policy(graph) + + assert graph.edges["a", "b"]["latency"] == 22 + + +def test_overrides_support_per_asset_overrides(): + graph = build_graph() + + policy = policies.degrade.reduce( + factor=0.5, + epochs=2, + node_assets="cpu", + edge_asset_overrides={ + "bandwidth": { + "factor": 0.25, + "epochs": 2, + } + }, + ) + + policy(graph) + policy(graph) + + assert graph.nodes["a"]["cpu"] == 40 + assert graph.edges["a", "b"]["bandwidth"] == 25 + + +def test_reduce_and_increase_target_direction_validation(): + graph = build_graph() + + with pytest.raises(ValueError): + policies.degrade.increase(target=5, epochs=2, edge_assets="latency")(graph) + + with pytest.raises(ValueError): + policies.degrade.reduce(target=20, epochs=2, edge_assets="latency")(graph) + + +def test_increase_and_reduce_can_be_composed_explicitly(): + graph = build_graph() + + reduce_policy = policies.degrade.reduce( + factor=0.5, + epochs=2, + node_assets="cpu", + edge_asset_overrides={ + "bandwidth": { + "factor": 0.25, + "epochs": 2, + } + }, + ) + increase_policy = policies.degrade.increase( + factor=2.0, + epochs=2, + edge_assets="latency", + node_asset_overrides={ + "ram": { + "target": 64, + "epochs": 2, + } + }, + ) + + for _ in range(2): + reduce_policy(graph) + increase_policy(graph) + + assert graph.nodes["a"]["cpu"] == 40 + assert graph.nodes["a"]["ram"] == 64 + assert graph.edges["a", "b"]["bandwidth"] == 25 + assert graph.edges["a", "b"]["latency"] == 20 diff --git a/tests/unit/policies/distribution/test_distribution.py b/tests/unit/policies/distribution/test_distribution.py new file mode 100644 index 0000000..bcc26cc --- /dev/null +++ b/tests/unit/policies/distribution/test_distribution.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +import math + +import pytest + +from eclypse import policies +from tests.unit.policies._helpers import build_graph + + +def test_uniform_distribution_policy_changes_only_selected_resources(): + graph = build_graph() + + policies.distribution.uniform( + node_assets="cpu", + edge_assets="bandwidth", + node_distribution=(1.5, 1.5), + edge_distribution=(0.5, 0.5), + )(graph) + + assert graph.nodes["a"]["cpu"] == 120 + assert graph.nodes["a"]["ram"] == 32 + assert graph.edges["a", "b"]["bandwidth"] == 50 + assert graph.edges["a", "b"]["latency"] == 10 + + +def test_uniform_distribution_validation_and_derived_asset_selection(): + with pytest.raises(ValueError): + policies.distribution.uniform(node_distribution=(2.0, 1.0)) + + with pytest.raises(ValueError): + policies.distribution.uniform(edge_distribution=(2.0, 1.0)) + + with pytest.raises(ValueError): + policies.distribution.uniform() + + graph = build_graph() + policies.distribution.uniform( + node_asset_distributions={"cpu": (0.5, 0.5)}, + edge_asset_distributions={"latency": (2.0, 2.0)}, + )(graph) + + assert graph.nodes["a"]["cpu"] == 40 + assert graph.nodes["a"]["ram"] == 32 + assert graph.edges["a", "b"]["latency"] == 20 + assert graph.edges["a", "b"]["bandwidth"] == 100 + + +def test_uniform_distribution_uses_union_of_assets_and_per_asset_overrides(): + graph = build_graph() + graph.nodes["a"]["storage"] = 40 + + policies.distribution.uniform( + node_assets=["cpu", "ram"], + node_distribution=(2.0, 2.0), + node_asset_distributions={"cpu": (0.5, 0.5), "storage": (3.0, 3.0)}, + )(graph) + + assert graph.nodes["a"]["cpu"] == 40 + assert graph.nodes["a"]["ram"] == 64 + assert graph.nodes["a"]["storage"] == 120 + + +def test_uniform_distribution_uses_graph_rng_reproducibly(): + first_graph = build_graph() + second_graph = build_graph() + + first_policy = policies.distribution.uniform( + node_assets="cpu", + edge_assets="latency", + node_distribution=(0.8, 1.2), + edge_distribution=(0.8, 1.2), + ) + second_policy = policies.distribution.uniform( + node_assets="cpu", + edge_assets="latency", + node_distribution=(0.8, 1.2), + edge_distribution=(0.8, 1.2), + ) + + first_policy(first_graph) + second_policy(second_graph) + + assert first_graph.nodes["a"]["cpu"] == second_graph.nodes["a"]["cpu"] + assert ( + first_graph.edges["a", "b"]["latency"] + == second_graph.edges["a", "b"]["latency"] + ) + + +def test_normal_distribution_policy_applies_selected_gaussian_multipliers(): + graph = build_graph() + + policies.distribution.normal( + node_assets="cpu", + edge_assets="bandwidth", + node_distribution=(1.5, 0.0), + edge_distribution=(0.5, 0.0), + )(graph) + + assert graph.nodes["a"]["cpu"] == 120 + assert graph.nodes["a"]["ram"] == 32 + assert graph.edges["a", "b"]["bandwidth"] == 50 + assert graph.edges["a", "b"]["latency"] == 10 + + +def test_normal_distribution_policy_validates_std_and_supports_per_asset_overrides(): + with pytest.raises(ValueError): + policies.distribution.normal(node_distribution=(1.0, -0.1)) + + graph = build_graph() + policies.distribution.normal( + node_asset_distributions={"cpu": (0.5, 0.0)}, + edge_asset_distributions={"latency": (2.0, 0.0)}, + )(graph) + + assert graph.nodes["a"]["cpu"] == 40 + assert graph.nodes["a"]["ram"] == 32 + assert graph.edges["a", "b"]["latency"] == 20 + assert graph.edges["a", "b"]["bandwidth"] == 100 + + +def test_normal_distribution_uses_graph_rng_reproducibly(): + first_graph = build_graph() + second_graph = build_graph() + + first_policy = policies.distribution.normal( + node_assets="cpu", + edge_assets="latency", + node_distribution=(1.0, 0.1), + edge_distribution=(1.0, 0.1), + ) + second_policy = policies.distribution.normal( + node_assets="cpu", + edge_assets="latency", + node_distribution=(1.0, 0.1), + edge_distribution=(1.0, 0.1), + ) + + first_policy(first_graph) + second_policy(second_graph) + + assert first_graph.nodes["a"]["cpu"] == second_graph.nodes["a"]["cpu"] + assert ( + first_graph.edges["a", "b"]["latency"] + == second_graph.edges["a", "b"]["latency"] + ) + + +def test_lognormal_distribution_policy_applies_selected_multipliers(): + graph = build_graph() + + policies.distribution.lognormal( + node_assets="cpu", + edge_assets="bandwidth", + node_distribution=(math.log(1.5), 0.0), + edge_distribution=(math.log(0.5), 0.0), + )(graph) + + assert graph.nodes["a"]["cpu"] == 120 + assert graph.edges["a", "b"]["bandwidth"] == 50 + + +def test_lognormal_distribution_validates_sigma(): + with pytest.raises(ValueError): + policies.distribution.lognormal(node_distribution=(0.0, -0.1)) + + +def test_beta_distribution_uses_graph_rng_reproducibly(): + first_graph = build_graph() + second_graph = build_graph() + + first_policy = policies.distribution.beta( + node_assets="cpu", + edge_assets="latency", + node_distribution=(2.0, 3.0), + edge_distribution=(2.0, 3.0), + ) + second_policy = policies.distribution.beta( + node_assets="cpu", + edge_assets="latency", + node_distribution=(2.0, 3.0), + edge_distribution=(2.0, 3.0), + ) + + first_policy(first_graph) + second_policy(second_graph) + + assert first_graph.nodes["a"]["cpu"] == second_graph.nodes["a"]["cpu"] + assert ( + first_graph.edges["a", "b"]["latency"] + == second_graph.edges["a", "b"]["latency"] + ) + + +def test_beta_distribution_validates_parameters(): + with pytest.raises(ValueError): + policies.distribution.beta(node_distribution=(0.0, 1.0)) + + +def test_gamma_distribution_uses_graph_rng_reproducibly(): + first_graph = build_graph() + second_graph = build_graph() + + first_policy = policies.distribution.gamma( + node_assets="cpu", + edge_assets="latency", + node_distribution=(2.0, 0.5), + edge_distribution=(2.0, 0.5), + ) + second_policy = policies.distribution.gamma( + node_assets="cpu", + edge_assets="latency", + node_distribution=(2.0, 0.5), + edge_distribution=(2.0, 0.5), + ) + + first_policy(first_graph) + second_policy(second_graph) + + assert first_graph.nodes["a"]["cpu"] == second_graph.nodes["a"]["cpu"] + assert ( + first_graph.edges["a", "b"]["latency"] + == second_graph.edges["a", "b"]["latency"] + ) + + +def test_gamma_distribution_validates_parameters(): + with pytest.raises(ValueError): + policies.distribution.gamma(node_distribution=(-1.0, 1.0)) + + +def test_triangular_distribution_applies_selected_multipliers(): + graph = build_graph() + + policies.distribution.triangular( + node_assets="cpu", + edge_assets="bandwidth", + node_distribution=(1.5, 1.5, 1.5), + edge_distribution=(0.5, 0.5, 0.5), + )(graph) + + assert graph.nodes["a"]["cpu"] == 120 + assert graph.edges["a", "b"]["bandwidth"] == 50 + + +def test_triangular_distribution_validates_shape(): + with pytest.raises(ValueError): + policies.distribution.triangular(node_distribution=(2.0, 1.0, 1.5)) + + with pytest.raises(ValueError): + policies.distribution.triangular(node_distribution=(1.0, 2.0, 3.0)) + + +def test_truncated_normal_distribution_applies_selected_multipliers(): + graph = build_graph() + + policies.distribution.truncated_normal( + node_assets="cpu", + edge_assets="bandwidth", + node_distribution=(1.5, 0.0), + edge_distribution=(0.5, 0.0), + lower=0.0, + upper=2.0, + )(graph) + + assert graph.nodes["a"]["cpu"] == 120 + assert graph.edges["a", "b"]["bandwidth"] == 50 + + +def test_truncated_normal_distribution_validates_bounds(): + with pytest.raises(ValueError): + policies.distribution.truncated_normal(node_distribution=(1.0, -0.1)) + + with pytest.raises(ValueError): + policies.distribution.truncated_normal(lower=2.0, upper=1.0) + + with pytest.raises(ValueError): + policies.distribution.truncated_normal(max_attempts=0) + + +def test_categorical_distribution_applies_selected_multipliers(): + graph = build_graph() + + policies.distribution.categorical( + node_assets="cpu", + edge_assets="bandwidth", + node_distribution=[1.5], + edge_distribution=[0.5], + )(graph) + + assert graph.nodes["a"]["cpu"] == 120 + assert graph.edges["a", "b"]["bandwidth"] == 50 + + +def test_categorical_distribution_supports_weights_and_overrides(): + graph = build_graph() + + policies.distribution.categorical( + node_asset_distributions={"cpu": [0.5, 1.5]}, + edge_asset_distributions={"latency": [2.0]}, + node_asset_weights={"cpu": [1.0, 0.0]}, + edge_asset_weights={"latency": [1.0]}, + )(graph) + + assert graph.nodes["a"]["cpu"] == 40 + assert graph.nodes["a"]["ram"] == 32 + assert graph.edges["a", "b"]["latency"] == 20 + + +def test_categorical_distribution_validates_inputs(): + with pytest.raises(ValueError): + policies.distribution.categorical(node_distribution=[]) + + with pytest.raises(ValueError): + policies.distribution.categorical(node_weights=[1.0, 2.0]) + + with pytest.raises(ValueError): + policies.distribution.categorical(node_distribution=[1.0], node_weights=[-1.0]) + + with pytest.raises(ValueError): + policies.distribution.categorical(node_distribution=[1.0], node_weights=[0.0]) + + with pytest.raises(ValueError): + policies.distribution.categorical(node_asset_weights={"cpu": [1.0]}) diff --git a/tests/unit/policies/failure/test_failure.py b/tests/unit/policies/failure/test_failure.py new file mode 100644 index 0000000..71a03eb --- /dev/null +++ b/tests/unit/policies/failure/test_failure.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import pytest + +from eclypse import policies +from tests.unit.policies._helpers import build_graph + + +def test_failure_policies_target_selected_nodes_and_edges(): + graph = build_graph() + + policies.failure.kill_nodes(1.0, node_ids=["a"])(graph) + assert graph.nodes["a"]["availability"] == 0.0 + assert graph.nodes["b"]["availability"] == 1.0 + + policies.failure.revive_nodes(1.0, node_ids=["a"])(graph) + assert graph.nodes["a"]["availability"] == 0.99 + + policies.failure.availability_flap(1.0, node_ids=["b"])(graph) + assert graph.nodes["b"]["availability"] == 0.0 + + policies.failure.latency_spike( + 1.0, min_increase=5.0, max_increase=5.0, edge_ids=[("a", "b")] + )(graph) + assert graph.edges["a", "b"]["latency"] == 15 + + +def test_failure_policy_validation_and_alternative_branches(): + with pytest.raises(ValueError): + policies.failure.kill_nodes(1.5) + + with pytest.raises(ValueError): + policies.failure.availability_flap(-0.1) + + with pytest.raises(ValueError): + policies.failure.latency_spike(1.0, factor=-1) + + with pytest.raises(ValueError): + policies.failure.latency_spike(1.0, min_increase=-1) + + with pytest.raises(ValueError): + policies.failure.latency_spike(1.0, min_increase=2, max_increase=1) + + graph = build_graph() + graph.nodes["a"]["availability"] = 0.0 + + policies.failure.availability_flap( + 0.0, + up_probability=1.0, + up_availability=0.75, + node_ids=["a"], + unavailable_at_or_below=0.0, + )(graph) + assert graph.nodes["a"]["availability"] == 0.75 + + policies.failure.latency_spike(1.0, factor=2.0)(graph) + assert graph.edges["a", "b"]["latency"] == 20 diff --git a/tests/unit/policies/noise/test_noise.py b/tests/unit/policies/noise/test_noise.py new file mode 100644 index 0000000..dd9d8e5 --- /dev/null +++ b/tests/unit/policies/noise/test_noise.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import pytest + +from eclypse import policies +from tests.unit.policies._helpers import build_graph + + +def test_bounded_random_walk_stays_within_bounds(): + graph = build_graph() + + policy = policies.noise.bounded_random_walk( + node_steps={"cpu": 25}, + edge_steps={"latency": 5}, + node_bounds={"cpu": (0, 90)}, + edge_bounds={"latency": (0, 12)}, + ) + + for _ in range(20): + policy(graph) + assert 0 <= graph.nodes["a"]["cpu"] <= 90 + assert 0 <= graph.edges["a", "b"]["latency"] <= 12 + + +def test_bounded_random_walk_validation(): + with pytest.raises(ValueError): + policies.noise.bounded_random_walk() + + with pytest.raises(ValueError): + policies.noise.bounded_random_walk(node_steps={"cpu": -1}) + + with pytest.raises(ValueError): + policies.noise.bounded_random_walk(edge_steps={"latency": -1}) + + +def test_momentum_walk_stays_within_bounds(): + graph = build_graph() + + policy = policies.noise.momentum_walk( + node_steps={"cpu": 25}, + edge_steps={"latency": 5}, + node_bounds={"cpu": (0, 90)}, + edge_bounds={"latency": (0, 12)}, + momentum=0.8, + ) + + for _ in range(20): + policy(graph) + assert 0 <= graph.nodes["a"]["cpu"] <= 90 + assert 0 <= graph.edges["a", "b"]["latency"] <= 12 + + +def test_momentum_walk_uses_graph_rng_reproducibly(): + first_graph = build_graph() + second_graph = build_graph() + + first_policy = policies.noise.momentum_walk( + node_steps={"cpu": 10}, + edge_steps={"latency": 2}, + momentum=0.6, + ) + second_policy = policies.noise.momentum_walk( + node_steps={"cpu": 10}, + edge_steps={"latency": 2}, + momentum=0.6, + ) + + for _ in range(5): + first_policy(first_graph) + second_policy(second_graph) + + assert first_graph.nodes["a"]["cpu"] == second_graph.nodes["a"]["cpu"] + assert ( + first_graph.edges["a", "b"]["latency"] + == second_graph.edges["a", "b"]["latency"] + ) + + +def test_momentum_walk_validation(): + with pytest.raises(ValueError): + policies.noise.momentum_walk() + + with pytest.raises(ValueError): + policies.noise.momentum_walk(node_steps={"cpu": -1}) + + with pytest.raises(ValueError): + policies.noise.momentum_walk(momentum=1.5, node_steps={"cpu": 1}) + + +def test_impulse_applies_selected_shocks(): + graph = build_graph() + + policies.noise.impulse( + node_assets="cpu", + edge_assets="bandwidth", + probability=1.0, + node_factor_range=(1.5, 1.5), + edge_factor_range=(0.5, 0.5), + )(graph) + + assert graph.nodes["a"]["cpu"] == 120 + assert graph.nodes["a"]["ram"] == 32 + assert graph.edges["a", "b"]["bandwidth"] == 50 + assert graph.edges["a", "b"]["latency"] == 10 + + +def test_impulse_can_skip_updates(): + graph = build_graph() + + policies.noise.impulse( + node_assets="cpu", + edge_assets="bandwidth", + probability=0.0, + )(graph) + + assert graph.nodes["a"]["cpu"] == 80 + assert graph.edges["a", "b"]["bandwidth"] == 100 + + +def test_impulse_validation(): + with pytest.raises(ValueError): + policies.noise.impulse() + + with pytest.raises(ValueError): + policies.noise.impulse(node_assets="cpu", probability=-0.1) + + with pytest.raises(ValueError): + policies.noise.impulse(node_assets="cpu", node_factor_range=(-1.0, 1.0)) + + with pytest.raises(ValueError): + policies.noise.impulse(node_assets="cpu", node_factor_range=(2.0, 1.0)) diff --git a/tests/unit/policies/replay/test_replay.py b/tests/unit/policies/replay/test_replay.py new file mode 100644 index 0000000..d606c8d --- /dev/null +++ b/tests/unit/policies/replay/test_replay.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + cast, +) + +import pytest + +from eclypse import policies +from tests.unit.policies._helpers import build_graph + +if TYPE_CHECKING: + from eclypse.utils.types import ReplayTarget + + +class FakeDataFrame: + def __init__(self, records): + self._records = records + + def to_dict(self, orient: str): + assert orient == "records" + return self._records + + +def test_replay_policies_replay_node_and_edge_records(): + graph = build_graph() + + node_policy = policies.replay.replay_nodes( + [ + {"time": 0, "node_id": "a", "cpu": 4}, + {"time": 1, "node_id": "a", "cpu": 6}, + ], + value_columns=["cpu"], + ) + edge_policy = policies.replay.replay_edges( + [ + {"time": 0, "source": "a", "target": "b", "latency": 14}, + {"time": 1, "source": "a", "target": "b", "latency": 20}, + ], + value_columns=["latency"], + ) + + node_policy(graph) + edge_policy(graph) + assert graph.nodes["a"]["cpu"] == 4 + assert graph.edges["a", "b"]["latency"] == 14 + + node_policy(graph) + edge_policy(graph) + assert graph.nodes["a"]["cpu"] == 6 + assert graph.edges["a", "b"]["latency"] == 20 + + +def test_replay_convenience_builders_accept_records_and_dataframe_like(): + graph = build_graph() + + policies.replay.from_records( + [ + {"time": 0, "node_id": "a", "cpu": 7}, + {"time": 1, "node_id": "a", "cpu": 9}, + ], + target="nodes", + value_columns=["cpu"], + )(graph) + + assert graph.nodes["a"]["cpu"] == 7 + + policies.replay.from_dataframe( + FakeDataFrame( + [ + {"time": 0, "source": "a", "target": "b", "bandwidth": 80}, + {"time": 1, "source": "a", "target": "b", "bandwidth": 60}, + ] + ), + target="edges", + value_columns=["bandwidth"], + )(graph) + + assert graph.edges["a", "b"]["bandwidth"] == 80 + + +def test_replay_builders_cover_invalid_targets_and_parquet_loading( + monkeypatch, +): + invalid_target = cast("ReplayTarget", "services") + with pytest.raises(ValueError): + policies.replay.from_records([], target=invalid_target) + + policies.replay.from_dataframe( + FakeDataFrame([]), + target="nodes", + ) + + class FakePandas: + @staticmethod + def read_parquet(path): + assert path == "trace.parquet" + return FakeDataFrame( + [ + {"time": 0, "node_id": "a", "cpu": 3}, + ] + ) + + monkeypatch.setitem(__import__("sys").modules, "pandas", FakePandas) + graph = build_graph() + policies.replay.from_parquet( + "trace.parquet", + target="nodes", + value_columns=["cpu"], + )(graph) + assert graph.nodes["a"]["cpu"] == 3 + + +def test_replay_missing_error_is_explicit(): + graph = build_graph() + policy = policies.replay.replay_nodes( + [{"time": 0, "node_id": "missing", "cpu": 10}], + value_columns=["cpu"], + missing="error", + ) + + with pytest.raises(KeyError): + policy(graph) + + +def test_replay_filters_start_step_and_edge_missing_behaviour(): + graph = build_graph() + + node_policy = policies.replay.replay_nodes( + [ + {"time": 3, "node_id": "a", "cpu": 11}, + {"time": 4, "node_id": "a", "cpu": 13}, + ], + node_filter=lambda node_id, _: node_id == "a", + start_step=3, + value_columns=["cpu"], + ) + node_policy(graph) + assert graph.nodes["a"]["cpu"] == 11 + + edge_policy = policies.replay.replay_edges( + [ + {"time": 1, "source": "a", "target": "b", "bandwidth": 77}, + {"time": 2, "source": "b", "target": "c", "bandwidth": 55}, + ], + edge_ids=[("a", "b")], + start_step=1, + value_columns=["bandwidth"], + ) + edge_policy(graph) + assert graph.edges["a", "b"]["bandwidth"] == 77 + + failing_edge_policy = policies.replay.replay_edges( + [{"time": 0, "source": "x", "target": "y", "bandwidth": 10}], + value_columns=["bandwidth"], + missing="error", + ) + with pytest.raises(KeyError): + failing_edge_policy(graph) diff --git a/tests/unit/policies/schedule/test_schedule.py b/tests/unit/policies/schedule/test_schedule.py new file mode 100644 index 0000000..339f43c --- /dev/null +++ b/tests/unit/policies/schedule/test_schedule.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import pytest + +from eclypse import policies +from eclypse.graph.asset_graph import AssetGraph +from eclypse.graph.assets import Additive + + +def test_schedule_wrappers_control_policy_timing(): + graph = AssetGraph( + "scheduled", + node_assets={"cpu": Additive(0, 100)}, + update_policies=[ + policies.every( + 2, + lambda graph: graph.nodes["a"].update(cpu=graph.nodes["a"]["cpu"] + 1), + ), + policies.after( + 1, + lambda graph: graph.nodes["a"].update(cpu=graph.nodes["a"]["cpu"] + 1), + ), + policies.between( + 1, + 2, + lambda graph: graph.nodes["a"].update(cpu=graph.nodes["a"]["cpu"] + 1), + ), + policies.once_at( + 2, + lambda graph: graph.nodes["a"].update(cpu=graph.nodes["a"]["cpu"] + 1), + ), + ], + ) + graph.add_node("a", cpu=0) + + for _ in range(4): + graph.evolve() + + assert graph.nodes["a"]["cpu"] == 8 + + +def test_schedule_wrapper_validation_errors(): + def noop(_graph): + return None + + with pytest.raises(ValueError): + policies.after(-1, noop) + + with pytest.raises(ValueError): + policies.between(-1, 1, noop) + + with pytest.raises(ValueError): + policies.between(3, 2, noop) + + with pytest.raises(ValueError): + policies.every(0, noop) + + with pytest.raises(ValueError): + policies.every(1, noop, start=-1) + + with pytest.raises(ValueError): + policies.once_at(-1, noop) diff --git a/tests/unit/remote/communication/test_communication_and_services.py b/tests/unit/remote/communication/test_communication_and_services.py index 4f0c400..6603fb4 100644 --- a/tests/unit/remote/communication/test_communication_and_services.py +++ b/tests/unit/remote/communication/test_communication_and_services.py @@ -25,12 +25,20 @@ register_endpoint, ) from eclypse.remote.communication.rest.methods import HTTPMethod -from eclypse.remote.communication.route import Route +from eclypse.remote.communication.route import ( + Route, + _get_bytes_size, +) from eclypse.remote.service.rest import RESTService from eclypse.remote.service.service import Service from eclypse.remote.utils import ResponseCode +class ExampleObject: + def __init__(self): + self.payload = {"items": [1, 2, 3], "name": "demo"} + + class DummyInterface(EclypseCommunicationInterface): async def _not_connected_response(self): return "offline" @@ -148,6 +156,13 @@ def fake_init(self, recipient_ids, data, _comm, timestamp=None): pending_request.body +def test_route_size_helper_handles_objects_and_nested_structures(): + obj = ExampleObject() + + assert _get_bytes_size({"nested": [1, 2, {"x": 3}]}) > 0 + assert _get_bytes_size(obj) == _get_bytes_size(obj.__dict__) + + @pytest.mark.asyncio async def test_request_initialisation_and_rest_request_wrappers(monkeypatch): loop = asyncio.get_running_loop() diff --git a/tests/unit/remote/service/test_service_runtime.py b/tests/unit/remote/service/test_service_runtime.py index 3f4d3d0..229cff6 100644 --- a/tests/unit/remote/service/test_service_runtime.py +++ b/tests/unit/remote/service/test_service_runtime.py @@ -196,7 +196,7 @@ async def test_service_run_handles_missing_routes_and_stored_steps(dummy_logger) assert any( "route to catalog was not found" in record[1][0] for record in dummy_logger.records - if record[0] == "error" + if record[0] == "warning" ) diff --git a/tests/unit/utils/test_tools_and_logging.py b/tests/unit/utils/test_tools_and_logging.py index 37c934d..efc6a8a 100644 --- a/tests/unit/utils/test_tools_and_logging.py +++ b/tests/unit/utils/test_tools_and_logging.py @@ -14,27 +14,6 @@ LOG_FILE, LOG_LEVEL, ) -from eclypse.utils.tools import ( - camel_to_snake, - get_bytes_size, - prune_assets, -) - - -class ExampleObject: - def __init__(self): - self.payload = {"items": [1, 2, 3], "name": "demo"} - - -def test_tools_measure_size_convert_names_and_prune_assets(sample_infrastructure): - obj = ExampleObject() - - assert camel_to_snake("MyHTTPService") == "my_h_t_t_p_service" - assert get_bytes_size({"nested": [1, 2, {"x": 3}]}) > 0 - assert get_bytes_size(obj) == get_bytes_size(obj.__dict__) - assert prune_assets(sample_infrastructure.node_assets, cpu=1, missing=2) == { - "cpu": 1 - } def test_logging_helpers_configure_and_format_messages( @@ -83,5 +62,11 @@ def test_logging_helpers_configure_and_format_messages( warning_messages = [ args[0] for level, args in dummy_logger.records if level == "warning" ] - assert any("featured 1 | required 2" in message for message in warning_messages) - assert any("upper_bound" in message for message in warning_messages) + trace_messages = [ + args[0] for level, args in dummy_logger.records if level == "trace" + ] + + assert not warning_messages + assert any("available=1 | required=2" in message for message in trace_messages) + assert any("lower_bound=" in message for message in trace_messages) + assert any("upper_bound=" in message for message in trace_messages) diff --git a/tests/unit/workflow/event/test_dispatch.py b/tests/unit/workflow/event/test_dispatch.py index 5f1a9a0..04eb315 100644 --- a/tests/unit/workflow/event/test_dispatch.py +++ b/tests/unit/workflow/event/test_dispatch.py @@ -356,7 +356,6 @@ def test_event_dispatch_by_type_and_runtime_logging( assert quiet_event.data == {"ok": True} assert any(level == "debug" for level, _ in dummy_logger.records) - assert any(level == "log" for level, _ in dummy_logger.records) def test_event_decorator_wraps_callable_classes(): diff --git a/tests/unit/workflow/event/test_event.py b/tests/unit/workflow/event/test_event.py index ff95f81..466bac3 100644 --- a/tests/unit/workflow/event/test_event.py +++ b/tests/unit/workflow/event/test_event.py @@ -8,6 +8,7 @@ EclypseEvent, EventRole, ) +from eclypse.workflow.event.decorator import _camel_to_snake class ConstantEvent(EclypseEvent): @@ -47,3 +48,7 @@ def test_event_properties_and_report_types(): _ = event_obj.simulator assert event_obj.report_types == ["json"] + + +def test_event_name_helper_converts_camel_case(): + assert _camel_to_snake("MyHTTPService") == "my_h_t_t_p_service" diff --git a/tests/unit/workflow/trigger/test_triggers.py b/tests/unit/workflow/trigger/test_triggers.py index 181527e..576d7d9 100644 --- a/tests/unit/workflow/trigger/test_triggers.py +++ b/tests/unit/workflow/trigger/test_triggers.py @@ -6,6 +6,8 @@ ) from types import SimpleNamespace +import pytest + from eclypse.workflow.event import EclypseEvent from eclypse.workflow.trigger import ( CascadeTrigger, @@ -17,6 +19,7 @@ ScheduledTrigger, ) from eclypse.workflow.trigger.bucket import TriggerBucket +from eclypse.workflow.trigger.trigger import Trigger class ConstantEvent(EclypseEvent): @@ -27,31 +30,62 @@ def __call__(self, *_args, **_kwargs): return {"value": 1} +class DummyTrigger(Trigger): + """Concrete trigger used to exercise base trigger helpers.""" + + def trigger(self, *_args, **_kwargs) -> bool: + return False + + def test_triggers_and_trigger_bucket(monkeypatch): periodic = PeriodicTrigger(10) assert periodic.trigger() periodic.reset() periodic.last_exec_time = datetime.now() - timedelta(milliseconds=15) assert periodic.trigger() + assert repr(periodic) == "PeriodicTrigger(trigger_every_ms=10)" scheduled = ScheduledTrigger(timedelta(seconds=1)) scheduled.init() scheduled._scheduled_times = [datetime.now() - timedelta(seconds=1)] # pylint: disable=protected-access assert scheduled.trigger() + assert ( + repr(scheduled) + == "ScheduledTrigger(scheduled_times=[datetime.timedelta(seconds=1)])" + ) monkeypatch.setenv("ECLYPSE_RND_SEED", "3") random_trigger = RandomTrigger(1.0) random_trigger.init() assert random_trigger.trigger() + assert repr(random_trigger) == "RandomTrigger(probability=1.0)" trigger_event = SimpleNamespace(name="step", n_triggers=2) - assert CascadeTrigger("step").trigger(trigger_event) - assert PeriodicCascadeTrigger("step", every_n_triggers=2).trigger(trigger_event) - assert ScheduledCascadeTrigger("step", [2]).trigger(trigger_event) + cascade = CascadeTrigger("step") + assert cascade.trigger(trigger_event) + assert repr(cascade) == "CascadeTrigger(trigger_event=step)" + + periodic_cascade = PeriodicCascadeTrigger("step", every_n_triggers=2) + assert periodic_cascade.trigger(trigger_event) + assert ( + repr(periodic_cascade) + == "PeriodicCascadeTrigger(trigger_event=step, every_n_triggers=2)" + ) + + scheduled_cascade = ScheduledCascadeTrigger("step", [2]) + assert scheduled_cascade.trigger(trigger_event) + assert ( + repr(scheduled_cascade) + == "ScheduledCascadeTrigger(trigger_event=step, scheduled_times=[])" + ) random_cascade = RandomCascadeTrigger("step", probability=1.0, seed=4) random_cascade.init() assert random_cascade.trigger(trigger_event) + assert ( + repr(random_cascade) + == "RandomCascadeTrigger(trigger_event=step, probability=1.0)" + ) bucket = TriggerBucket([CascadeTrigger("step")], condition="any", max_triggers=2) bucket.event = ConstantEvent() @@ -59,3 +93,50 @@ def test_triggers_and_trigger_bucket(monkeypatch): bucket._manual_activation = 1 # pylint: disable=protected-access assert bucket.trigger() assert bucket._n_triggers == 2 # pylint: disable=protected-access + assert repr(bucket) == "TriggerBucket" + + +def test_trigger_helpers_cover_error_and_reset_paths(monkeypatch): + dummy = DummyTrigger() + assert dummy.init() is None + assert dummy.reset() is None + assert repr(dummy) == "DummyTrigger" + + scheduled = ScheduledTrigger() + with pytest.raises(RuntimeError, match="Trigger not initialised"): + scheduled.trigger() + + monkeypatch.setenv("ECLYPSE_RND_SEED", "11") + seeded_random = RandomTrigger(0.5) + seeded_random.init() + assert seeded_random.rnd is not None + + uninitialised_random = RandomTrigger(0.5) + with pytest.raises(RuntimeError, match="Trigger not initialised"): + uninitialised_random.trigger() + + with pytest.raises(ValueError, match="cannot be empty"): + ScheduledCascadeTrigger("step", []) + + uninitialised_random_cascade = RandomCascadeTrigger("step", probability=0.5) + with pytest.raises(RuntimeError, match="Trigger not initialised"): + uninitialised_random_cascade.trigger(SimpleNamespace(name="step")) + + all_bucket = TriggerBucket( + [CascadeTrigger("step"), PeriodicCascadeTrigger("step", every_n_triggers=3)], + condition="all", + max_triggers=1, + ) + all_bucket.event = ConstantEvent() + assert all_bucket.trigger(SimpleNamespace(name="step", n_triggers=3)) is True + assert all_bucket.trigger(SimpleNamespace(name="step", n_triggers=3)) is False + all_bucket.reset() + assert all_bucket._n_executions == 1 # pylint: disable=protected-access + + empty_bucket = TriggerBucket(condition="any") + assert empty_bucket.trigger() is False + with pytest.raises(ValueError, match="Event is not set"): + empty_bucket.logger() + + empty_bucket.event = ConstantEvent("bucket") + assert empty_bucket.logger() is not None