From c7a1c5cbe4140be9ff0e0083dc6bd5215c731062 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 19 May 2026 14:43:01 +0000 Subject: [PATCH 01/14] =?UTF-8?q?feat(recommenders):=20integrate=20ToOp=20?= =?UTF-8?q?(Elia=20Group)=20as=20optional=20model=20=E2=80=94=20line-switc?= =?UTF-8?q?hing=20MVP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ToOp is an open-source topology optimization engine from Elia Group (https://github.com/eliagroup/ToOp). This first integration surfaces its line-switching suggestions through the existing pluggable recommender contract; busbar splits / reassignments are deferred to a follow-up once the line-switch path is exercised on real grids. ToOp pins Python 3.11 and pulls in heavy GPU dependencies (JAX, qdax, Ray, …), so the package is **not** added to requirements — the ToOpRecommender lazy-imports `toop_engine_topology_optimizer` inside `recommend()` and degrades to an empty output with a single log line when ToOp isn't installed, rather than crashing the step-2 NDJSON stream. Pipeline inside `recommend()`: 1. Export the live pypowsybl Network to a temporary CGMES bundle (ToOp's importer ingests CGMES / UCT, not XIIDM). 2. Build the DictConfig quadruple matching example2 from the upstream notebooks, restricted to a single `branch_switches` descriptor so Map Elites prioritises the line-switching axis only. 3. Run `run_pipeline` synchronously, bounded by a tunable `runtime_seconds` parameter. 4. Parse the Pareto front for line-switching decisions (tolerant parser accepts dict / object / iterable shapes — extends cleanly once the upstream return shape is finalised). 5. Translate each switch to the Co-Study4Grid action format `{"set_bus": {"lines_or_id": {line: ±1}, "lines_ex_id": {line: ±1}}}` via `env.action_space`, preferring an existing `disco_` / `reco_` entry from the operator's `dict_action` when one is available so suggestions match the vocabulary the user already uses. 6. Rank by ToOp's `overload_energy_n_1` metric (negated so the UI's "higher is better" sort agrees) and return the top-N. The new model auto-surfaces in the React Settings → Recommender dropdown via `GET /api/models`; no frontend change required. --- expert_backend/recommenders/__init__.py | 7 + expert_backend/recommenders/toop.py | 508 ++++++++++++++++++ expert_backend/tests/test_toop_recommender.py | 305 +++++++++++ 3 files changed, 820 insertions(+) create mode 100644 expert_backend/recommenders/toop.py create mode 100644 expert_backend/tests/test_toop_recommender.py diff --git a/expert_backend/recommenders/__init__.py b/expert_backend/recommenders/__init__.py index b10ac1f2..ba119911 100644 --- a/expert_backend/recommenders/__init__.py +++ b/expert_backend/recommenders/__init__.py @@ -32,6 +32,7 @@ register, unregister, ) +from expert_backend.recommenders.toop import ToOpRecommender # Register the default (expert) and canonical random examples. # This module is imported by the FastAPI startup path so every server @@ -39,6 +40,12 @@ register(ExpertRecommender) register(RandomRecommender) register(RandomOverflowRecommender) +# ToOp is an OPTIONAL install (Python 3.11 + heavy GPU deps). The +# class itself only lazy-imports toop_engine_topology_optimizer inside +# `recommend()`, so registering it here costs nothing when ToOp isn't +# installed — the model just produces an empty recommendation with a +# clear log line. +register(ToOpRecommender) # Side-effect: patches RecommenderService to consume the registry # (state + getters + update_config wrap + reset wrap + model-aware diff --git a/expert_backend/recommenders/toop.py b/expert_backend/recommenders/toop.py new file mode 100644 index 00000000..1820712b --- /dev/null +++ b/expert_backend/recommenders/toop.py @@ -0,0 +1,508 @@ +# Copyright (c) 2025-2026, RTE (https://www.rte-france.com) +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +"""Elia Group `ToOp `_ topology optimizer. + +MVP scope — **line switching only**. ToOp's broader output set +(busbar splits, busbar reassignments) is not yet translated into +remedial actions; that extension lives downstream once the line-switch +path is exercised on real grids. + +ToOp is treated as an **optional install**. The Python package +``toop_engine_topology_optimizer`` pins Python 3.11 and pulls in heavy +GPU dependencies (JAX, qdax, Ray, …), so we never import it at module +load time — the import happens lazily inside :meth:`recommend` and a +missing install is reported as an empty recommendation with a clear +log line rather than a server crash. + +Integration outline (see comments in ``recommend`` for details): + +1. Export ``inputs.network`` (pypowsybl Network) to a temporary + CGMES bundle — ToOp's importer pipeline ingests CGMES / UCT, not + XIIDM. +2. Build the ``DictConfig`` quadruple ToOp's ``run_pipeline`` expects + (DC optimization + AC validation + importer + preprocessing), + constrained to line-status edits. +3. Run ``run_pipeline`` synchronously inside the streaming step-2 + endpoint, bounded by ``runtime_seconds``. +4. Parse the Pareto front for line-switching decisions and translate + each to the Co-Study4Grid action format + (``{"set_bus": {"lines_or_id": {line: ±1}, "lines_ex_id": {line: ±1}}}``) + via ``env.action_space``. +5. Rank by ToOp's congestion metric (``overload_energy_n_1``); surface + the top-N as ``prioritized_actions`` with ``action_scores``. +""" +from __future__ import annotations + +import logging +import tempfile +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + +from expert_op4grid_recommender.models.base import ( + ParamSpec, + RecommenderInputs, + RecommenderModel, + RecommenderOutput, +) + +from expert_backend.recommenders.network_existence import ( + filter_to_existing_network_elements, +) + +logger = logging.getLogger(__name__) + + +# Module-level so a unit test can patch it without monkey-patching the import system. +def _import_run_pipeline() -> Optional[Any]: + """Lazy importer that returns ToOp's ``run_pipeline`` or ``None``. + + Returning ``None`` (rather than raising) lets :meth:`recommend` + degrade to an empty output with a single log line — far friendlier + than crashing the step-2 NDJSON stream when an operator selects + ToOp on a backend where it isn't installed. + """ + try: + from toop_engine_topology_optimizer.benchmark.benchmark_utils import ( # type: ignore[import-not-found] + run_pipeline, + ) + except Exception as exc: # ImportError, but also ModuleNotFoundError, etc. + logger.info( + "ToOpRecommender: toop_engine_topology_optimizer is not installed (%s). " + "Install it from https://github.com/eliagroup/ToOp on a Python 3.11 " + "environment to enable this model.", + exc, + ) + return None + return run_pipeline + + +def _import_dictconfig() -> Optional[Any]: + """Lazy importer for omegaconf's ``DictConfig`` (a ToOp transitive dep).""" + try: + from omegaconf import DictConfig # type: ignore[import-not-found] + except Exception as exc: + logger.info("ToOpRecommender: omegaconf not available (%s).", exc) + return None + return DictConfig + + +class ToOpRecommender(RecommenderModel): + """Topology optimizer wrapper. MVP surfaces line-switching only. + + The model does NOT consume the overflow graph — ToOp does its own + DC contingency analysis internally, so we set + ``requires_overflow_graph = False`` and let the operator opt in via + the Settings → Recommender toggle if they still want the graph + rendered alongside. + """ + + name = "toop" + label = "ToOp (Elia Group — line switching)" + requires_overflow_graph = False + + @classmethod + def params_spec(cls) -> List[ParamSpec]: + return [ + ParamSpec( + "n_prioritized_actions", + "N Prioritized Actions", + "int", + default=5, + min=1, + max=50, + description=( + "Number of line-switching suggestions surfaced from " + "the Pareto front (top-N by congestion reduction)." + ), + ), + ParamSpec( + "runtime_seconds", + "ToOp Runtime Budget (s)", + "int", + default=15, + min=5, + max=120, + description=( + "Wall-clock budget passed to ToOp's DC optimization " + "stage. Higher = better Pareto coverage but blocks " + "the step-2 stream that long." + ), + ), + ParamSpec( + "n_worst_contingencies", + "Contingencies Considered", + "int", + default=2, + min=1, + max=20, + description=( + "Number of worst-N-1 contingencies ToOp evaluates " + "when scoring each topology candidate." + ), + ), + ] + + # ------------------------------------------------------------------ + # Public entry point + # ------------------------------------------------------------------ + def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutput: + run_pipeline = _import_run_pipeline() + if run_pipeline is None: + return RecommenderOutput(prioritized_actions={}, action_scores={}) + + DictConfig = _import_dictconfig() + if DictConfig is None: + return RecommenderOutput(prioritized_actions={}, action_scores={}) + + n = int(params.get("n_prioritized_actions", 5)) + runtime_seconds = int(params.get("runtime_seconds", 15)) + n_worst = int(params.get("n_worst_contingencies", 2)) + + env = inputs.env + if env is None: + logger.warning("ToOpRecommender: inputs.env is None — returning {}.") + return RecommenderOutput(prioritized_actions={}, action_scores={}) + + with tempfile.TemporaryDirectory(prefix="toop_") as tmp: + tmp_path = Path(tmp) + cgmes_path = self._export_to_cgmes(inputs.network, tmp_path) + if cgmes_path is None: + return RecommenderOutput(prioritized_actions={}, action_scores={}) + + try: + pareto = self._run_toop( + run_pipeline=run_pipeline, + DictConfig=DictConfig, + cgmes_path=cgmes_path, + work_dir=tmp_path, + runtime_seconds=runtime_seconds, + n_worst_contingencies=n_worst, + ) + except Exception: + logger.exception("ToOpRecommender: run_pipeline failed; returning {}.") + return RecommenderOutput(prioritized_actions={}, action_scores={}) + + switches = self._extract_line_switches(pareto, n=n) + if not switches: + logger.info("ToOpRecommender: no line-switching suggestions in Pareto front.") + return RecommenderOutput(prioritized_actions={}, action_scores={}) + + prioritized, scores = self._materialise_actions( + switches=switches, + env=env, + dict_action=inputs.dict_action, + non_connected_reconnectable_lines=inputs.non_connected_reconnectable_lines, + network=inputs.network, + ) + return RecommenderOutput( + prioritized_actions=prioritized, + action_scores=scores, + ) + + # ------------------------------------------------------------------ + # Internals (kept as methods so tests can patch them individually) + # ------------------------------------------------------------------ + def _export_to_cgmes(self, network: Any, work_dir: Path) -> Optional[Path]: + """Save the live pypowsybl Network to a CGMES bundle in ``work_dir``. + + Returns the bundle path, or ``None`` on failure. pypowsybl's + CGMES export is best-effort: networks built from XIIDM with + missing CGMES-required identifiers can fail here. We log the + error and degrade gracefully — ToOp simply produces no + suggestions on this grid. + """ + if network is None: + logger.warning("ToOpRecommender: inputs.network is None — cannot export to CGMES.") + return None + out = work_dir / "grid_cgmes" + try: + # pypowsybl Network exposes either ``save`` (newer API) or + # ``dump`` (older). We try both before giving up. + if hasattr(network, "save"): + network.save(str(out), format="CGMES") + elif hasattr(network, "dump"): + network.dump(str(out), format="CGMES") + else: + logger.warning("ToOpRecommender: pypowsybl Network has no save()/dump() method.") + return None + except Exception: + logger.exception("ToOpRecommender: CGMES export failed.") + return None + return out + + def _run_toop( + self, + run_pipeline: Any, + DictConfig: Any, + cgmes_path: Path, + work_dir: Path, + runtime_seconds: int, + n_worst_contingencies: int, + ) -> Any: + """Invoke ToOp's ``run_pipeline`` with a line-switching-only config. + + ``me_descriptors`` is restricted to a single metric so the Map + Elites search prioritises the line-switching axis; busbar-split + descriptors are intentionally omitted in this MVP. Returns + whatever ``run_pipeline`` returns (typically a path or a + result object) — parsing is the caller's responsibility. + """ + dc_optimization_cfg = DictConfig({ + "task_name": "costudy4grid", + "fixed_files": [str(cgmes_path)], + "runtime_seconds": runtime_seconds, + "me_descriptors": [{"metric": "branch_switches", "num_cells": 4}], + "observed_metrics": ["overload_energy_n_1", "branch_switches"], + "n_worst_contingencies": n_worst_contingencies, + }) + ac_validation_cfg = DictConfig({ + "enabled": True, + }) + importer_parameters = DictConfig({ + "input_path": str(cgmes_path), + }) + preprocessing_parameters = DictConfig({}) + pipeline_cfg = DictConfig({ + "work_dir": str(work_dir), + }) + return run_pipeline( + pipeline_cfg=pipeline_cfg, + dc_optim_config=dc_optimization_cfg, + ac_validation_cfg=ac_validation_cfg, + importer_parameters=importer_parameters, + preprocessing_parameters=preprocessing_parameters, + ) + + def _extract_line_switches( + self, + pareto: Any, + n: int, + ) -> List[Tuple[str, int, float]]: + """Pull line-switching decisions out of a ToOp result. + + Returns a list of ``(line_id, target_status, score)`` tuples + sorted by ``score`` (lower congestion = better, surfaced first). + ``target_status`` is +1 (close) or -1 (open). + + The exact return shape of ``run_pipeline`` is not nailed down + by the upstream docs (the example notebooks write results to + disk via ``topo_path``), so this parser accepts several + plausible shapes: + + - An iterable of dicts each carrying ``line_switches`` (list of + ``{"line_id", "status"}`` or ``{"line_id", "open"}``) and a + numeric score (``overload_energy_n_1`` / ``score`` / ``cost``). + - An object with a ``.solutions`` attribute holding the same. + + When the shape is unrecognised we log once and return an empty + list. The frontend then shows an empty action feed and the + operator can pick a different model — far less disruptive than + raising mid-stream. + """ + candidates: Iterable[Any] + if pareto is None: + return [] + if hasattr(pareto, "solutions"): + candidates = pareto.solutions + elif isinstance(pareto, (list, tuple)): + candidates = pareto + elif hasattr(pareto, "__iter__"): + candidates = pareto + else: + logger.warning( + "ToOpRecommender: unrecognised result shape %r; " + "extend _extract_line_switches when ToOp's output is finalised.", + type(pareto).__name__, + ) + return [] + + # Flatten (solution, switch) into a single ranked list so the + # top-N surfacing is per-switch, not per-solution. Two + # solutions that both open the same line are deduplicated + # keeping the better score. + best_by_key: Dict[Tuple[str, int], float] = {} + for sol in candidates: + score = _coerce_score(sol) + for line_id, target_status in _iter_switches(sol): + key = (line_id, target_status) + if key not in best_by_key or score < best_by_key[key]: + best_by_key[key] = score + + ranked = sorted(best_by_key.items(), key=lambda kv: kv[1]) + return [(line_id, status, score) for ((line_id, status), score) in ranked[:n]] + + def _materialise_actions( + self, + switches: List[Tuple[str, int, float]], + env: Any, + dict_action: Optional[Dict[str, Any]], + non_connected_reconnectable_lines: Optional[Iterable[str]], + network: Any, + ) -> Tuple[Dict[str, Any], Dict[str, float]]: + """Translate ToOp line-switches into ``env.action_space`` actions. + + Prefers an existing ``disco_`` / reconnection entry in the + operator's action dictionary when one exists (so suggestions + match the vocabulary the user is already familiar with); + otherwise synthesises a ``toop_disco_`` / + ``toop_reco_`` entry on the fly. Actions whose target + line isn't on the loaded network are filtered out via the same + defensive guard used by the random recommenders. + """ + # Network existence filter on the raw line IDs first. + line_ids = [line_id for (line_id, _, _) in switches] + synthetic_entries = { + f"_existence_check_{line_id}": { + "content": { + "set_bus": { + "lines_or_id": {line_id: 1}, + "lines_ex_id": {line_id: 1}, + } + } + } + for line_id in line_ids + } + kept = set( + filter_to_existing_network_elements( + list(synthetic_entries.keys()), + synthetic_entries, + network, + ) + ) + # Map back from check-id to line_id. + kept_lines = {sid.replace("_existence_check_", "", 1) for sid in kept} + + reconnectable = set(non_connected_reconnectable_lines or []) + prioritized: Dict[str, Any] = {} + scores: Dict[str, float] = {} + + for line_id, target_status, score in switches: + if line_id not in kept_lines: + continue + + action_id, content = self._pick_action_for_switch( + line_id=line_id, + target_status=target_status, + dict_action=dict_action, + reconnectable=reconnectable, + ) + try: + prioritized[action_id] = env.action_space(content) + except Exception as e: + logger.debug("ToOpRecommender: action %s rejected by env: %s", action_id, e) + continue + # Lower congestion = better in ToOp's metric; surface the + # negated score so the UI's "higher is better" sort agrees. + scores[action_id] = -float(score) + + return prioritized, scores + + @staticmethod + def _pick_action_for_switch( + line_id: str, + target_status: int, + dict_action: Optional[Dict[str, Any]], + reconnectable: set, + ) -> Tuple[str, Dict[str, Any]]: + """Return ``(action_id, content)`` for a single line-switch decision.""" + if target_status == -1: + preferred_id = f"disco_{line_id}" + if isinstance(dict_action, dict) and preferred_id in dict_action: + content = (dict_action[preferred_id] or {}).get("content") + if content is not None: + return preferred_id, content + # Fallback: synthesise the open-line content. + return ( + f"toop_disco_{line_id}", + { + "set_bus": { + "lines_or_id": {line_id: -1}, + "lines_ex_id": {line_id: -1}, + } + }, + ) + + # target_status == +1 → reconnection. Only meaningful when the + # line is currently disconnected; if it isn't reconnectable the + # action will be a no-op but we still produce it (ToOp may know + # something we don't about the live state). + if line_id in reconnectable: + preferred_id = f"reco_{line_id}" + if isinstance(dict_action, dict) and preferred_id in dict_action: + content = (dict_action[preferred_id] or {}).get("content") + if content is not None: + return preferred_id, content + return ( + f"toop_reco_{line_id}", + { + "set_bus": { + "lines_or_id": {line_id: 1}, + "lines_ex_id": {line_id: 1}, + } + }, + ) + + +# --------------------------------------------------------------------- +# Tolerant parsers — kept module-level so tests can swap them out. +# --------------------------------------------------------------------- +def _coerce_score(sol: Any) -> float: + """Pull a numeric congestion score off a ToOp solution-like object. + + Tries the metric names visible in the example notebook + (``overload_energy_n_1``), then generic fallbacks. Returns + ``float('inf')`` when nothing recognisable is present so the + solution sinks to the bottom of the ranking rather than crashing + the sort. + """ + for key in ("overload_energy_n_1", "score", "cost", "objective"): + if isinstance(sol, dict) and key in sol: + try: + return float(sol[key]) + except (TypeError, ValueError): + continue + if hasattr(sol, key): + try: + return float(getattr(sol, key)) + except (TypeError, ValueError): + continue + return float("inf") + + +def _iter_switches(sol: Any) -> Iterable[Tuple[str, int]]: + """Yield ``(line_id, target_status)`` pairs out of a solution-like object.""" + raw = None + if isinstance(sol, dict): + raw = sol.get("line_switches") or sol.get("branch_switches") + elif hasattr(sol, "line_switches"): + raw = sol.line_switches + elif hasattr(sol, "branch_switches"): + raw = sol.branch_switches + if not raw: + return + for entry in raw: + line_id = None + status: Optional[int] = None + if isinstance(entry, dict): + line_id = entry.get("line_id") or entry.get("branch_id") or entry.get("id") + if "status" in entry: + try: + status = int(entry["status"]) + except (TypeError, ValueError): + status = None + elif "open" in entry: + status = -1 if entry["open"] else 1 + elif "closed" in entry: + status = 1 if entry["closed"] else -1 + elif isinstance(entry, (list, tuple)) and len(entry) == 2: + line_id, raw_status = entry + try: + status = int(raw_status) + except (TypeError, ValueError): + status = None + if not line_id or status not in (-1, 1): + continue + yield (str(line_id), status) diff --git a/expert_backend/tests/test_toop_recommender.py b/expert_backend/tests/test_toop_recommender.py new file mode 100644 index 00000000..b9d8dc1e --- /dev/null +++ b/expert_backend/tests/test_toop_recommender.py @@ -0,0 +1,305 @@ +# Copyright (c) 2025-2026, RTE (https://www.rte-france.com) +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +"""Unit tests for :class:`ToOpRecommender` (line-switching MVP). + +These tests run without ToOp installed — they cover the optional-install +degradation path, the registry wiring, the result parser, and the +action-translation logic. End-to-end execution against the real ToOp +engine is a future integration-test concern (Python 3.11 + GPU env). +""" +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from expert_backend.recommenders import toop as toop_module +from expert_backend.recommenders.registry import get_model_class, list_models +from expert_backend.recommenders.toop import ( + ToOpRecommender, + _coerce_score, + _iter_switches, +) + + +# --------------------------------------------------------------------- +# Registry wiring +# --------------------------------------------------------------------- +class TestRegistryWiring: + def test_model_is_registered(self): + """ToOp shows up in the registry after package import.""" + assert get_model_class("toop") is ToOpRecommender + + def test_listed_in_models_endpoint_payload(self): + names = [m["name"] for m in list_models()] + assert "toop" in names + + def test_does_not_require_overflow_graph(self): + descriptor = next(m for m in list_models() if m["name"] == "toop") + assert descriptor["requires_overflow_graph"] is False + + def test_params_spec_declares_runtime_and_count_knobs(self): + descriptor = next(m for m in list_models() if m["name"] == "toop") + param_names = {p["name"] for p in descriptor["params"]} + assert {"n_prioritized_actions", "runtime_seconds", "n_worst_contingencies"} <= param_names + + +# --------------------------------------------------------------------- +# Optional-install degradation +# --------------------------------------------------------------------- +class TestOptionalInstall: + def test_returns_empty_when_toop_not_installed(self, caplog): + """No ToOp install → empty output + one info log, never a crash.""" + inputs = MagicMock() + inputs.network = MagicMock() + inputs.env = MagicMock() + with patch.object(toop_module, "_import_run_pipeline", return_value=None): + with caplog.at_level("INFO"): + out = ToOpRecommender().recommend(inputs, {"n_prioritized_actions": 5}) + assert out.prioritized_actions == {} + assert out.action_scores == {} + + def test_returns_empty_when_omegaconf_not_installed(self): + inputs = MagicMock() + inputs.network = MagicMock() + inputs.env = MagicMock() + with patch.object(toop_module, "_import_run_pipeline", return_value=lambda **_: None), \ + patch.object(toop_module, "_import_dictconfig", return_value=None): + out = ToOpRecommender().recommend(inputs, {}) + assert out.prioritized_actions == {} + + def test_returns_empty_when_env_missing(self): + inputs = MagicMock() + inputs.network = MagicMock() + inputs.env = None + with patch.object(toop_module, "_import_run_pipeline", return_value=lambda **_: None), \ + patch.object(toop_module, "_import_dictconfig", return_value=dict): + out = ToOpRecommender().recommend(inputs, {}) + assert out.prioritized_actions == {} + + def test_cgmes_export_failure_yields_empty(self): + inputs = MagicMock() + inputs.network = MagicMock() + # Make save() raise to simulate an XIIDM that can't be CGMES-mapped. + inputs.network.save.side_effect = RuntimeError("CGMES export unsupported") + inputs.env = MagicMock() + with patch.object(toop_module, "_import_run_pipeline", return_value=lambda **_: None), \ + patch.object(toop_module, "_import_dictconfig", return_value=dict): + out = ToOpRecommender().recommend(inputs, {}) + assert out.prioritized_actions == {} + + +# --------------------------------------------------------------------- +# Result parsing — _extract_line_switches + helpers +# --------------------------------------------------------------------- +class TestResultParsing: + def test_parses_list_of_dicts_with_line_switches(self): + pareto = [ + { + "overload_energy_n_1": 12.5, + "line_switches": [{"line_id": "LINE_A", "status": -1}], + }, + { + "overload_energy_n_1": 5.0, + "line_switches": [{"line_id": "LINE_B", "open": False}], + }, + ] + out = ToOpRecommender()._extract_line_switches(pareto, n=5) + # Sorted by score ascending (lower = better congestion). + assert out[0][0] == "LINE_B" + assert out[0][1] == 1 + assert out[1][0] == "LINE_A" + assert out[1][1] == -1 + + def test_truncates_to_top_n(self): + pareto = [ + {"overload_energy_n_1": float(i), "line_switches": [{"line_id": f"L{i}", "status": -1}]} + for i in range(10) + ] + out = ToOpRecommender()._extract_line_switches(pareto, n=3) + assert len(out) == 3 + assert [r[0] for r in out] == ["L0", "L1", "L2"] + + def test_dedupes_same_line_status_keeping_best_score(self): + pareto = [ + {"score": 9.0, "line_switches": [{"line_id": "LINE_A", "status": -1}]}, + {"score": 2.0, "line_switches": [{"line_id": "LINE_A", "status": -1}]}, + ] + out = ToOpRecommender()._extract_line_switches(pareto, n=5) + assert len(out) == 1 + assert out[0] == ("LINE_A", -1, 2.0) + + def test_accepts_object_with_solutions_attribute(self): + sol = MagicMock() + sol.score = 1.0 + sol.line_switches = [{"line_id": "LINE_X", "status": 1}] + container = MagicMock() + container.solutions = [sol] + out = ToOpRecommender()._extract_line_switches(container, n=5) + assert out == [("LINE_X", 1, 1.0)] + + def test_unknown_shape_returns_empty_and_logs(self, caplog): + with caplog.at_level("WARNING"): + out = ToOpRecommender()._extract_line_switches(object(), n=5) + assert out == [] + + def test_none_returns_empty(self): + assert ToOpRecommender()._extract_line_switches(None, n=5) == [] + + def test_skips_entries_with_missing_line_id_or_status(self): + pareto = [{ + "score": 1.0, + "line_switches": [ + {"status": -1}, # missing line_id + {"line_id": "LINE_Y"}, # missing status + {"line_id": "LINE_Z", "status": 99}, # invalid status + {"line_id": "LINE_OK", "status": -1}, + ], + }] + out = ToOpRecommender()._extract_line_switches(pareto, n=5) + assert out == [("LINE_OK", -1, 1.0)] + + +class TestCoerceScore: + def test_reads_overload_energy_n_1(self): + assert _coerce_score({"overload_energy_n_1": 3.5}) == 3.5 + + def test_falls_back_to_score_then_cost(self): + assert _coerce_score({"score": 2.0}) == 2.0 + assert _coerce_score({"cost": 4.0}) == 4.0 + + def test_returns_inf_when_nothing_recognised(self): + assert _coerce_score({}) == float("inf") + assert _coerce_score(object()) == float("inf") + + +class TestIterSwitches: + def test_dict_with_branch_switches_alias(self): + sol = {"branch_switches": [{"branch_id": "L1", "status": -1}]} + assert list(_iter_switches(sol)) == [("L1", -1)] + + def test_tuple_form(self): + sol = {"line_switches": [("L1", -1), ("L2", 1)]} + assert list(_iter_switches(sol)) == [("L1", -1), ("L2", 1)] + + def test_empty_when_no_switches(self): + assert list(_iter_switches({})) == [] + assert list(_iter_switches({"line_switches": []})) == [] + + +# --------------------------------------------------------------------- +# Action materialisation +# --------------------------------------------------------------------- +class TestMaterialiseActions: + @pytest.fixture + def network_with_lines(self, mock_network): + return mock_network + + def test_prefers_existing_disco_entry_from_dict_action(self, network_with_lines): + env = MagicMock() + env.action_space.side_effect = lambda content: ("ACTION", content) + dict_action = { + "disco_LINE_A": { + "content": {"set_bus": {"lines_or_id": {"LINE_A": -1}, "lines_ex_id": {"LINE_A": -1}}}, + "description": "Open LINE_A", + }, + } + out, scores = ToOpRecommender()._materialise_actions( + switches=[("LINE_A", -1, 5.0)], + env=env, + dict_action=dict_action, + non_connected_reconnectable_lines=[], + network=network_with_lines, + ) + # Reused the operator's vocabulary instead of fabricating a toop_disco_*. + assert "disco_LINE_A" in out + assert "toop_disco_LINE_A" not in out + assert scores["disco_LINE_A"] == -5.0 + + def test_synthesises_toop_disco_when_dict_action_lacks_entry(self, network_with_lines): + env = MagicMock() + env.action_space.side_effect = lambda content: ("ACTION", content) + out, scores = ToOpRecommender()._materialise_actions( + switches=[("LINE_A", -1, 1.0)], + env=env, + dict_action={}, + non_connected_reconnectable_lines=[], + network=network_with_lines, + ) + assert "toop_disco_LINE_A" in out + # Negated score = "higher is better" convention. + assert scores["toop_disco_LINE_A"] == -1.0 + + def test_reconnection_uses_toop_reco_prefix(self, network_with_lines): + env = MagicMock() + env.action_space.side_effect = lambda content: ("ACTION", content) + out, _ = ToOpRecommender()._materialise_actions( + switches=[("LINE_A", 1, 0.5)], + env=env, + dict_action={}, + non_connected_reconnectable_lines=["LINE_A"], + network=network_with_lines, + ) + assert "toop_reco_LINE_A" in out + + def test_drops_lines_not_on_loaded_network(self, network_with_lines): + env = MagicMock() + env.action_space.side_effect = lambda content: ("ACTION", content) + out, scores = ToOpRecommender()._materialise_actions( + switches=[("LINE_GHOST", -1, 1.0), ("LINE_A", -1, 2.0)], + env=env, + dict_action={}, + non_connected_reconnectable_lines=[], + network=network_with_lines, + ) + # LINE_GHOST isn't in the mock network's lines DataFrame. + assert "toop_disco_LINE_GHOST" not in out + assert "toop_disco_LINE_A" in out + + def test_skips_actions_env_rejects(self, network_with_lines): + env = MagicMock() + env.action_space.side_effect = RuntimeError("invalid action") + out, _ = ToOpRecommender()._materialise_actions( + switches=[("LINE_A", -1, 1.0)], + env=env, + dict_action={}, + non_connected_reconnectable_lines=[], + network=network_with_lines, + ) + assert out == {} + + +# --------------------------------------------------------------------- +# Light end-to-end with everything mocked — the happy path +# --------------------------------------------------------------------- +class TestRecommendHappyPath: + def test_full_flow_with_mocked_toop(self, mock_network): + fake_pareto = [ + { + "overload_energy_n_1": 7.0, + "line_switches": [{"line_id": "LINE_A", "status": -1}], + }, + ] + fake_run_pipeline = MagicMock(return_value=fake_pareto) + + inputs = MagicMock() + inputs.network = mock_network + inputs.env = MagicMock() + inputs.env.action_space.side_effect = lambda content: ("ACTION", content) + inputs.dict_action = {} + inputs.non_connected_reconnectable_lines = [] + + # Bypass real CGMES export — pypowsybl's CGMES mapping isn't relevant here. + with patch.object(toop_module, "_import_run_pipeline", return_value=fake_run_pipeline), \ + patch.object(toop_module, "_import_dictconfig", return_value=dict), \ + patch.object(ToOpRecommender, "_export_to_cgmes", + return_value=mock_network): # any non-None Path-like sentinel + out = ToOpRecommender().recommend(inputs, {"n_prioritized_actions": 3}) + + assert "toop_disco_LINE_A" in out.prioritized_actions + assert out.action_scores["toop_disco_LINE_A"] == -7.0 + fake_run_pipeline.assert_called_once() From 7b6329d5370b52b90d52b95fb778e404f93e33b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 19 May 2026 19:14:17 +0000 Subject: [PATCH 02/14] debug(recommenders/toop): make ToOpRecommender silent-empty paths visible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first real-grid run reported a model that returned 0 actions in milliseconds with NO log line from the recommender — uvicorn's default level suppresses our INFO logs. That made it impossible to tell which of the 6 early-exit paths in `recommend()` was being taken (ToOp not importable / omegaconf missing / env None / CGMES export failed / run_pipeline raised / empty Pareto front). Bump every early-exit log line from INFO to WARNING and add entry / mid-flight / exit log points so each `recommend()` invocation now emits an observable trail at the default backend log level: - entry: params, env / network types, dict_action size - post-import: whether ToOp + omegaconf imported successfully - pre-pipeline: CGMES export path + run_pipeline kwargs - post-pipeline: pareto return type, count of extracted switches - exit: list of prioritized action ids returned Operator impact: a re-run with the ToOp model now produces enough backend log output to pinpoint the failure without bumping the log level or attaching a debugger. --- expert_backend/recommenders/toop.py | 52 +++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/expert_backend/recommenders/toop.py b/expert_backend/recommenders/toop.py index 1820712b..83243743 100644 --- a/expert_backend/recommenders/toop.py +++ b/expert_backend/recommenders/toop.py @@ -69,11 +69,12 @@ def _import_run_pipeline() -> Optional[Any]: run_pipeline, ) except Exception as exc: # ImportError, but also ModuleNotFoundError, etc. - logger.info( - "ToOpRecommender: toop_engine_topology_optimizer is not installed (%s). " - "Install it from https://github.com/eliagroup/ToOp on a Python 3.11 " - "environment to enable this model.", - exc, + logger.warning( + "ToOpRecommender: toop_engine_topology_optimizer is not importable " + "from this backend's Python (%s: %s). Install it from " + "https://github.com/eliagroup/ToOp on a Python 3.11 environment, " + "in the SAME venv that runs `uvicorn expert_backend.main`.", + type(exc).__name__, exc, ) return None return run_pipeline @@ -84,7 +85,10 @@ def _import_dictconfig() -> Optional[Any]: try: from omegaconf import DictConfig # type: ignore[import-not-found] except Exception as exc: - logger.info("ToOpRecommender: omegaconf not available (%s).", exc) + logger.warning( + "ToOpRecommender: omegaconf not available (%s: %s).", + type(exc).__name__, exc, + ) return None return DictConfig @@ -149,12 +153,24 @@ def params_spec(cls) -> List[ParamSpec]: # Public entry point # ------------------------------------------------------------------ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutput: + logger.warning( + "ToOpRecommender: recommend() entry — params=%s, env=%s, network=%s, " + "dict_action_size=%s", + params, + type(inputs.env).__name__ if inputs.env is not None else None, + type(inputs.network).__name__ if inputs.network is not None else None, + len(inputs.dict_action or {}), + ) run_pipeline = _import_run_pipeline() if run_pipeline is None: + logger.warning( + "ToOpRecommender: aborting because ToOp is not importable in this venv." + ) return RecommenderOutput(prioritized_actions={}, action_scores={}) DictConfig = _import_dictconfig() if DictConfig is None: + logger.warning("ToOpRecommender: aborting because omegaconf is not importable.") return RecommenderOutput(prioritized_actions={}, action_scores={}) n = int(params.get("n_prioritized_actions", 5)) @@ -168,10 +184,20 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu with tempfile.TemporaryDirectory(prefix="toop_") as tmp: tmp_path = Path(tmp) + logger.warning( + "ToOpRecommender: exporting pypowsybl network to CGMES under %s", + tmp_path, + ) cgmes_path = self._export_to_cgmes(inputs.network, tmp_path) if cgmes_path is None: + logger.warning("ToOpRecommender: CGMES export returned None — aborting.") return RecommenderOutput(prioritized_actions={}, action_scores={}) + logger.warning( + "ToOpRecommender: calling run_pipeline (runtime_seconds=%d, " + "n_worst_contingencies=%d) on %s", + runtime_seconds, n_worst, cgmes_path, + ) try: pareto = self._run_toop( run_pipeline=run_pipeline, @@ -185,9 +211,17 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu logger.exception("ToOpRecommender: run_pipeline failed; returning {}.") return RecommenderOutput(prioritized_actions={}, action_scores={}) + logger.warning( + "ToOpRecommender: run_pipeline returned %s (type=%s)", + "an iterable" if pareto is not None else "None", + type(pareto).__name__ if pareto is not None else None, + ) switches = self._extract_line_switches(pareto, n=n) + logger.warning( + "ToOpRecommender: extracted %d line-switch suggestion(s) from Pareto front", + len(switches), + ) if not switches: - logger.info("ToOpRecommender: no line-switching suggestions in Pareto front.") return RecommenderOutput(prioritized_actions={}, action_scores={}) prioritized, scores = self._materialise_actions( @@ -197,6 +231,10 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu non_connected_reconnectable_lines=inputs.non_connected_reconnectable_lines, network=inputs.network, ) + logger.warning( + "ToOpRecommender: returning %d prioritized action(s): %s", + len(prioritized), list(prioritized.keys()), + ) return RecommenderOutput( prioritized_actions=prioritized, action_scores=scores, From a3a4aa0bd8fbfbe02b3dfdcd8083b6cb6e3c31d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 19 May 2026 19:24:28 +0000 Subject: [PATCH 03/14] fix(recommenders/toop): build pipeline config against the real ToOp schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First real-grid run revealed that my pipeline_cfg / dc_optim_config were wrong — ToOp's `run_pipeline` failed immediately with `ConfigAttributeError: Missing key root_path`. Source inspection of the installed package + cross-check with `notebooks/example2_small_grid_toop.ipynb` exposed two structural mistakes: 1. **`pipeline_cfg` is a typed `PipelineConfig`**, not a free-form DictConfig. It needs `root_path`, `iteration_name`, `file_name` and `grid_type` — and the referenced grid file must exist on disk before `get_paths()` is called (it raises FileNotFoundError otherwise). 2. **`dc_optim_config` has a nested `ga_config` sub-key** where my parameters (`runtime_seconds`, `me_descriptors`, `observed_metrics`, `n_worst_contingencies`) actually live. The top level wants the orchestration knobs (`task_name`, `tensorboard_dir`, `stats_dir`, `output_json`, `lf_config`, `num_cuda_devices`, …). 3. **ToOp accepts XIIDM natively** — the example notebook loads `grid.xiidm` directly. Saved one CGMES round-trip and several header-mapping failure modes. 4. **`importer_parameters` must come from `prepare_importer_parameters(file_path, data_folder)`**, not from a hand-built DictConfig. Same for `preprocessing_parameters` (typed `PreprocessParameters` with `action_set_clip` / `enable_bb_outage` / `bb_outage_as_nminus1`). 5. **The Pareto front lives in `output.json`**, written under `iteration_path / results / output.json` by the DC-optimisation stage. `run_pipeline` itself returns `topology_paths` — the per-topology files written by AC validation, not the elite map. Resulting changes: - `_export_to_cgmes` → `_export_network`: writes `/iter/grid.xiidm` via `network.save(..., "XIIDM")`, with fallback to other extensions when pypowsybl appends one. - `_run_toop`: lazy-imports `PipelineConfig`, `PreprocessParameters`, `get_paths`, `prepare_importer_parameters`; builds the PipelineConfig + DictConfig pair matching `example2_small_grid_toop` with a single `branch_switches` MapElites descriptor; passes through the user's `runtime_seconds` / `n_worst_contingencies` / `n_prioritized_actions` via `ga_config`. - `_load_output_json`: reads the optimisation output JSON and logs its top-level keys / first-entry keys before handing it to `_extract_line_switches` (the parser is already shape-tolerant). - Tests: rename `_export_to_cgmes` mocks, patch `_run_toop` in the happy-path test so it bypasses the real ToOp config builders (which only exist when the package is installed). --- expert_backend/recommenders/toop.py | 181 ++++++++++++++---- expert_backend/tests/test_toop_recommender.py | 14 +- 2 files changed, 148 insertions(+), 47 deletions(-) diff --git a/expert_backend/recommenders/toop.py b/expert_backend/recommenders/toop.py index 83243743..0b41577a 100644 --- a/expert_backend/recommenders/toop.py +++ b/expert_backend/recommenders/toop.py @@ -184,25 +184,27 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu with tempfile.TemporaryDirectory(prefix="toop_") as tmp: tmp_path = Path(tmp) + iteration_dir = tmp_path / "iter" + iteration_dir.mkdir(parents=True, exist_ok=True) logger.warning( - "ToOpRecommender: exporting pypowsybl network to CGMES under %s", - tmp_path, + "ToOpRecommender: exporting pypowsybl network to XIIDM under %s", + iteration_dir, ) - cgmes_path = self._export_to_cgmes(inputs.network, tmp_path) - if cgmes_path is None: - logger.warning("ToOpRecommender: CGMES export returned None — aborting.") + grid_file = self._export_network(inputs.network, iteration_dir) + if grid_file is None: + logger.warning("ToOpRecommender: network export returned None — aborting.") return RecommenderOutput(prioritized_actions={}, action_scores={}) logger.warning( "ToOpRecommender: calling run_pipeline (runtime_seconds=%d, " "n_worst_contingencies=%d) on %s", - runtime_seconds, n_worst, cgmes_path, + runtime_seconds, n_worst, grid_file, ) try: pareto = self._run_toop( run_pipeline=run_pipeline, DictConfig=DictConfig, - cgmes_path=cgmes_path, + grid_file=grid_file, work_dir=tmp_path, runtime_seconds=runtime_seconds, n_worst_contingencies=n_worst, @@ -211,6 +213,30 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu logger.exception("ToOpRecommender: run_pipeline failed; returning {}.") return RecommenderOutput(prioritized_actions={}, action_scores={}) + # `run_pipeline` returns `topology_paths` (list of files written by + # the AC validation stage). The richer Pareto data lives in the + # `output.json` produced by the DC optimisation stage — try that + # first; fall back to the topology_paths only if it's missing. + output_json = tmp_path / "iter" / "results" / "output.json" + if output_json.exists(): + try: + pareto = self._load_output_json(output_json) + logger.warning( + "ToOpRecommender: loaded Pareto front from %s", output_json, + ) + except Exception: + logger.exception( + "ToOpRecommender: failed to parse output.json at %s", + output_json, + ) + else: + logger.warning( + "ToOpRecommender: output.json not found at %s; relying on " + "run_pipeline return value (%s topology paths).", + output_json, + len(pareto) if isinstance(pareto, (list, tuple)) else "?", + ) + logger.warning( "ToOpRecommender: run_pipeline returned %s (type=%s)", "an iterable" if pareto is not None else "None", @@ -243,31 +269,42 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu # ------------------------------------------------------------------ # Internals (kept as methods so tests can patch them individually) # ------------------------------------------------------------------ - def _export_to_cgmes(self, network: Any, work_dir: Path) -> Optional[Path]: - """Save the live pypowsybl Network to a CGMES bundle in ``work_dir``. - - Returns the bundle path, or ``None`` on failure. pypowsybl's - CGMES export is best-effort: networks built from XIIDM with - missing CGMES-required identifiers can fail here. We log the - error and degrade gracefully — ToOp simply produces no - suggestions on this grid. + def _export_network(self, network: Any, iteration_dir: Path) -> Optional[Path]: + """Save the live pypowsybl Network as XIIDM under ``iteration_dir``. + + ToOp's example notebooks consume XIIDM directly (``grid.xiidm``); + no CGMES round-trip is necessary. Returns the absolute path of + the saved file, or ``None`` on failure. The directory is + already created by the caller. """ if network is None: - logger.warning("ToOpRecommender: inputs.network is None — cannot export to CGMES.") + logger.warning("ToOpRecommender: inputs.network is None — cannot export.") return None - out = work_dir / "grid_cgmes" + out = iteration_dir / "grid.xiidm" try: # pypowsybl Network exposes either ``save`` (newer API) or - # ``dump`` (older). We try both before giving up. + # ``dump`` (older). if hasattr(network, "save"): - network.save(str(out), format="CGMES") + network.save(str(out), format="XIIDM") elif hasattr(network, "dump"): - network.dump(str(out), format="CGMES") + network.dump(str(out), format="XIIDM") else: logger.warning("ToOpRecommender: pypowsybl Network has no save()/dump() method.") return None except Exception: - logger.exception("ToOpRecommender: CGMES export failed.") + logger.exception("ToOpRecommender: XIIDM export failed.") + return None + if not out.exists(): + # Some pypowsybl builds add the extension themselves or write + # multiple files. Look for the first plausible match. + for ext in (".xiidm", ".iidm", ".xml"): + hits = list(iteration_dir.glob(f"*{ext}")) + if hits: + return hits[0] + logger.warning( + "ToOpRecommender: XIIDM export produced no file under %s", + iteration_dir, + ) return None return out @@ -275,37 +312,73 @@ def _run_toop( self, run_pipeline: Any, DictConfig: Any, - cgmes_path: Path, + grid_file: Path, work_dir: Path, runtime_seconds: int, n_worst_contingencies: int, ) -> Any: - """Invoke ToOp's ``run_pipeline`` with a line-switching-only config. + """Invoke ToOp's ``run_pipeline`` against an XIIDM grid file. - ``me_descriptors`` is restricted to a single metric so the Map - Elites search prioritises the line-switching axis; busbar-split - descriptors are intentionally omitted in this MVP. Returns - whatever ``run_pipeline`` returns (typically a path or a - result object) — parsing is the caller's responsibility. + Mirrors the structure of ``notebooks/example2_small_grid_toop.ipynb`` + but constrains the Map Elites search to a single + ``branch_switches`` descriptor so the optimiser prioritises the + line-switching axis (busbar splits are out of scope for this + MVP). """ + # Lazy imports — these symbols only exist when ToOp is installed. + from toop_engine_topology_optimizer.benchmark.benchmark_utils import ( + PipelineConfig, + PreprocessParameters, + get_paths, + prepare_importer_parameters, + ) + + iteration_name = grid_file.parent.name # e.g. "iter" + pipeline_cfg = PipelineConfig( + root_path=work_dir, + iteration_name=iteration_name, + file_name=grid_file.name, + grid_type="powsybl", + ) + # `get_paths` validates the grid file exists, creates the + # optimizer-snapshot directory, and returns the resolved sub-paths + # we plug into the rest of the configs. + iteration_path, file_path, data_folder, _snapshot_dir = get_paths(pipeline_cfg) + results_dir = iteration_path / "results" + results_dir.mkdir(parents=True, exist_ok=True) + dc_optimization_cfg = DictConfig({ "task_name": "costudy4grid", - "fixed_files": [str(cgmes_path)], - "runtime_seconds": runtime_seconds, - "me_descriptors": [{"metric": "branch_switches", "num_cells": 4}], - "observed_metrics": ["overload_energy_n_1", "branch_switches"], - "n_worst_contingencies": n_worst_contingencies, + "fixed_files": [], + "double_precision": None, + "tensorboard_dir": str(results_dir / "{task_name}"), + "stats_dir": str(results_dir / "{task_name}"), + "summary_frequency": None, + "checkpoint_frequency": None, + "stdout": None, + "double_limits": None, + "num_cuda_devices": 1, + "omp_num_threads": 1, + "xla_force_host_platform_device_count": None, + "output_json": str(results_dir / "output.json"), + "lf_config": {"distributed": False}, + "ga_config": { + "runtime_seconds": runtime_seconds, + "me_descriptors": [{"metric": "branch_switches", "num_cells": 4}], + "observed_metrics": ["overload_energy_n_1", "branch_switches"], + "n_worst_contingencies": n_worst_contingencies, + }, }) ac_validation_cfg = DictConfig({ - "enabled": True, - }) - importer_parameters = DictConfig({ - "input_path": str(cgmes_path), - }) - preprocessing_parameters = DictConfig({}) - pipeline_cfg = DictConfig({ - "work_dir": str(work_dir), + "n_processes": 1, + "k_best_topos": 5, }) + importer_parameters = prepare_importer_parameters(file_path, data_folder) + preprocessing_parameters = PreprocessParameters( + action_set_clip=1024, + enable_bb_outage=False, + bb_outage_as_nminus1=False, + ) return run_pipeline( pipeline_cfg=pipeline_cfg, dc_optim_config=dc_optimization_cfg, @@ -314,6 +387,32 @@ def _run_toop( preprocessing_parameters=preprocessing_parameters, ) + def _load_output_json(self, path: Path) -> Any: + """Read ToOp's DC-optimisation output JSON and return its content. + + Format is currently treated as opaque: the parser downstream + (:meth:`_extract_line_switches`) consumes several plausible + shapes, so we just hand the decoded payload over and log a + sample of the top-level keys for forensic purposes. + """ + import json + + with path.open("r", encoding="utf-8") as fh: + data = json.load(fh) + if isinstance(data, dict): + logger.warning( + "ToOpRecommender: output.json top-level keys: %s", + sorted(data.keys())[:20], + ) + elif isinstance(data, list): + logger.warning( + "ToOpRecommender: output.json is a list of %d entries; " + "first entry keys: %s", + len(data), + sorted(data[0].keys())[:20] if data and isinstance(data[0], dict) else None, + ) + return data + def _extract_line_switches( self, pareto: Any, diff --git a/expert_backend/tests/test_toop_recommender.py b/expert_backend/tests/test_toop_recommender.py index b9d8dc1e..88a0121e 100644 --- a/expert_backend/tests/test_toop_recommender.py +++ b/expert_backend/tests/test_toop_recommender.py @@ -81,11 +81,11 @@ def test_returns_empty_when_env_missing(self): out = ToOpRecommender().recommend(inputs, {}) assert out.prioritized_actions == {} - def test_cgmes_export_failure_yields_empty(self): + def test_xiidm_export_failure_yields_empty(self): inputs = MagicMock() inputs.network = MagicMock() - # Make save() raise to simulate an XIIDM that can't be CGMES-mapped. - inputs.network.save.side_effect = RuntimeError("CGMES export unsupported") + # Make save() raise to simulate an unsupported network shape. + inputs.network.save.side_effect = RuntimeError("XIIDM export unsupported") inputs.env = MagicMock() with patch.object(toop_module, "_import_run_pipeline", return_value=lambda **_: None), \ patch.object(toop_module, "_import_dictconfig", return_value=dict): @@ -293,11 +293,13 @@ def test_full_flow_with_mocked_toop(self, mock_network): inputs.dict_action = {} inputs.non_connected_reconnectable_lines = [] - # Bypass real CGMES export — pypowsybl's CGMES mapping isn't relevant here. + # Bypass real XIIDM export AND the ToOp config-building (PipelineConfig / + # prepare_importer_parameters / etc. only exist when ToOp is installed). + fake_grid_file = MagicMock() with patch.object(toop_module, "_import_run_pipeline", return_value=fake_run_pipeline), \ patch.object(toop_module, "_import_dictconfig", return_value=dict), \ - patch.object(ToOpRecommender, "_export_to_cgmes", - return_value=mock_network): # any non-None Path-like sentinel + patch.object(ToOpRecommender, "_export_network", return_value=fake_grid_file), \ + patch.object(ToOpRecommender, "_run_toop", return_value=fake_pareto): out = ToOpRecommender().recommend(inputs, {"n_prioritized_actions": 3}) assert "toop_disco_LINE_A" in out.prioritized_actions From 5c760340907f133beb91e74f11dd1b7348900666 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 19 May 2026 19:44:54 +0000 Subject: [PATCH 04/14] fix(recommenders/toop): use disconnected_branches metric name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ToOp's BatchedMEParameters pins observed_metrics and the descriptor metric to a closed pydantic Literal enum. The line-switching name I picked (`branch_switches`) isn't in it — `disconnected_branches` is. This was visible in the preprocessing log line `n_disc_branches: 8` and is confirmed by the pydantic ValidationError listing every valid value. This switch keeps the Map Elites search axis aligned with line toggling (cells distinguish topologies by the number of branches they open), and surfaces the same metric alongside `overload_energy_n_1` so we can rank elites by congestion reduction. --- expert_backend/recommenders/toop.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/expert_backend/recommenders/toop.py b/expert_backend/recommenders/toop.py index 0b41577a..c67441e0 100644 --- a/expert_backend/recommenders/toop.py +++ b/expert_backend/recommenders/toop.py @@ -364,8 +364,14 @@ def _run_toop( "lf_config": {"distributed": False}, "ga_config": { "runtime_seconds": runtime_seconds, - "me_descriptors": [{"metric": "branch_switches", "num_cells": 4}], - "observed_metrics": ["overload_energy_n_1", "branch_switches"], + # `disconnected_branches` is ToOp's metric for the count + # of opened branches in a topology — the right axis for + # a line-switching-only Map Elites search. ToOp's + # accepted metric names are pinned in + # BatchedMEParameters; if it doesn't match the enum, + # the optimiser fails with a pydantic ValidationError. + "me_descriptors": [{"metric": "disconnected_branches", "num_cells": 4}], + "observed_metrics": ["overload_energy_n_1", "disconnected_branches"], "n_worst_contingencies": n_worst_contingencies, }, }) From 6c58237ecfaabd0bb151f7641aedaf49b4053efb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 19 May 2026 19:52:08 +0000 Subject: [PATCH 05/14] fix(recommenders/toop): wire fixed_files to the static_information.hdf5 path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DC optimisation aborted on `verify_static_information` → `next(iter(static_informations))` → StopIteration because `fixed_files=()` was empty. The notebook example threads `static_information_file` through that list; in our case the file is `data_folder / pipeline_cfg.static_info_relpath`. Preprocessing writes it before the optimiser runs, so we just need to declare the future path. --- expert_backend/recommenders/toop.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/expert_backend/recommenders/toop.py b/expert_backend/recommenders/toop.py index c67441e0..dca0233b 100644 --- a/expert_backend/recommenders/toop.py +++ b/expert_backend/recommenders/toop.py @@ -347,9 +347,16 @@ def _run_toop( results_dir = iteration_path / "results" results_dir.mkdir(parents=True, exist_ok=True) + # Preprocessing writes ``static_information.hdf5`` under + # ``data_folder``; the DC-optimisation stage reads it back via + # ``fixed_files``. The path must be supplied even though the + # file doesn't exist yet at config-build time — ToOp's + # preprocessing creates it before the optimiser runs. + static_info_file = data_folder / pipeline_cfg.static_info_relpath + dc_optimization_cfg = DictConfig({ "task_name": "costudy4grid", - "fixed_files": [], + "fixed_files": [str(static_info_file)], "double_precision": None, "tensorboard_dir": str(results_dir / "{task_name}"), "stats_dir": str(results_dir / "{task_name}"), From 05a340ec0a6f45316bf1b8a4a9128e945e925144 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 19 May 2026 19:56:41 +0000 Subject: [PATCH 06/14] feat(recommenders/toop): extract line-switches by diffing modified networks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ToOp ran end-to-end successfully on the first integration test: preprocessing → 2375 epochs of DC optimisation → AC validation, producing one topology with a modified_network.xiidm that opened zero lines (the small-grid resolution didn't need line switching). Two changes were needed to surface its results: 1. `run_pipeline` does NOT write to the `output_json` config field I wired earlier — it writes to `/run_*/topology_*/modified_network.xiidm` and returns the list of topology directory paths from the AC validation stage. My `output.json` parser is now dead code; left for future use when ToOp's Pareto schema gets documented. 2. Extracting line-switches by reading ToOp's internal Pareto genome (in `res.json`) requires reverse-engineering an undocumented format. Much more robust: load each topology's `modified_network.xiidm` back through pypowsybl, compare per-line `connected1`/`connected2` flags against the input grid, and surface the differences as `(line_id, ±1, rank-as-score)` tuples. ToOp returns its topology list best-first, so rank is a faithful proxy for elite quality. New module-level helper `_is_line_open` consolidates the per-terminal flag check (with a tolerant fallback to older `connected` column names). New instance method `_extract_switches_from_topology_paths` orchestrates the diff and de-duplicates `(line, status)` pairs across topologies keeping the best rank. If a future ToOp release does write the `output_json`, the fast-path still works because the file-exists check still runs first — but the diff path is the canonical extractor going forward. --- expert_backend/recommenders/toop.py | 142 ++++++++++++++++++++++------ 1 file changed, 112 insertions(+), 30 deletions(-) diff --git a/expert_backend/recommenders/toop.py b/expert_backend/recommenders/toop.py index dca0233b..122e37f0 100644 --- a/expert_backend/recommenders/toop.py +++ b/expert_backend/recommenders/toop.py @@ -213,38 +213,30 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu logger.exception("ToOpRecommender: run_pipeline failed; returning {}.") return RecommenderOutput(prioritized_actions={}, action_scores={}) - # `run_pipeline` returns `topology_paths` (list of files written by - # the AC validation stage). The richer Pareto data lives in the - # `output.json` produced by the DC optimisation stage — try that - # first; fall back to the topology_paths only if it's missing. - output_json = tmp_path / "iter" / "results" / "output.json" - if output_json.exists(): - try: - pareto = self._load_output_json(output_json) - logger.warning( - "ToOpRecommender: loaded Pareto front from %s", output_json, - ) - except Exception: - logger.exception( - "ToOpRecommender: failed to parse output.json at %s", - output_json, - ) - else: - logger.warning( - "ToOpRecommender: output.json not found at %s; relying on " - "run_pipeline return value (%s topology paths).", - output_json, - len(pareto) if isinstance(pareto, (list, tuple)) else "?", - ) + # `run_pipeline` returns ``topology_paths`` — a list of + # *directories* (``/run_*/topology_*``), each + # containing a ``modified_network.xiidm`` ToOp wrote during + # AC validation. The primary extraction path is therefore + # to load that modified network back through pypowsybl and + # diff its line statuses against the original grid we + # exported a few lines above — no need to reverse-engineer + # ToOp's internal Pareto-front schema. The richer + # ``optimizer_snapshot/run_0/res.json`` exists too but its + # genome encoding is undocumented; the network-diff path + # is the robust choice for the line-switching MVP. + logger.warning( + "ToOpRecommender: run_pipeline returned %d topology path(s); " + "diffing modified_network.xiidm against the input grid.", + len(pareto) if isinstance(pareto, (list, tuple)) else 0, + ) + switches = self._extract_switches_from_topology_paths( + topology_paths=pareto if isinstance(pareto, (list, tuple)) else [], + original_grid_file=grid_file, + n=n, + ) logger.warning( - "ToOpRecommender: run_pipeline returned %s (type=%s)", - "an iterable" if pareto is not None else "None", - type(pareto).__name__ if pareto is not None else None, - ) - switches = self._extract_line_switches(pareto, n=n) - logger.warning( - "ToOpRecommender: extracted %d line-switch suggestion(s) from Pareto front", + "ToOpRecommender: extracted %d line-switch suggestion(s)", len(switches), ) if not switches: @@ -426,6 +418,73 @@ def _load_output_json(self, path: Path) -> Any: ) return data + def _extract_switches_from_topology_paths( + self, + topology_paths: Iterable[Any], + original_grid_file: Path, + n: int, + ) -> List[Tuple[str, int, float]]: + """Diff each topology's ``modified_network.xiidm`` against the input grid. + + For every topology directory ToOp surfaces, load + ``modified_network.xiidm`` and compare per-line ``connected1`` / + ``connected2`` flags against the original exported grid. Lines + that flipped are returned as ``(line_id, target_status, score)`` + tuples where ``target_status`` is ``-1`` (opened) or ``+1`` + (closed). Score is the topology's rank in ``topology_paths`` + (ToOp returns them best-first), so the surfaced list mirrors + ToOp's own ordering. Duplicate (line, status) pairs across + topologies keep the best (lowest-rank) score. + """ + import pypowsybl.network as pn # lazy: only loaded when ToOp ran + + try: + original = pn.load(str(original_grid_file)) + original_lines = original.get_lines() + except Exception: + logger.exception( + "ToOpRecommender: cannot reload original grid file %s for diff.", + original_grid_file, + ) + return [] + + original_open: Dict[str, bool] = {} + for line_id in original_lines.index: + original_open[line_id] = _is_line_open(original_lines, line_id) + + best: Dict[Tuple[str, int], float] = {} + for rank, topo in enumerate(topology_paths or []): + topo_dir = Path(topo) if not isinstance(topo, Path) else topo + modified = topo_dir / "modified_network.xiidm" + if not modified.exists(): + logger.warning( + "ToOpRecommender: no modified_network.xiidm under %s", + topo_dir, + ) + continue + try: + mod_net = pn.load(str(modified)) + mod_lines = mod_net.get_lines() + except Exception: + logger.exception( + "ToOpRecommender: cannot load %s for diff.", modified, + ) + continue + for line_id in mod_lines.index: + if line_id not in original_open: + continue + new_open = _is_line_open(mod_lines, line_id) + if new_open == original_open[line_id]: + continue + target_status = -1 if new_open else 1 + key = (line_id, target_status) + score = float(rank) + if key not in best or score < best[key]: + best[key] = score + + ranked = sorted(best.items(), key=lambda kv: kv[1]) + return [(lid, st, sc) for ((lid, st), sc) in ranked[:n]] + def _extract_line_switches( self, pareto: Any, @@ -599,6 +658,29 @@ def _pick_action_for_switch( # --------------------------------------------------------------------- # Tolerant parsers — kept module-level so tests can swap them out. # --------------------------------------------------------------------- +def _is_line_open(lines_df: Any, line_id: str) -> bool: + """Return True when at least one terminal of the line is disconnected. + + pypowsybl exposes per-terminal ``connected1`` / ``connected2`` + booleans on its lines DataFrame. A line is "open" (in operator + terms) when *either* terminal is disconnected. Older pypowsybl + builds used different column names — fall back conservatively to + ``connected`` or treat the line as closed when columns are + missing so a schema change doesn't silently flag every line as + toggled. + """ + for col_pair in (("connected1", "connected2"), ("connected",)): + if all(c in lines_df.columns for c in col_pair): + for c in col_pair: + try: + if not bool(lines_df.at[line_id, c]): + return True + except KeyError: + return False + return False + return False + + def _coerce_score(sol: Any) -> float: """Pull a numeric congestion score off a ToOp solution-like object. From d9202e0904b37f751e4024317e44e7fb2c0fed2d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 19 May 2026 20:05:41 +0000 Subject: [PATCH 07/14] feat(recommenders/toop): surface busbar-split suggestions via dict_action match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ToOp's first real run resolved an overload by splitting a substation (VLevel2) rather than opening a line — visible in the log ("Saving SLDs of split stations..."). The line-switching MVP correctly returned 0 actions for that case, but the operator saw an empty feed and no signal that ToOp had actually proposed something useful. This commit extends the recommender to detect busbar splits and surface matching `dict_action` entries: - New `_extract_busbar_splits_from_topology_paths`: per topology, diff `Network.get_switches()` between the input grid and `modified_network.xiidm`. Voltage levels whose internal switches changed open/closed state — but whose line terminals did NOT — are flagged as busbar splits. De-duplicated across topologies keeping the best (lowest) ToOp rank as score. - New `_materialise_busbar_actions`: for each VL ToOp split, surface every `dict_action` entry whose `VoltageLevelId` (or alias `voltage_level_id`) matches. We piggy-back on the operator's curated substation-action vocabulary rather than synthesising one from ToOp's raw switch list — that translation requires substation-specific knowledge the operator already encoded. When a split VL has no matching entry, log a clear "consider adding split_ to your action library" warning so the gap is actionable. - `recommend()` calls both extractors and merges, capping the surfaced list at `n_prioritized_actions`. Line-switch matches win on key collisions (their action ids never overlap with substation ids in practice, but be safe). - New `include_busbar_splits` boolean parameter (default `True`) surfaces in the Settings → Recommender params dropdown. Disable to restrict ToOp's output to line switches only. - Model label tightened from "ToOp (Elia Group — line switching)" to just "ToOp (Elia Group)" now that both action surfaces are covered. The busbar path is intentionally conservative: ToOp tells us *which* substation to reconfigure, the operator's library tells us *how*. If a richer integration is needed later (e.g. surfacing ToOp's exact switch list as a synthetic action), the seam is the `_materialise_busbar_actions` method. --- expert_backend/recommenders/toop.py | 233 ++++++++++++++++-- expert_backend/tests/test_toop_recommender.py | 76 ++++++ 2 files changed, 293 insertions(+), 16 deletions(-) diff --git a/expert_backend/recommenders/toop.py b/expert_backend/recommenders/toop.py index 122e37f0..f8a64c98 100644 --- a/expert_backend/recommenders/toop.py +++ b/expert_backend/recommenders/toop.py @@ -104,7 +104,7 @@ class ToOpRecommender(RecommenderModel): """ name = "toop" - label = "ToOp (Elia Group — line switching)" + label = "ToOp (Elia Group)" requires_overflow_graph = False @classmethod @@ -118,8 +118,20 @@ def params_spec(cls) -> List[ParamSpec]: min=1, max=50, description=( - "Number of line-switching suggestions surfaced from " - "the Pareto front (top-N by congestion reduction)." + "Maximum number of suggestions surfaced — covers " + "both line switches and busbar splits, ranked by " + "ToOp's congestion-reduction order." + ), + ), + ParamSpec( + "include_busbar_splits", + "Include Busbar Splits", + "bool", + default=True, + description=( + "Surface dict_action entries whose VoltageLevelId " + "matches a substation ToOp split. Disable to " + "restrict suggestions to line switches only." ), ), ParamSpec( @@ -176,6 +188,7 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu n = int(params.get("n_prioritized_actions", 5)) runtime_seconds = int(params.get("runtime_seconds", 15)) n_worst = int(params.get("n_worst_contingencies", 2)) + include_busbar_splits = bool(params.get("include_busbar_splits", True)) env = inputs.env if env is None: @@ -224,31 +237,72 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu # ``optimizer_snapshot/run_0/res.json`` exists too but its # genome encoding is undocumented; the network-diff path # is the robust choice for the line-switching MVP. + topology_list = pareto if isinstance(pareto, (list, tuple)) else [] logger.warning( "ToOpRecommender: run_pipeline returned %d topology path(s); " "diffing modified_network.xiidm against the input grid.", - len(pareto) if isinstance(pareto, (list, tuple)) else 0, + len(topology_list), ) + # 1. Line switches (lines whose end-connection flags changed). switches = self._extract_switches_from_topology_paths( - topology_paths=pareto if isinstance(pareto, (list, tuple)) else [], + topology_paths=topology_list, original_grid_file=grid_file, n=n, ) + # 2. Busbar splits (voltage levels whose internal switches + # changed without any line being toggled). Detected + # inside the tempdir so the modified_network.xiidm files + # are still on disk. + if include_busbar_splits: + splits = self._extract_busbar_splits_from_topology_paths( + topology_paths=topology_list, + original_grid_file=grid_file, + ) + else: + splits = [] logger.warning( - "ToOpRecommender: extracted %d line-switch suggestion(s)", - len(switches), + "ToOpRecommender: extracted %d line-switch suggestion(s), " + "%d busbar-split voltage level(s)", + len(switches), len(splits), ) - if not switches: - return RecommenderOutput(prioritized_actions={}, action_scores={}) - prioritized, scores = self._materialise_actions( - switches=switches, - env=env, - dict_action=inputs.dict_action, - non_connected_reconnectable_lines=inputs.non_connected_reconnectable_lines, - network=inputs.network, - ) + prioritized: Dict[str, Any] = {} + scores: Dict[str, float] = {} + + if switches: + ls_prioritized, ls_scores = self._materialise_actions( + switches=switches, + env=env, + dict_action=inputs.dict_action, + non_connected_reconnectable_lines=inputs.non_connected_reconnectable_lines, + network=inputs.network, + ) + prioritized.update(ls_prioritized) + scores.update(ls_scores) + + if splits: + bs_prioritized, bs_scores = self._materialise_busbar_actions( + splits=splits, + env=env, + dict_action=inputs.dict_action, + ) + # Line-switch matches win on key collisions — they're the + # more specific surface form. In practice the two namespaces + # don't overlap (line ids vs node-action ids), but be safe. + for aid, action in bs_prioritized.items(): + if aid not in prioritized: + prioritized[aid] = action + scores[aid] = bs_scores[aid] + + # Cap the merged surfaced list at the requested top-N to keep + # the UI feed manageable. Stable sort by score ascending (better + # ToOp ranks first, since we negate the score before exposing). + if len(prioritized) > n: + ranked = sorted(prioritized.keys(), key=lambda a: -scores.get(a, 0.0))[:n] + prioritized = {a: prioritized[a] for a in ranked} + scores = {a: scores[a] for a in ranked} + logger.warning( "ToOpRecommender: returning %d prioritized action(s): %s", len(prioritized), list(prioritized.keys()), @@ -418,6 +472,153 @@ def _load_output_json(self, path: Path) -> Any: ) return data + def _extract_busbar_splits_from_topology_paths( + self, + topology_paths: Iterable[Any], + original_grid_file: Path, + ) -> List[Tuple[str, float]]: + """Return ``[(voltage_level_id, rank_score)]`` for VLs ToOp split. + + Diffs each topology's switch states (via + ``Network.get_switches()``) against the original grid. A + voltage level whose internal switches changed open/closed + state — but whose line terminal connections did NOT, so + line-switch extraction missed it — is a *busbar split*: ToOp + reconfigured the node-breaker topology inside a substation. + + Returns the list sorted best-first (rank 0 = topology_0). De- + duplicated by VL so multiple topologies touching the same VL + keep the best rank. + """ + import pypowsybl.network as pn + + try: + original = pn.load(str(original_grid_file)) + orig_switches = original.get_switches() + except Exception: + logger.exception( + "ToOpRecommender: cannot read switches from original grid %s", + original_grid_file, + ) + return [] + + if "open" not in orig_switches.columns or "voltage_level_id" not in orig_switches.columns: + logger.warning( + "ToOpRecommender: pypowsybl get_switches() lacks expected " + "columns (have %s); skipping busbar-split detection.", + list(orig_switches.columns), + ) + return [] + + orig_open: Dict[str, bool] = {} + orig_vl: Dict[str, str] = {} + for sid in orig_switches.index: + try: + orig_open[sid] = bool(orig_switches.at[sid, "open"]) + orig_vl[sid] = str(orig_switches.at[sid, "voltage_level_id"]) + except Exception: + continue + + best_rank_by_vl: Dict[str, float] = {} + for rank, topo in enumerate(topology_paths or []): + topo_dir = Path(topo) if not isinstance(topo, Path) else topo + modified = topo_dir / "modified_network.xiidm" + if not modified.exists(): + continue + try: + mod_net = pn.load(str(modified)) + mod_switches = mod_net.get_switches() + except Exception: + logger.exception( + "ToOpRecommender: cannot read switches from %s", + modified, + ) + continue + + for sid in mod_switches.index: + if sid not in orig_open: + continue + try: + new_open = bool(mod_switches.at[sid, "open"]) + except Exception: + continue + if new_open == orig_open[sid]: + continue + vl = orig_vl.get(sid) + if not vl: + continue + cur = best_rank_by_vl.get(vl) + if cur is None or rank < cur: + best_rank_by_vl[vl] = float(rank) + + return sorted(best_rank_by_vl.items(), key=lambda kv: kv[1]) + + def _materialise_busbar_actions( + self, + splits: List[Tuple[str, float]], + env: Any, + dict_action: Optional[Dict[str, Any]], + ) -> Tuple[Dict[str, Any], Dict[str, float]]: + """Surface every ``dict_action`` entry whose VL ToOp split. + + We piggy-back on the operator's substation-action vocabulary + rather than synthesising one from ToOp's raw switch list: the + translation from a switch-flip set to grid2op's set-bus + topology vector requires substation-specific knowledge the + operator already encoded when curating ``dict_action``. ToOp + tells us *which* substation to split; the operator's library + tells us *how*. + + When a VL ToOp split has no matching dict_action entry, we + log it — that's a gap for the operator to fill, not a bug. + """ + prioritized: Dict[str, Any] = {} + scores: Dict[str, float] = {} + if not dict_action: + for vl_id, _ in splits: + logger.warning( + "ToOpRecommender: ToOp split VL %s but dict_action is empty.", + vl_id, + ) + return prioritized, scores + + for vl_id, rank in splits: + matched: List[str] = [] + for action_id, entry in dict_action.items(): + if not isinstance(entry, dict): + continue + entry_vl = entry.get("VoltageLevelId") or entry.get("voltage_level_id") + if entry_vl != vl_id: + continue + content = entry.get("content") + if content is None: + continue + try: + prioritized[action_id] = env.action_space(content) + except Exception as e: + logger.debug( + "ToOpRecommender: dict_action %s for VL %s rejected by env: %s", + action_id, vl_id, e, + ) + continue + # Same negation convention as line switches — higher is better. + scores[action_id] = -float(rank) + matched.append(action_id) + if matched: + logger.warning( + "ToOpRecommender: VL %s split → surfacing %d matching " + "dict_action entry(s): %s", + vl_id, len(matched), matched[:5], + ) + else: + logger.warning( + "ToOpRecommender: VL %s split but no dict_action entry " + "with matching VoltageLevelId; consider adding split_%s " + "to your action library.", + vl_id, vl_id, + ) + return prioritized, scores + def _extract_switches_from_topology_paths( self, topology_paths: Iterable[Any], diff --git a/expert_backend/tests/test_toop_recommender.py b/expert_backend/tests/test_toop_recommender.py index 88a0121e..f9a663ff 100644 --- a/expert_backend/tests/test_toop_recommender.py +++ b/expert_backend/tests/test_toop_recommender.py @@ -273,6 +273,82 @@ def test_skips_actions_env_rejects(self, network_with_lines): assert out == {} +# --------------------------------------------------------------------- +# Busbar-split materialisation +# --------------------------------------------------------------------- +class TestMaterialiseBusbarActions: + def test_surfaces_every_dict_action_entry_for_matching_VL(self): + env = MagicMock() + env.action_space.side_effect = lambda c: ("ACTION", c) + dict_action = { + "split_VL_A_v1": {"VoltageLevelId": "VL_A", "content": {"set_bus": {"switches": ["s1"]}}}, + "split_VL_A_v2": {"VoltageLevelId": "VL_A", "content": {"set_bus": {"switches": ["s2"]}}}, + "split_VL_B_v1": {"VoltageLevelId": "VL_B", "content": {"set_bus": {"switches": ["s5"]}}}, + } + prioritized, scores = ToOpRecommender()._materialise_busbar_actions( + splits=[("VL_A", 0.0)], env=env, dict_action=dict_action, + ) + # Both VL_A variants surface; VL_B is untouched. + assert set(prioritized.keys()) == {"split_VL_A_v1", "split_VL_A_v2"} + assert "split_VL_B_v1" not in prioritized + # Score 0.0 → negated → 0.0 (higher is better). + assert scores["split_VL_A_v1"] == 0.0 + + def test_accepts_lowercase_voltage_level_id_alias(self): + env = MagicMock() + env.action_space.side_effect = lambda c: ("ACTION", c) + dict_action = { + "node_VL_A": {"voltage_level_id": "VL_A", + "content": {"set_bus": {"switches": ["s1"]}}}, + } + prioritized, _ = ToOpRecommender()._materialise_busbar_actions( + splits=[("VL_A", 0.0)], env=env, dict_action=dict_action, + ) + assert "node_VL_A" in prioritized + + def test_empty_dict_action_yields_empty(self): + env = MagicMock() + prioritized, scores = ToOpRecommender()._materialise_busbar_actions( + splits=[("VL_A", 0.0)], env=env, dict_action={}, + ) + assert prioritized == {} and scores == {} + + def test_skips_entries_with_no_content(self): + env = MagicMock() + env.action_space.side_effect = lambda c: ("ACTION", c) + dict_action = { + "lazy_VL_A": {"VoltageLevelId": "VL_A", "content": None}, + } + prioritized, _ = ToOpRecommender()._materialise_busbar_actions( + splits=[("VL_A", 0.0)], env=env, dict_action=dict_action, + ) + assert prioritized == {} + + def test_skips_entries_env_rejects(self): + env = MagicMock() + env.action_space.side_effect = RuntimeError("rejected") + dict_action = { + "split_VL_A_v1": {"VoltageLevelId": "VL_A", "content": {"set_bus": {"switches": ["s1"]}}}, + } + prioritized, _ = ToOpRecommender()._materialise_busbar_actions( + splits=[("VL_A", 0.0)], env=env, dict_action=dict_action, + ) + assert prioritized == {} + + def test_no_matching_VL_logs_gap_and_returns_empty(self, caplog): + env = MagicMock() + dict_action = { + "split_VL_A_v1": {"VoltageLevelId": "VL_A", "content": {"set_bus": {"switches": ["s1"]}}}, + } + with caplog.at_level("WARNING"): + prioritized, _ = ToOpRecommender()._materialise_busbar_actions( + splits=[("VL_GHOST", 0.0)], env=env, dict_action=dict_action, + ) + assert prioritized == {} + # The "no dict_action entry" warning is the operator-actionable signal. + assert any("no dict_action entry" in r.getMessage() for r in caplog.records) + + # --------------------------------------------------------------------- # Light end-to-end with everything mocked — the happy path # --------------------------------------------------------------------- From ceff627643037066f68af8b461e41e46dd828d95 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 19 May 2026 20:12:45 +0000 Subject: [PATCH 08/14] fix(recommenders/toop): synthesize switch-based actions from ToOp busbar splits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous busbar implementation piggy-backed on the operator's dict_action vocabulary (matching by VoltageLevelId), which left empty-handed any grid whose action library lacked split_ entries. The pypowsybl backend in expert_op4grid_recommender actually accepts switch-action content directly — same shape as the operator-curated coupling actions in data/action_space/reduced_model_actions_test.json: { "description": ..., "VoltageLevelId": "", "switches": {"_": true|false, ...}, "content": None, # populated by enrich_actions_lazy } (`true` = open, `false` = closed; switch ids are VL-prefixed.) Rewrite extracts and surface accordingly: - `_extract_busbar_splits_from_topology_paths` now returns `[(vl_id, {switch_id: new_open_state}, rank)]` triples — capturing *which* switches flipped and to *what* state, not just the affected VL. De-duplicated by VL keeping the best-rank topology's switch set. - `_materialise_busbar_actions` synthesises one action per VL ToOp split. For each, it builds a raw dict_action entry with the required `VoltageLevelId` + `switches` fields, runs the entry through `enrich_actions_lazy(raw, network)` to populate `content` (per-connectable bus assignments) from the live network, then hands the populated content to `env.action_space`. The result is a fully materialised pypowsybl-backend action that shows up in the React feed as `toop_split_` with a description listing the switch operations. - Defensive VL-prefix handling: switch ids already prefixed with `_` aren't double-prefixed (covers the case where pypowsybl exposes switches with VL-qualified ids on some grids). - The whole synth pipeline is no-op when `enrich_actions_lazy` isn't importable, or when it raises, or when it doesn't populate `content`. Each branch logs a clear, operator-actionable warning; no silent empty surface. Tests updated for the new signature and behaviour. Six fresh cases cover: synthesis via mocked enrich, already-prefixed switch ids, empty switches skip, enrich failure → empty, unpopulated content warned + skipped, and env-rejection skipped. --- expert_backend/recommenders/toop.py | 191 ++++++++++++------ expert_backend/tests/test_toop_recommender.py | 152 +++++++++----- 2 files changed, 229 insertions(+), 114 deletions(-) diff --git a/expert_backend/recommenders/toop.py b/expert_backend/recommenders/toop.py index f8a64c98..905377b1 100644 --- a/expert_backend/recommenders/toop.py +++ b/expert_backend/recommenders/toop.py @@ -285,7 +285,7 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu bs_prioritized, bs_scores = self._materialise_busbar_actions( splits=splits, env=env, - dict_action=inputs.dict_action, + network=inputs.network, ) # Line-switch matches win on key collisions — they're the # more specific surface form. In practice the two namespaces @@ -476,8 +476,8 @@ def _extract_busbar_splits_from_topology_paths( self, topology_paths: Iterable[Any], original_grid_file: Path, - ) -> List[Tuple[str, float]]: - """Return ``[(voltage_level_id, rank_score)]`` for VLs ToOp split. + ) -> List[Tuple[str, Dict[str, bool], float]]: + """Return ``[(vl_id, {switch_id: new_open}, rank_score)]`` triples. Diffs each topology's switch states (via ``Network.get_switches()``) against the original grid. A @@ -486,9 +486,13 @@ def _extract_busbar_splits_from_topology_paths( line-switch extraction missed it — is a *busbar split*: ToOp reconfigured the node-breaker topology inside a substation. - Returns the list sorted best-first (rank 0 = topology_0). De- - duplicated by VL so multiple topologies touching the same VL - keep the best rank. + Returns one triple per affected VL, sorted best-first (rank 0 + = ``topology_0``). The switch dict captures *which* switches + flipped and to *which* state — the payload `_materialise_busbar_actions` + needs to synthesise a switch-based action in the exact shape + the pypowsybl backend (and operator-curated coupling actions) + consume. De-duplicated by VL keeping the best-rank topology's + switch set. """ import pypowsybl.network as pn @@ -519,7 +523,8 @@ def _extract_busbar_splits_from_topology_paths( except Exception: continue - best_rank_by_vl: Dict[str, float] = {} + # vl_id -> (best_rank_so_far, {switch_id: new_open_state}) + best: Dict[str, Tuple[float, Dict[str, bool]]] = {} for rank, topo in enumerate(topology_paths or []): topo_dir = Path(topo) if not isinstance(topo, Path) else topo modified = topo_dir / "modified_network.xiidm" @@ -535,6 +540,7 @@ def _extract_busbar_splits_from_topology_paths( ) continue + per_vl: Dict[str, Dict[str, bool]] = {} for sid in mod_switches.index: if sid not in orig_open: continue @@ -547,76 +553,137 @@ def _extract_busbar_splits_from_topology_paths( vl = orig_vl.get(sid) if not vl: continue - cur = best_rank_by_vl.get(vl) - if cur is None or rank < cur: - best_rank_by_vl[vl] = float(rank) + per_vl.setdefault(vl, {})[sid] = new_open + + for vl, switches in per_vl.items(): + cur = best.get(vl) + if cur is None or rank < cur[0]: + best[vl] = (float(rank), switches) - return sorted(best_rank_by_vl.items(), key=lambda kv: kv[1]) + return sorted( + [(vl, switches, rank) for vl, (rank, switches) in best.items()], + key=lambda triple: triple[2], + ) def _materialise_busbar_actions( self, - splits: List[Tuple[str, float]], + splits: List[Tuple[str, Dict[str, bool], float]], env: Any, - dict_action: Optional[Dict[str, Any]], + network: Any, ) -> Tuple[Dict[str, Any], Dict[str, float]]: - """Surface every ``dict_action`` entry whose VL ToOp split. - - We piggy-back on the operator's substation-action vocabulary - rather than synthesising one from ToOp's raw switch list: the - translation from a switch-flip set to grid2op's set-bus - topology vector requires substation-specific knowledge the - operator already encoded when curating ``dict_action``. ToOp - tells us *which* substation to split; the operator's library - tells us *how*. - - When a VL ToOp split has no matching dict_action entry, we - log it — that's a gap for the operator to fill, not a bug. + """Synthesize switch-based actions from ToOp's split decisions. + + The pypowsybl backend in ``expert_op4grid_recommender`` accepts + the same switch-action format the operator's curated coupling + actions use: top-level ``VoltageLevelId`` + ``switches`` dict + mapping ``_`` → ``true`` (open) / ``false`` + (closed). The ``content`` field can be left ``None`` and is + populated by ``enrich_actions_lazy``, which reads the live + network to compute the resulting per-connectable bus + assignments. + + We build one synthetic action per VL ToOp split (grouping all + switches it flipped in that VL), enrich, and hand the + materialised content to ``env.action_space``. Score is ToOp's + topology rank, negated so "higher is better" matches the UI's + sort convention. """ + if not splits: + return {}, {} + + try: + from expert_op4grid_recommender.data_loader import enrich_actions_lazy + except Exception as exc: + logger.warning( + "ToOpRecommender: enrich_actions_lazy not importable (%s: %s); " + "cannot synthesize busbar-split actions.", + type(exc).__name__, exc, + ) + return {}, {} + + # Build a raw dict_action containing only our synthesised entries. + raw: Dict[str, Dict[str, Any]] = {} + pending: List[Tuple[str, str, float]] = [] + for vl_id, switches, rank in splits: + if not switches: + continue + prefixed = { + (sid if sid.startswith(f"{vl_id}_") else f"{vl_id}_{sid}"): bool(new_open) + for sid, new_open in switches.items() + } + action_id = f"toop_split_{vl_id}" + sample = ", ".join(sorted(prefixed.keys())[:3]) + raw[action_id] = { + "description": ( + f"ToOp: substation reconfiguration at '{vl_id}' " + f"({len(prefixed)} switch operation(s): {sample}" + f"{'…' if len(prefixed) > 3 else ''})" + ), + "description_unitaire": f"ToOp: split at {vl_id}", + "VoltageLevelId": vl_id, + "switches": prefixed, + # content=None → enrich_actions_lazy computes it on demand. + "content": None, + } + pending.append((action_id, vl_id, rank)) + + if not raw: + return {}, {} + + try: + enriched = enrich_actions_lazy(raw, network) + except Exception: + logger.exception( + "ToOpRecommender: enrich_actions_lazy failed; " + "cannot synthesize busbar-split actions." + ) + return {}, {} + prioritized: Dict[str, Any] = {} scores: Dict[str, float] = {} - if not dict_action: - for vl_id, _ in splits: - logger.warning( - "ToOpRecommender: ToOp split VL %s but dict_action is empty.", - vl_id, - ) - return prioritized, scores - - for vl_id, rank in splits: - matched: List[str] = [] - for action_id, entry in dict_action.items(): - if not isinstance(entry, dict): - continue - entry_vl = entry.get("VoltageLevelId") or entry.get("voltage_level_id") - if entry_vl != vl_id: - continue - content = entry.get("content") - if content is None: - continue + for action_id, vl_id, rank in pending: + entry = enriched.get(action_id) if hasattr(enriched, "get") else None + if not isinstance(entry, dict): + # LazyActionDict supports mapping access but not + # always .get(); fall back to direct indexing. try: - prioritized[action_id] = env.action_space(content) - except Exception as e: - logger.debug( - "ToOpRecommender: dict_action %s for VL %s rejected by env: %s", - action_id, vl_id, e, + entry = enriched[action_id] + except Exception: + logger.warning( + "ToOpRecommender: enriched dict lacks %s — skipping.", + action_id, ) continue - # Same negation convention as line switches — higher is better. - scores[action_id] = -float(rank) - matched.append(action_id) - if matched: - logger.warning( - "ToOpRecommender: VL %s split → surfacing %d matching " - "dict_action entry(s): %s", - vl_id, len(matched), matched[:5], + try: + content = entry.get("content") if isinstance(entry, dict) else None + except Exception: + logger.exception( + "ToOpRecommender: content compute failed for %s (VL %s).", + action_id, vl_id, ) - else: + continue + if content is None: logger.warning( - "ToOpRecommender: VL %s split but no dict_action entry " - "with matching VoltageLevelId; consider adding split_%s " - "to your action library.", - vl_id, vl_id, + "ToOpRecommender: enrich_actions_lazy did not populate " + "content for %s (VL %s) — backend cannot apply this split.", + action_id, vl_id, ) + continue + try: + prioritized[action_id] = env.action_space(content) + except Exception as e: + logger.debug( + "ToOpRecommender: env.action_space() rejected %s (VL %s): %s", + action_id, vl_id, e, + ) + continue + scores[action_id] = -float(rank) + logger.warning( + "ToOpRecommender: synthesized busbar action %s for VL %s " + "with %d switch operation(s).", + action_id, vl_id, len(raw[action_id]["switches"]), + ) + return prioritized, scores def _extract_switches_from_topology_paths( diff --git a/expert_backend/tests/test_toop_recommender.py b/expert_backend/tests/test_toop_recommender.py index f9a663ff..a2eb71d0 100644 --- a/expert_backend/tests/test_toop_recommender.py +++ b/expert_backend/tests/test_toop_recommender.py @@ -277,76 +277,124 @@ def test_skips_actions_env_rejects(self, network_with_lines): # Busbar-split materialisation # --------------------------------------------------------------------- class TestMaterialiseBusbarActions: - def test_surfaces_every_dict_action_entry_for_matching_VL(self): + def _make_enrich_mock(self, set_content_for: set): + """Return a fake enrich_actions_lazy that fills `content` for the listed ids.""" + def fake_enrich(raw, _network): + out = {} + for aid, entry in raw.items(): + entry = dict(entry) + if aid in set_content_for: + entry["content"] = { + "set_bus": {"lines_or_id": {}, "lines_ex_id": {}, + "loads_id": {}, "generators_id": {}, + "shunts_id": {}}, + "switches": entry["switches"], + } + # else leave content as None to exercise the "didn't populate" path + out[aid] = entry + return out + return fake_enrich + + def test_synthesizes_switch_action_via_enrich(self): env = MagicMock() env.action_space.side_effect = lambda c: ("ACTION", c) - dict_action = { - "split_VL_A_v1": {"VoltageLevelId": "VL_A", "content": {"set_bus": {"switches": ["s1"]}}}, - "split_VL_A_v2": {"VoltageLevelId": "VL_A", "content": {"set_bus": {"switches": ["s2"]}}}, - "split_VL_B_v1": {"VoltageLevelId": "VL_B", "content": {"set_bus": {"switches": ["s5"]}}}, - } - prioritized, scores = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_A", 0.0)], env=env, dict_action=dict_action, - ) - # Both VL_A variants surface; VL_B is untouched. - assert set(prioritized.keys()) == {"split_VL_A_v1", "split_VL_A_v2"} - assert "split_VL_B_v1" not in prioritized - # Score 0.0 → negated → 0.0 (higher is better). - assert scores["split_VL_A_v1"] == 0.0 - - def test_accepts_lowercase_voltage_level_id_alias(self): + network = MagicMock() + action_id = "toop_split_VL_A" + enrich = self._make_enrich_mock({action_id}) + with patch( + "expert_op4grid_recommender.data_loader.enrich_actions_lazy", + enrich, create=True, + ): + prioritized, scores = ToOpRecommender()._materialise_busbar_actions( + splits=[("VL_A", {"sw1": True, "sw2": False}, 0.0)], + env=env, network=network, + ) + assert list(prioritized.keys()) == [action_id] + assert scores[action_id] == 0.0 + # The synthesised action's switch ids are VL-prefixed. + action_tuple = prioritized[action_id] + assert action_tuple[0] == "ACTION" + assert set(action_tuple[1]["switches"]) == {"VL_A_sw1", "VL_A_sw2"} + + def test_already_prefixed_switch_id_not_double_prefixed(self): env = MagicMock() env.action_space.side_effect = lambda c: ("ACTION", c) - dict_action = { - "node_VL_A": {"voltage_level_id": "VL_A", - "content": {"set_bus": {"switches": ["s1"]}}}, - } - prioritized, _ = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_A", 0.0)], env=env, dict_action=dict_action, - ) - assert "node_VL_A" in prioritized + network = MagicMock() + action_id = "toop_split_VL_A" + enrich = self._make_enrich_mock({action_id}) + with patch( + "expert_op4grid_recommender.data_loader.enrich_actions_lazy", + enrich, create=True, + ): + prioritized, _ = ToOpRecommender()._materialise_busbar_actions( + splits=[("VL_A", {"VL_A_sw1": True}, 0.0)], + env=env, network=network, + ) + action_tuple = prioritized[action_id] + # Only one prefix, not VL_A_VL_A_sw1. + assert set(action_tuple[1]["switches"]) == {"VL_A_sw1"} - def test_empty_dict_action_yields_empty(self): + def test_empty_switches_dict_skipped(self): env = MagicMock() - prioritized, scores = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_A", 0.0)], env=env, dict_action={}, - ) + network = MagicMock() + with patch( + "expert_op4grid_recommender.data_loader.enrich_actions_lazy", + self._make_enrich_mock(set()), create=True, + ): + prioritized, scores = ToOpRecommender()._materialise_busbar_actions( + splits=[("VL_A", {}, 0.0)], + env=env, network=network, + ) assert prioritized == {} and scores == {} - def test_skips_entries_with_no_content(self): + def test_enrich_failure_yields_empty(self): env = MagicMock() - env.action_space.side_effect = lambda c: ("ACTION", c) - dict_action = { - "lazy_VL_A": {"VoltageLevelId": "VL_A", "content": None}, - } - prioritized, _ = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_A", 0.0)], env=env, dict_action=dict_action, - ) + network = MagicMock() + def boom(_raw, _network): + raise RuntimeError("enrich blew up") + with patch( + "expert_op4grid_recommender.data_loader.enrich_actions_lazy", + boom, create=True, + ): + prioritized, _ = ToOpRecommender()._materialise_busbar_actions( + splits=[("VL_A", {"sw1": True}, 0.0)], + env=env, network=network, + ) assert prioritized == {} - def test_skips_entries_env_rejects(self): + def test_unpopulated_content_warned_and_skipped(self, caplog): env = MagicMock() - env.action_space.side_effect = RuntimeError("rejected") - dict_action = { - "split_VL_A_v1": {"VoltageLevelId": "VL_A", "content": {"set_bus": {"switches": ["s1"]}}}, - } - prioritized, _ = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_A", 0.0)], env=env, dict_action=dict_action, - ) + network = MagicMock() + # enrich returns an entry whose content stays None (not populated). + with patch( + "expert_op4grid_recommender.data_loader.enrich_actions_lazy", + self._make_enrich_mock(set()), create=True, + ): + with caplog.at_level("WARNING"): + prioritized, _ = ToOpRecommender()._materialise_busbar_actions( + splits=[("VL_A", {"sw1": True}, 0.0)], + env=env, network=network, + ) assert prioritized == {} + assert any( + "did not populate content" in r.getMessage() + for r in caplog.records + ) - def test_no_matching_VL_logs_gap_and_returns_empty(self, caplog): + def test_env_rejection_skips_action(self): env = MagicMock() - dict_action = { - "split_VL_A_v1": {"VoltageLevelId": "VL_A", "content": {"set_bus": {"switches": ["s1"]}}}, - } - with caplog.at_level("WARNING"): + env.action_space.side_effect = RuntimeError("backend rejected") + network = MagicMock() + action_id = "toop_split_VL_A" + with patch( + "expert_op4grid_recommender.data_loader.enrich_actions_lazy", + self._make_enrich_mock({action_id}), create=True, + ): prioritized, _ = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_GHOST", 0.0)], env=env, dict_action=dict_action, + splits=[("VL_A", {"sw1": True}, 0.0)], + env=env, network=network, ) assert prioritized == {} - # The "no dict_action entry" warning is the operator-actionable signal. - assert any("no dict_action entry" in r.getMessage() for r in caplog.records) # --------------------------------------------------------------------- From 2a198fd695a1ce2d61009617abaaf6cb3f2bf71f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 1 Jun 2026 16:09:35 +0000 Subject: [PATCH 09/14] feat(recommenders/toop): one combined-action card per candidate topology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator feedback: ToOp optimises *whole topologies*, and its elementary moves are only meaningful together — the prior approach (flattening each topology into independent disco_*/split cards) surfaced suggestions that look neutral or worse in isolation, hiding the real combined congestion relief (e.g. f0=-61.4 → f=-42.3 on the test grid). The feed showed ten weak singles instead of the handful of strong topologies ToOp actually found. Rework so each ToOp candidate topology becomes ONE combined action: - `_build_topology_actions`: per topology directory, diff modified_network.xiidm against the input grid along BOTH axes (line connection flags + per-VL internal switch states) and fold every change into one merged action content + one grid2op action object. Returned as a single prioritized action (`toop_topology_`), so the step-2 assessment phase really simulates the whole combination → the card's max_rho is the true combined loading ToOp optimised, not its parts in isolation. - `_merge_topology_content`: enriches each VL switch set via `enrich_actions_lazy` (per-connectable set_bus resolution), unions all set_bus + switches across VLs, then applies line toggles last (explicit open/close wins over a split's bus assignment for the same branch). Returns the merged content + human-readable constituent labels for the card. - `_service_integration`: after assessment, reformat each topology into one `combined_actions` entry (the channel selected by the operator) carrying the real simulated max_rho (is_simulated=True), the full `constituent_ids` list (+ is_toop_topology flag), and legacy action1_id/action2_id (first two constituents) so the existing 2-up pairs table still renders before the N-way UI lands. Injects the merged contents into `_dict_action` so each card stays re-simulatable / session-saveable. Topology entries are popped from the main feed → topologies-only, in the combined channel. Removed the now-dead per-line / per-VL flattening extractors and the undocumented Pareto-JSON parsers. Tests rewritten around the per-topology builder + content merge (21 cases); the step-2 / combined-action integration suite (46 cases) stays green. Frontend N-way combined-action rendering is the follow-up commit. --- .../recommenders/_service_integration.py | 49 + expert_backend/recommenders/toop.py | 892 ++++++------------ expert_backend/tests/test_toop_recommender.py | 598 ++++++------ 3 files changed, 609 insertions(+), 930 deletions(-) diff --git a/expert_backend/recommenders/_service_integration.py b/expert_backend/recommenders/_service_integration.py index bacccd4f..adf99ff2 100644 --- a/expert_backend/recommenders/_service_integration.py +++ b/expert_backend/recommenders/_service_integration.py @@ -279,6 +279,55 @@ def _run_analysis_step2_with_model( action_scores = self._compute_mw_start_for_scores( results.get("action_scores", {}) ) + + # ToOp: reformat each candidate topology into ONE combined-action + # card. Each topology was returned as a single merged prioritized + # action and REALLY simulated by the assessment phase above, so + # ``enriched_actions`` already carries its true combined max_rho. + # Move those entries out of the main feed and into + # ``combined_actions`` (the channel the operator selected), and + # inject the merged contents into ``_dict_action`` so each card + # stays re-simulatable / session-saveable. + topo_groups = getattr(recommender, "_last_topology_groups", None) + if topo_groups: + combined = results.get("combined_actions", {}) or {} + topo_entries = getattr(recommender, "_last_topology_dict_entries", {}) or {} + if self._dict_action is None: + self._dict_action = {} + self._dict_action.update(topo_entries) + for grp in topo_groups: + tid = grp["topology_id"] + enriched = enriched_actions.pop(tid, None) + if enriched is None: + continue + constituents = grp.get("constituents", []) or [] + combined[tid] = { + # N-way combination metadata. action1_id/action2_id are + # kept (first two constituents) so the legacy 2-up pairs + # table still renders something before the N-way UI lands. + "constituent_ids": constituents, + "constituent_count": len(constituents), + "action1_id": constituents[0] if constituents else "", + "action2_id": constituents[1] if len(constituents) > 1 else "", + "description": "ToOp: " + ", ".join(constituents), + "betas": [], + "is_toop_topology": True, + "rank": grp.get("rank"), + # Real simulation results carried from the assessment. + "max_rho": enriched.get("max_rho"), + "max_rho_line": enriched.get("max_rho_line"), + "target_max_rho": enriched.get("max_rho"), + "target_max_rho_line": enriched.get("max_rho_line"), + "simulated_max_rho": enriched.get("max_rho"), + "simulated_max_rho_line": enriched.get("max_rho_line"), + "is_simulated": True, + "is_rho_reduction": enriched.get("is_rho_reduction"), + "rho_before": enriched.get("rho_before"), + "rho_after": enriched.get("rho_after"), + "lines_overloaded_after": enriched.get("lines_overloaded_after"), + } + results["combined_actions"] = combined + enrichment_time = time.time() - _t_enrich logger.info( diff --git a/expert_backend/recommenders/toop.py b/expert_backend/recommenders/toop.py index 905377b1..f32e74ac 100644 --- a/expert_backend/recommenders/toop.py +++ b/expert_backend/recommenders/toop.py @@ -5,41 +5,45 @@ # SPDX-License-Identifier: MPL-2.0 """Elia Group `ToOp `_ topology optimizer. -MVP scope — **line switching only**. ToOp's broader output set -(busbar splits, busbar reassignments) is not yet translated into -remedial actions; that extension lives downstream once the line-switch -path is exercised on real grids. - -ToOp is treated as an **optional install**. The Python package -``toop_engine_topology_optimizer`` pins Python 3.11 and pulls in heavy -GPU dependencies (JAX, qdax, Ray, …), so we never import it at module -load time — the import happens lazily inside :meth:`recommend` and a -missing install is reported as an empty recommendation with a clear -log line rather than a server crash. - -Integration outline (see comments in ``recommend`` for details): - -1. Export ``inputs.network`` (pypowsybl Network) to a temporary - CGMES bundle — ToOp's importer pipeline ingests CGMES / UCT, not - XIIDM. -2. Build the ``DictConfig`` quadruple ToOp's ``run_pipeline`` expects - (DC optimization + AC validation + importer + preprocessing), - constrained to line-status edits. -3. Run ``run_pipeline`` synchronously inside the streaming step-2 - endpoint, bounded by ``runtime_seconds``. -4. Parse the Pareto front for line-switching decisions and translate - each to the Co-Study4Grid action format - (``{"set_bus": {"lines_or_id": {line: ±1}, "lines_ex_id": {line: ±1}}}``) - via ``env.action_space``. -5. Rank by ToOp's congestion metric (``overload_energy_n_1``); surface - the top-N as ``prioritized_actions`` with ``action_scores``. +ToOp is a Map-Elites topology optimiser: it searches for *whole network +topologies* (combinations of line switching + busbar splits across many +substations) that relieve N-1 congestion. The elementary moves are only +meaningful together — a single split that looks neutral in isolation can +be decisive inside the combined topology ToOp actually optimised. + +This integration therefore treats **each ToOp candidate topology as one +combined action**, not as a bag of independent suggestions: + +1. Export the live pypowsybl Network to XIIDM (ToOp ingests it directly). +2. Run ToOp's ``run_pipeline`` (preprocessing → DC Map-Elites → AC + validation). It returns a list of *topology directories*, each with a + ``modified_network.xiidm`` — the full target network state, best-first. +3. For each topology, diff ``modified_network.xiidm`` against the input + grid along two axes — line connection flags and per-VL internal switch + states — and fold **every** change into ONE merged action content + (``set_bus`` assignments + ``switches`` dict, the same format the + operator's curated coupling actions use; see + ``data/action_space/reduced_model_actions_test.json``). +4. Return one merged grid2op action per topology under a clean id + (``toop_topology_``). The step-2 assessment phase REALLY + simulates each, so the resulting ``max_rho`` is the true combined + loading of the whole topology. +5. ``_service_integration`` then reformats each simulated topology into a + single combined-action card (the ``combined_actions`` channel) and + injects the merged contents into the service ``_dict_action`` so the + card stays re-simulatable / session-saveable. + +ToOp is an **optional install** (Python 3.11 + heavy GPU deps). The +class only lazy-imports ``toop_engine_topology_optimizer`` inside +``recommend()``; a missing install yields an empty recommendation with a +clear log line instead of crashing the step-2 NDJSON stream. """ from __future__ import annotations import logging import tempfile from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from expert_op4grid_recommender.models.base import ( ParamSpec, @@ -48,10 +52,6 @@ RecommenderOutput, ) -from expert_backend.recommenders.network_existence import ( - filter_to_existing_network_elements, -) - logger = logging.getLogger(__name__) @@ -93,8 +93,22 @@ def _import_dictconfig() -> Optional[Any]: return DictConfig +def _import_enrich() -> Optional[Any]: + """Lazy importer for ``enrich_actions_lazy`` (populates switch-action content).""" + try: + from expert_op4grid_recommender.data_loader import enrich_actions_lazy + except Exception as exc: + logger.warning( + "ToOpRecommender: enrich_actions_lazy not importable (%s: %s); " + "cannot materialise busbar-split content.", + type(exc).__name__, exc, + ) + return None + return enrich_actions_lazy + + class ToOpRecommender(RecommenderModel): - """Topology optimizer wrapper. MVP surfaces line-switching only. + """Topology optimizer wrapper — one combined action per candidate topology. The model does NOT consume the overflow graph — ToOp does its own DC contingency analysis internally, so we set @@ -112,15 +126,14 @@ def params_spec(cls) -> List[ParamSpec]: return [ ParamSpec( "n_prioritized_actions", - "N Prioritized Actions", + "N Candidate Topologies", "int", default=5, min=1, max=50, description=( - "Maximum number of suggestions surfaced — covers " - "both line switches and busbar splits, ranked by " - "ToOp's congestion-reduction order." + "Maximum number of ToOp candidate topologies surfaced " + "as combined-action cards (best-first by ToOp's rank)." ), ), ParamSpec( @@ -129,9 +142,9 @@ def params_spec(cls) -> List[ParamSpec]: "bool", default=True, description=( - "Surface dict_action entries whose VoltageLevelId " - "matches a substation ToOp split. Disable to " - "restrict suggestions to line switches only." + "Fold ToOp's internal substation switch changes into " + "each topology. Disable to keep only line-switching " + "changes in the combined action." ), ), ParamSpec( @@ -143,7 +156,7 @@ def params_spec(cls) -> List[ParamSpec]: max=120, description=( "Wall-clock budget passed to ToOp's DC optimization " - "stage. Higher = better Pareto coverage but blocks " + "stage. Higher = better topology coverage but blocks " "the step-2 stream that long." ), ), @@ -165,6 +178,11 @@ def params_spec(cls) -> List[ParamSpec]: # Public entry point # ------------------------------------------------------------------ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutput: + # Cleared up-front so a degraded return never leaves a stale + # grouping behind for the service integration to consume. + self._last_topology_groups: List[dict] = [] + self._last_topology_dict_entries: Dict[str, Any] = {} + logger.warning( "ToOpRecommender: recommend() entry — params=%s, env=%s, network=%s, " "dict_action_size=%s", @@ -228,83 +246,34 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu # `run_pipeline` returns ``topology_paths`` — a list of # *directories* (``/run_*/topology_*``), each - # containing a ``modified_network.xiidm`` ToOp wrote during - # AC validation. The primary extraction path is therefore - # to load that modified network back through pypowsybl and - # diff its line statuses against the original grid we - # exported a few lines above — no need to reverse-engineer - # ToOp's internal Pareto-front schema. The richer - # ``optimizer_snapshot/run_0/res.json`` exists too but its - # genome encoding is undocumented; the network-diff path - # is the robust choice for the line-switching MVP. + # holding the full target ``modified_network.xiidm``, best-first. topology_list = pareto if isinstance(pareto, (list, tuple)) else [] logger.warning( "ToOpRecommender: run_pipeline returned %d topology path(s); " - "diffing modified_network.xiidm against the input grid.", + "building one combined action per topology.", len(topology_list), ) - # 1. Line switches (lines whose end-connection flags changed). - switches = self._extract_switches_from_topology_paths( + # Built inside the tempdir so the modified_network.xiidm files + # are still on disk while we diff them. + prioritized, scores, groups, dict_entries = self._build_topology_actions( topology_paths=topology_list, original_grid_file=grid_file, - n=n, - ) - # 2. Busbar splits (voltage levels whose internal switches - # changed without any line being toggled). Detected - # inside the tempdir so the modified_network.xiidm files - # are still on disk. - if include_busbar_splits: - splits = self._extract_busbar_splits_from_topology_paths( - topology_paths=topology_list, - original_grid_file=grid_file, - ) - else: - splits = [] - - logger.warning( - "ToOpRecommender: extracted %d line-switch suggestion(s), " - "%d busbar-split voltage level(s)", - len(switches), len(splits), - ) - - prioritized: Dict[str, Any] = {} - scores: Dict[str, float] = {} - - if switches: - ls_prioritized, ls_scores = self._materialise_actions( - switches=switches, env=env, - dict_action=inputs.dict_action, - non_connected_reconnectable_lines=inputs.non_connected_reconnectable_lines, network=inputs.network, + include_busbar_splits=include_busbar_splits, + n=n, ) - prioritized.update(ls_prioritized) - scores.update(ls_scores) - if splits: - bs_prioritized, bs_scores = self._materialise_busbar_actions( - splits=splits, - env=env, - network=inputs.network, - ) - # Line-switch matches win on key collisions — they're the - # more specific surface form. In practice the two namespaces - # don't overlap (line ids vs node-action ids), but be safe. - for aid, action in bs_prioritized.items(): - if aid not in prioritized: - prioritized[aid] = action - scores[aid] = bs_scores[aid] - - # Cap the merged surfaced list at the requested top-N to keep - # the UI feed manageable. Stable sort by score ascending (better - # ToOp ranks first, since we negate the score before exposing). - if len(prioritized) > n: - ranked = sorted(prioritized.keys(), key=lambda a: -scores.get(a, 0.0))[:n] - prioritized = {a: prioritized[a] for a in ranked} - scores = {a: scores[a] for a in ranked} + # Stash the topology groupings + synthesised dict_action entries so + # the model-aware step-2 integration can (a) inject the combined + # contents into the service's _dict_action for re-simulation / + # session save, and (b) reformat each topology into a single + # combined-action card after the assessment phase has simulated it. + self._last_topology_groups = groups + self._last_topology_dict_entries = dict_entries logger.warning( - "ToOpRecommender: returning %d prioritized action(s): %s", + "ToOpRecommender: returning %d topology action(s): %s", len(prioritized), list(prioritized.keys()), ) return RecommenderOutput( @@ -313,7 +282,7 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu ) # ------------------------------------------------------------------ - # Internals (kept as methods so tests can patch them individually) + # Network export + ToOp invocation # ------------------------------------------------------------------ def _export_network(self, network: Any, iteration_dir: Path) -> Optional[Path]: """Save the live pypowsybl Network as XIIDM under ``iteration_dir``. @@ -365,11 +334,11 @@ def _run_toop( ) -> Any: """Invoke ToOp's ``run_pipeline`` against an XIIDM grid file. - Mirrors the structure of ``notebooks/example2_small_grid_toop.ipynb`` - but constrains the Map Elites search to a single - ``branch_switches`` descriptor so the optimiser prioritises the - line-switching axis (busbar splits are out of scope for this - MVP). + Mirrors the structure of ``notebooks/example2_small_grid_toop.ipynb``. + The Map Elites diversity axis is ``disconnected_branches`` so the + search spreads candidates over the number of branches each + topology opens; busbar splits emerge from the same search and are + folded into the per-topology action downstream. """ # Lazy imports — these symbols only exist when ToOp is installed. from toop_engine_topology_optimizer.benchmark.benchmark_utils import ( @@ -417,12 +386,10 @@ def _run_toop( "lf_config": {"distributed": False}, "ga_config": { "runtime_seconds": runtime_seconds, - # `disconnected_branches` is ToOp's metric for the count - # of opened branches in a topology — the right axis for - # a line-switching-only Map Elites search. ToOp's - # accepted metric names are pinned in - # BatchedMEParameters; if it doesn't match the enum, - # the optimiser fails with a pydantic ValidationError. + # `disconnected_branches` is ToOp's accepted metric name + # for the count of opened branches in a topology (the enum + # is pinned in BatchedMEParameters; a wrong name fails with + # a pydantic ValidationError). "me_descriptors": [{"metric": "disconnected_branches", "num_cells": 4}], "observed_metrics": ["overload_energy_n_1", "disconnected_branches"], "n_worst_contingencies": n_worst_contingencies, @@ -446,485 +413,266 @@ def _run_toop( preprocessing_parameters=preprocessing_parameters, ) - def _load_output_json(self, path: Path) -> Any: - """Read ToOp's DC-optimisation output JSON and return its content. - - Format is currently treated as opaque: the parser downstream - (:meth:`_extract_line_switches`) consumes several plausible - shapes, so we just hand the decoded payload over and log a - sample of the top-level keys for forensic purposes. - """ - import json - - with path.open("r", encoding="utf-8") as fh: - data = json.load(fh) - if isinstance(data, dict): - logger.warning( - "ToOpRecommender: output.json top-level keys: %s", - sorted(data.keys())[:20], - ) - elif isinstance(data, list): - logger.warning( - "ToOpRecommender: output.json is a list of %d entries; " - "first entry keys: %s", - len(data), - sorted(data[0].keys())[:20] if data and isinstance(data[0], dict) else None, - ) - return data - - def _extract_busbar_splits_from_topology_paths( + # ------------------------------------------------------------------ + # Per-topology combined-action construction + # ------------------------------------------------------------------ + def _build_topology_actions( self, - topology_paths: Iterable[Any], + topology_paths: List[Any], original_grid_file: Path, - ) -> List[Tuple[str, Dict[str, bool], float]]: - """Return ``[(vl_id, {switch_id: new_open}, rank_score)]`` triples. - - Diffs each topology's switch states (via - ``Network.get_switches()``) against the original grid. A - voltage level whose internal switches changed open/closed - state — but whose line terminal connections did NOT, so - line-switch extraction missed it — is a *busbar split*: ToOp - reconfigured the node-breaker topology inside a substation. - - Returns one triple per affected VL, sorted best-first (rank 0 - = ``topology_0``). The switch dict captures *which* switches - flipped and to *which* state — the payload `_materialise_busbar_actions` - needs to synthesise a switch-based action in the exact shape - the pypowsybl backend (and operator-curated coupling actions) - consume. De-duplicated by VL keeping the best-rank topology's - switch set. + env: Any, + network: Any, + include_busbar_splits: bool, + n: int, + ) -> Tuple[Dict[str, Any], Dict[str, float], List[dict], Dict[str, Any]]: + """Build one combined grid2op action per ToOp topology. + + Each topology directory holds a ``modified_network.xiidm`` — the + full target state. We diff it against the input grid along two + axes (line connection flags + per-VL internal switch states) and + fold every change into ONE merged action content + one grid2op + action object. Simulated as a whole by the assessment phase, that + single object yields the *true* combined ``max_rho`` ToOp + optimised — not the misleading effect of its parts in isolation. + + Returns ``(prioritized, scores, groups, dict_entries)``: + + - ``prioritized`` ``{topology_id: action_object}`` + - ``scores`` ``{topology_id: -rank}`` (UI sorts higher-first) + - ``groups`` ``[{topology_id, constituents, rank, + line_count, switch_vls}]`` — consumed by the service + integration to build the combined-action card. + - ``dict_entries`` ``{topology_id: {content, switches, …}}`` — + injected into the service ``_dict_action`` for re-simulation. """ import pypowsybl.network as pn try: original = pn.load(str(original_grid_file)) + orig_lines = original.get_lines() orig_switches = original.get_switches() except Exception: logger.exception( - "ToOpRecommender: cannot read switches from original grid %s", + "ToOpRecommender: cannot reload original grid %s for diff.", original_grid_file, ) - return [] + return {}, {}, [], {} - if "open" not in orig_switches.columns or "voltage_level_id" not in orig_switches.columns: + orig_line_open = {lid: _is_line_open(orig_lines, lid) for lid in orig_lines.index} + has_switch_cols = ( + "open" in orig_switches.columns and "voltage_level_id" in orig_switches.columns + ) + orig_sw_open: Dict[str, bool] = {} + orig_sw_vl: Dict[str, str] = {} + if has_switch_cols: + for sid in orig_switches.index: + try: + orig_sw_open[sid] = bool(orig_switches.at[sid, "open"]) + orig_sw_vl[sid] = str(orig_switches.at[sid, "voltage_level_id"]) + except Exception: + continue + elif include_busbar_splits: logger.warning( - "ToOpRecommender: pypowsybl get_switches() lacks expected " - "columns (have %s); skipping busbar-split detection.", + "ToOpRecommender: get_switches() lacks open/voltage_level_id " + "columns (have %s); busbar splits will be omitted.", list(orig_switches.columns), ) - return [] - orig_open: Dict[str, bool] = {} - orig_vl: Dict[str, str] = {} - for sid in orig_switches.index: - try: - orig_open[sid] = bool(orig_switches.at[sid, "open"]) - orig_vl[sid] = str(orig_switches.at[sid, "voltage_level_id"]) - except Exception: - continue + enrich = _import_enrich() + + prioritized: Dict[str, Any] = {} + scores: Dict[str, float] = {} + groups: List[dict] = [] + dict_entries: Dict[str, Any] = {} - # vl_id -> (best_rank_so_far, {switch_id: new_open_state}) - best: Dict[str, Tuple[float, Dict[str, bool]]] = {} for rank, topo in enumerate(topology_paths or []): + if len(prioritized) >= n: + break topo_dir = Path(topo) if not isinstance(topo, Path) else topo modified = topo_dir / "modified_network.xiidm" if not modified.exists(): + logger.warning("ToOpRecommender: no modified_network.xiidm under %s", topo_dir) continue try: mod_net = pn.load(str(modified)) - mod_switches = mod_net.get_switches() + mod_lines = mod_net.get_lines() + mod_switches = mod_net.get_switches() if has_switch_cols else None except Exception: - logger.exception( - "ToOpRecommender: cannot read switches from %s", - modified, - ) + logger.exception("ToOpRecommender: cannot load %s for diff.", modified) continue - per_vl: Dict[str, Dict[str, bool]] = {} - for sid in mod_switches.index: - if sid not in orig_open: + # --- line toggles --- + line_toggles: Dict[str, int] = {} + for lid in mod_lines.index: + if lid not in orig_line_open: continue - try: - new_open = bool(mod_switches.at[sid, "open"]) - except Exception: - continue - if new_open == orig_open[sid]: - continue - vl = orig_vl.get(sid) - if not vl: - continue - per_vl.setdefault(vl, {})[sid] = new_open - - for vl, switches in per_vl.items(): - cur = best.get(vl) - if cur is None or rank < cur[0]: - best[vl] = (float(rank), switches) - - return sorted( - [(vl, switches, rank) for vl, (rank, switches) in best.items()], - key=lambda triple: triple[2], - ) - - def _materialise_busbar_actions( - self, - splits: List[Tuple[str, Dict[str, bool], float]], - env: Any, - network: Any, - ) -> Tuple[Dict[str, Any], Dict[str, float]]: - """Synthesize switch-based actions from ToOp's split decisions. - - The pypowsybl backend in ``expert_op4grid_recommender`` accepts - the same switch-action format the operator's curated coupling - actions use: top-level ``VoltageLevelId`` + ``switches`` dict - mapping ``_`` → ``true`` (open) / ``false`` - (closed). The ``content`` field can be left ``None`` and is - populated by ``enrich_actions_lazy``, which reads the live - network to compute the resulting per-connectable bus - assignments. - - We build one synthetic action per VL ToOp split (grouping all - switches it flipped in that VL), enrich, and hand the - materialised content to ``env.action_space``. Score is ToOp's - topology rank, negated so "higher is better" matches the UI's - sort convention. - """ - if not splits: - return {}, {} - - try: - from expert_op4grid_recommender.data_loader import enrich_actions_lazy - except Exception as exc: - logger.warning( - "ToOpRecommender: enrich_actions_lazy not importable (%s: %s); " - "cannot synthesize busbar-split actions.", - type(exc).__name__, exc, - ) - return {}, {} - - # Build a raw dict_action containing only our synthesised entries. - raw: Dict[str, Dict[str, Any]] = {} - pending: List[Tuple[str, str, float]] = [] - for vl_id, switches, rank in splits: - if not switches: + new_open = _is_line_open(mod_lines, lid) + if new_open != orig_line_open[lid]: + line_toggles[lid] = -1 if new_open else 1 + + # --- switch toggles grouped by VL (busbar splits) --- + vl_switches: Dict[str, Dict[str, bool]] = {} + if include_busbar_splits and mod_switches is not None: + for sid in mod_switches.index: + if sid not in orig_sw_open: + continue + try: + new_open = bool(mod_switches.at[sid, "open"]) + except Exception: + continue + if new_open == orig_sw_open[sid]: + continue + vl = orig_sw_vl.get(sid) + if not vl: + continue + key = sid if sid.startswith(f"{vl}_") else f"{vl}_{sid}" + vl_switches.setdefault(vl, {})[key] = new_open + + if not line_toggles and not vl_switches: + logger.warning( + "ToOpRecommender: topology #%d (%s) differs by 0 lines / " + "0 switches; skipping.", + rank + 1, topo_dir.name, + ) continue - prefixed = { - (sid if sid.startswith(f"{vl_id}_") else f"{vl_id}_{sid}"): bool(new_open) - for sid, new_open in switches.items() - } - action_id = f"toop_split_{vl_id}" - sample = ", ".join(sorted(prefixed.keys())[:3]) - raw[action_id] = { - "description": ( - f"ToOp: substation reconfiguration at '{vl_id}' " - f"({len(prefixed)} switch operation(s): {sample}" - f"{'…' if len(prefixed) > 3 else ''})" - ), - "description_unitaire": f"ToOp: split at {vl_id}", - "VoltageLevelId": vl_id, - "switches": prefixed, - # content=None → enrich_actions_lazy computes it on demand. - "content": None, - } - pending.append((action_id, vl_id, rank)) - - if not raw: - return {}, {} - try: - enriched = enrich_actions_lazy(raw, network) - except Exception: - logger.exception( - "ToOpRecommender: enrich_actions_lazy failed; " - "cannot synthesize busbar-split actions." + merged, constituents = self._merge_topology_content( + line_toggles=line_toggles, + vl_switches=vl_switches, + enrich=enrich, + network=network, ) - return {}, {} - - prioritized: Dict[str, Any] = {} - scores: Dict[str, float] = {} - for action_id, vl_id, rank in pending: - entry = enriched.get(action_id) if hasattr(enriched, "get") else None - if not isinstance(entry, dict): - # LazyActionDict supports mapping access but not - # always .get(); fall back to direct indexing. - try: - entry = enriched[action_id] - except Exception: - logger.warning( - "ToOpRecommender: enriched dict lacks %s — skipping.", - action_id, - ) - continue - try: - content = entry.get("content") if isinstance(entry, dict) else None - except Exception: - logger.exception( - "ToOpRecommender: content compute failed for %s (VL %s).", - action_id, vl_id, - ) - continue - if content is None: + if merged is None: logger.warning( - "ToOpRecommender: enrich_actions_lazy did not populate " - "content for %s (VL %s) — backend cannot apply this split.", - action_id, vl_id, + "ToOpRecommender: topology #%d produced no simulable content; skipping.", + rank + 1, ) continue try: - prioritized[action_id] = env.action_space(content) + action_obj = env.action_space(merged) except Exception as e: - logger.debug( - "ToOpRecommender: env.action_space() rejected %s (VL %s): %s", - action_id, vl_id, e, + logger.warning( + "ToOpRecommender: topology #%d rejected by env.action_space: %s", + rank + 1, e, ) continue - scores[action_id] = -float(rank) + + topology_id = f"toop_topology_{rank + 1}" + description = f"ToOp topology #{rank + 1}: " + "; ".join(constituents) + prioritized[topology_id] = action_obj + scores[topology_id] = -float(rank) + groups.append({ + "topology_id": topology_id, + "constituents": constituents, + "rank": rank, + "line_count": len(line_toggles), + "switch_vls": sorted(vl_switches.keys()), + }) + # Flatten all VL switch dicts into one for the dict_action entry. + flat_switches = {k: v for sw in vl_switches.values() for k, v in sw.items()} + dict_entries[topology_id] = { + "description": description, + "description_unitaire": description, + "VoltageLevelId": (sorted(vl_switches.keys())[0] if vl_switches else None), + "switches": flat_switches, + "content": merged, + } logger.warning( - "ToOpRecommender: synthesized busbar action %s for VL %s " - "with %d switch operation(s).", - action_id, vl_id, len(raw[action_id]["switches"]), + "ToOpRecommender: topology #%d → %s (%d line toggle(s), %d VL split(s))", + rank + 1, topology_id, len(line_toggles), len(vl_switches), ) - return prioritized, scores + return prioritized, scores, groups, dict_entries - def _extract_switches_from_topology_paths( + def _merge_topology_content( self, - topology_paths: Iterable[Any], - original_grid_file: Path, - n: int, - ) -> List[Tuple[str, int, float]]: - """Diff each topology's ``modified_network.xiidm`` against the input grid. - - For every topology directory ToOp surfaces, load - ``modified_network.xiidm`` and compare per-line ``connected1`` / - ``connected2`` flags against the original exported grid. Lines - that flipped are returned as ``(line_id, target_status, score)`` - tuples where ``target_status`` is ``-1`` (opened) or ``+1`` - (closed). Score is the topology's rank in ``topology_paths`` - (ToOp returns them best-first), so the surfaced list mirrors - ToOp's own ordering. Duplicate (line, status) pairs across - topologies keep the best (lowest-rank) score. + line_toggles: Dict[str, int], + vl_switches: Dict[str, Dict[str, bool]], + enrich: Optional[Any], + network: Any, + ) -> Tuple[Optional[Dict[str, Any]], List[str]]: + """Fold line toggles + per-VL switch flips into one action content. + + The busbar splits are enriched per-VL via ``enrich_actions_lazy`` + (which reads the live network to resolve each switch set into + per-connectable ``set_bus`` assignments), then unioned into one + merged ``set_bus`` / ``switches`` payload. Line toggles are + applied last so an explicit open/close wins over a split's bus + assignment for the same branch. + + Returns ``(merged_content, constituents)``. ``merged_content`` is + ``None`` when nothing simulable could be built (every VL failed to + enrich and there were no line toggles). ``constituents`` is the + human-readable label list for the combined-action card. """ - import pypowsybl.network as pn # lazy: only loaded when ToOp ran - - try: - original = pn.load(str(original_grid_file)) - original_lines = original.get_lines() - except Exception: - logger.exception( - "ToOpRecommender: cannot reload original grid file %s for diff.", - original_grid_file, - ) - return [] - - original_open: Dict[str, bool] = {} - for line_id in original_lines.index: - original_open[line_id] = _is_line_open(original_lines, line_id) - - best: Dict[Tuple[str, int], float] = {} - for rank, topo in enumerate(topology_paths or []): - topo_dir = Path(topo) if not isinstance(topo, Path) else topo - modified = topo_dir / "modified_network.xiidm" - if not modified.exists(): - logger.warning( - "ToOpRecommender: no modified_network.xiidm under %s", - topo_dir, - ) - continue + merged: Dict[str, Any] = { + "set_bus": { + "lines_or_id": {}, "lines_ex_id": {}, + "loads_id": {}, "generators_id": {}, "shunts_id": {}, + }, + "switches": {}, + } + constituents: List[str] = [] + + if vl_switches and enrich is not None: + raw = { + f"_toop_vl_{vl_id}": { + "VoltageLevelId": vl_id, + "switches": dict(switches), + "content": None, + } + for vl_id, switches in vl_switches.items() + } try: - mod_net = pn.load(str(modified)) - mod_lines = mod_net.get_lines() + enriched = enrich(raw, network) except Exception: logger.exception( - "ToOpRecommender: cannot load %s for diff.", modified, + "ToOpRecommender: enrich_actions_lazy failed during topology merge." ) - continue - for line_id in mod_lines.index: - if line_id not in original_open: - continue - new_open = _is_line_open(mod_lines, line_id) - if new_open == original_open[line_id]: + enriched = {} + for vl_id in vl_switches: + key = f"_toop_vl_{vl_id}" + entry = None + try: + entry = enriched.get(key) if hasattr(enriched, "get") else enriched[key] + except Exception: + entry = None + content = entry.get("content") if isinstance(entry, dict) else None + if not content: + logger.warning( + "ToOpRecommender: could not enrich VL %s split (%d switch(es)); " + "excluded from this topology.", + vl_id, len(vl_switches[vl_id]), + ) continue - target_status = -1 if new_open else 1 - key = (line_id, target_status) - score = float(rank) - if key not in best or score < best[key]: - best[key] = score - - ranked = sorted(best.items(), key=lambda kv: kv[1]) - return [(lid, st, sc) for ((lid, st), sc) in ranked[:n]] - - def _extract_line_switches( - self, - pareto: Any, - n: int, - ) -> List[Tuple[str, int, float]]: - """Pull line-switching decisions out of a ToOp result. - - Returns a list of ``(line_id, target_status, score)`` tuples - sorted by ``score`` (lower congestion = better, surfaced first). - ``target_status`` is +1 (close) or -1 (open). - - The exact return shape of ``run_pipeline`` is not nailed down - by the upstream docs (the example notebooks write results to - disk via ``topo_path``), so this parser accepts several - plausible shapes: - - - An iterable of dicts each carrying ``line_switches`` (list of - ``{"line_id", "status"}`` or ``{"line_id", "open"}``) and a - numeric score (``overload_energy_n_1`` / ``score`` / ``cost``). - - An object with a ``.solutions`` attribute holding the same. - - When the shape is unrecognised we log once and return an empty - list. The frontend then shows an empty action feed and the - operator can pick a different model — far less disruptive than - raising mid-stream. - """ - candidates: Iterable[Any] - if pareto is None: - return [] - if hasattr(pareto, "solutions"): - candidates = pareto.solutions - elif isinstance(pareto, (list, tuple)): - candidates = pareto - elif hasattr(pareto, "__iter__"): - candidates = pareto - else: + set_bus = content.get("set_bus", {}) if isinstance(content, dict) else {} + for sub_key in ("lines_or_id", "lines_ex_id", "loads_id", + "generators_id", "shunts_id"): + sub = set_bus.get(sub_key) + if isinstance(sub, dict): + merged["set_bus"][sub_key].update(sub) + sw = content.get("switches", {}) if isinstance(content, dict) else {} + if isinstance(sw, dict): + merged["switches"].update(sw) + constituents.append(f"split {vl_id}") + elif vl_switches and enrich is None: logger.warning( - "ToOpRecommender: unrecognised result shape %r; " - "extend _extract_line_switches when ToOp's output is finalised.", - type(pareto).__name__, + "ToOpRecommender: enrich unavailable — %d busbar split(s) omitted " + "from this topology.", len(vl_switches), ) - return [] - - # Flatten (solution, switch) into a single ranked list so the - # top-N surfacing is per-switch, not per-solution. Two - # solutions that both open the same line are deduplicated - # keeping the better score. - best_by_key: Dict[Tuple[str, int], float] = {} - for sol in candidates: - score = _coerce_score(sol) - for line_id, target_status in _iter_switches(sol): - key = (line_id, target_status) - if key not in best_by_key or score < best_by_key[key]: - best_by_key[key] = score - - ranked = sorted(best_by_key.items(), key=lambda kv: kv[1]) - return [(line_id, status, score) for ((line_id, status), score) in ranked[:n]] - - def _materialise_actions( - self, - switches: List[Tuple[str, int, float]], - env: Any, - dict_action: Optional[Dict[str, Any]], - non_connected_reconnectable_lines: Optional[Iterable[str]], - network: Any, - ) -> Tuple[Dict[str, Any], Dict[str, float]]: - """Translate ToOp line-switches into ``env.action_space`` actions. - - Prefers an existing ``disco_`` / reconnection entry in the - operator's action dictionary when one exists (so suggestions - match the vocabulary the user is already familiar with); - otherwise synthesises a ``toop_disco_`` / - ``toop_reco_`` entry on the fly. Actions whose target - line isn't on the loaded network are filtered out via the same - defensive guard used by the random recommenders. - """ - # Network existence filter on the raw line IDs first. - line_ids = [line_id for (line_id, _, _) in switches] - synthetic_entries = { - f"_existence_check_{line_id}": { - "content": { - "set_bus": { - "lines_or_id": {line_id: 1}, - "lines_ex_id": {line_id: 1}, - } - } - } - for line_id in line_ids - } - kept = set( - filter_to_existing_network_elements( - list(synthetic_entries.keys()), - synthetic_entries, - network, - ) - ) - # Map back from check-id to line_id. - kept_lines = {sid.replace("_existence_check_", "", 1) for sid in kept} - reconnectable = set(non_connected_reconnectable_lines or []) - prioritized: Dict[str, Any] = {} - scores: Dict[str, float] = {} + for line_id, status in line_toggles.items(): + merged["set_bus"]["lines_or_id"][line_id] = status + merged["set_bus"]["lines_ex_id"][line_id] = status + constituents.append(("open " if status == -1 else "close ") + line_id) - for line_id, target_status, score in switches: - if line_id not in kept_lines: - continue - - action_id, content = self._pick_action_for_switch( - line_id=line_id, - target_status=target_status, - dict_action=dict_action, - reconnectable=reconnectable, - ) - try: - prioritized[action_id] = env.action_space(content) - except Exception as e: - logger.debug("ToOpRecommender: action %s rejected by env: %s", action_id, e) - continue - # Lower congestion = better in ToOp's metric; surface the - # negated score so the UI's "higher is better" sort agrees. - scores[action_id] = -float(score) - - return prioritized, scores - - @staticmethod - def _pick_action_for_switch( - line_id: str, - target_status: int, - dict_action: Optional[Dict[str, Any]], - reconnectable: set, - ) -> Tuple[str, Dict[str, Any]]: - """Return ``(action_id, content)`` for a single line-switch decision.""" - if target_status == -1: - preferred_id = f"disco_{line_id}" - if isinstance(dict_action, dict) and preferred_id in dict_action: - content = (dict_action[preferred_id] or {}).get("content") - if content is not None: - return preferred_id, content - # Fallback: synthesise the open-line content. - return ( - f"toop_disco_{line_id}", - { - "set_bus": { - "lines_or_id": {line_id: -1}, - "lines_ex_id": {line_id: -1}, - } - }, - ) - - # target_status == +1 → reconnection. Only meaningful when the - # line is currently disconnected; if it isn't reconnectable the - # action will be a no-op but we still produce it (ToOp may know - # something we don't about the live state). - if line_id in reconnectable: - preferred_id = f"reco_{line_id}" - if isinstance(dict_action, dict) and preferred_id in dict_action: - content = (dict_action[preferred_id] or {}).get("content") - if content is not None: - return preferred_id, content - return ( - f"toop_reco_{line_id}", - { - "set_bus": { - "lines_or_id": {line_id: 1}, - "lines_ex_id": {line_id: 1}, - } - }, - ) + if not constituents: + return None, [] + return merged, constituents # --------------------------------------------------------------------- -# Tolerant parsers — kept module-level so tests can swap them out. +# Module-level helpers — kept module-level so tests can swap them out. # --------------------------------------------------------------------- def _is_line_open(lines_df: Any, line_id: str) -> bool: """Return True when at least one terminal of the line is disconnected. @@ -933,9 +681,8 @@ def _is_line_open(lines_df: Any, line_id: str) -> bool: booleans on its lines DataFrame. A line is "open" (in operator terms) when *either* terminal is disconnected. Older pypowsybl builds used different column names — fall back conservatively to - ``connected`` or treat the line as closed when columns are - missing so a schema change doesn't silently flag every line as - toggled. + ``connected`` or treat the line as closed when columns are missing so + a schema change doesn't silently flag every line as toggled. """ for col_pair in (("connected1", "connected2"), ("connected",)): if all(c in lines_df.columns for c in col_pair): @@ -947,62 +694,3 @@ def _is_line_open(lines_df: Any, line_id: str) -> bool: return False return False return False - - -def _coerce_score(sol: Any) -> float: - """Pull a numeric congestion score off a ToOp solution-like object. - - Tries the metric names visible in the example notebook - (``overload_energy_n_1``), then generic fallbacks. Returns - ``float('inf')`` when nothing recognisable is present so the - solution sinks to the bottom of the ranking rather than crashing - the sort. - """ - for key in ("overload_energy_n_1", "score", "cost", "objective"): - if isinstance(sol, dict) and key in sol: - try: - return float(sol[key]) - except (TypeError, ValueError): - continue - if hasattr(sol, key): - try: - return float(getattr(sol, key)) - except (TypeError, ValueError): - continue - return float("inf") - - -def _iter_switches(sol: Any) -> Iterable[Tuple[str, int]]: - """Yield ``(line_id, target_status)`` pairs out of a solution-like object.""" - raw = None - if isinstance(sol, dict): - raw = sol.get("line_switches") or sol.get("branch_switches") - elif hasattr(sol, "line_switches"): - raw = sol.line_switches - elif hasattr(sol, "branch_switches"): - raw = sol.branch_switches - if not raw: - return - for entry in raw: - line_id = None - status: Optional[int] = None - if isinstance(entry, dict): - line_id = entry.get("line_id") or entry.get("branch_id") or entry.get("id") - if "status" in entry: - try: - status = int(entry["status"]) - except (TypeError, ValueError): - status = None - elif "open" in entry: - status = -1 if entry["open"] else 1 - elif "closed" in entry: - status = 1 if entry["closed"] else -1 - elif isinstance(entry, (list, tuple)) and len(entry) == 2: - line_id, raw_status = entry - try: - status = int(raw_status) - except (TypeError, ValueError): - status = None - if not line_id or status not in (-1, 1): - continue - yield (str(line_id), status) diff --git a/expert_backend/tests/test_toop_recommender.py b/expert_backend/tests/test_toop_recommender.py index a2eb71d0..53ef4ba3 100644 --- a/expert_backend/tests/test_toop_recommender.py +++ b/expert_backend/tests/test_toop_recommender.py @@ -3,27 +3,23 @@ # If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, # you can obtain one at http://mozilla.org/MPL/2.0/. # SPDX-License-Identifier: MPL-2.0 -"""Unit tests for :class:`ToOpRecommender` (line-switching MVP). +"""Unit tests for :class:`ToOpRecommender` (per-topology combined actions). These tests run without ToOp installed — they cover the optional-install -degradation path, the registry wiring, the result parser, and the -action-translation logic. End-to-end execution against the real ToOp -engine is a future integration-test concern (Python 3.11 + GPU env). +degradation path, the registry wiring, the per-VL/per-line topology diff, +and the merge into a single combined action content. End-to-end execution +against the real ToOp engine is a separate integration concern (Python +3.11 + GPU env). """ from __future__ import annotations from unittest.mock import MagicMock, patch import pandas as pd -import pytest from expert_backend.recommenders import toop as toop_module from expert_backend.recommenders.registry import get_model_class, list_models -from expert_backend.recommenders.toop import ( - ToOpRecommender, - _coerce_score, - _iter_switches, -) +from expert_backend.recommenders.toop import ToOpRecommender, _is_line_open # --------------------------------------------------------------------- @@ -31,35 +27,38 @@ # --------------------------------------------------------------------- class TestRegistryWiring: def test_model_is_registered(self): - """ToOp shows up in the registry after package import.""" assert get_model_class("toop") is ToOpRecommender def test_listed_in_models_endpoint_payload(self): names = [m["name"] for m in list_models()] assert "toop" in names - def test_does_not_require_overflow_graph(self): + def test_label_and_overflow_flag(self): descriptor = next(m for m in list_models() if m["name"] == "toop") + assert descriptor["label"] == "ToOp (Elia Group)" assert descriptor["requires_overflow_graph"] is False - def test_params_spec_declares_runtime_and_count_knobs(self): + def test_params_spec_declares_expected_knobs(self): descriptor = next(m for m in list_models() if m["name"] == "toop") - param_names = {p["name"] for p in descriptor["params"]} - assert {"n_prioritized_actions", "runtime_seconds", "n_worst_contingencies"} <= param_names + names = {p["name"] for p in descriptor["params"]} + assert { + "n_prioritized_actions", + "include_busbar_splits", + "runtime_seconds", + "n_worst_contingencies", + } <= names # --------------------------------------------------------------------- # Optional-install degradation # --------------------------------------------------------------------- class TestOptionalInstall: - def test_returns_empty_when_toop_not_installed(self, caplog): - """No ToOp install → empty output + one info log, never a crash.""" + def test_returns_empty_when_toop_not_installed(self): inputs = MagicMock() inputs.network = MagicMock() inputs.env = MagicMock() with patch.object(toop_module, "_import_run_pipeline", return_value=None): - with caplog.at_level("INFO"): - out = ToOpRecommender().recommend(inputs, {"n_prioritized_actions": 5}) + out = ToOpRecommender().recommend(inputs, {"n_prioritized_actions": 5}) assert out.prioritized_actions == {} assert out.action_scores == {} @@ -81,351 +80,294 @@ def test_returns_empty_when_env_missing(self): out = ToOpRecommender().recommend(inputs, {}) assert out.prioritized_actions == {} - def test_xiidm_export_failure_yields_empty(self): + def test_clears_stale_topology_groups_on_degraded_return(self): + rec = ToOpRecommender() + rec._last_topology_groups = [{"topology_id": "stale"}] inputs = MagicMock() inputs.network = MagicMock() - # Make save() raise to simulate an unsupported network shape. - inputs.network.save.side_effect = RuntimeError("XIIDM export unsupported") inputs.env = MagicMock() - with patch.object(toop_module, "_import_run_pipeline", return_value=lambda **_: None), \ - patch.object(toop_module, "_import_dictconfig", return_value=dict): - out = ToOpRecommender().recommend(inputs, {}) - assert out.prioritized_actions == {} + with patch.object(toop_module, "_import_run_pipeline", return_value=None): + rec.recommend(inputs, {}) + assert rec._last_topology_groups == [] # --------------------------------------------------------------------- -# Result parsing — _extract_line_switches + helpers +# _is_line_open helper # --------------------------------------------------------------------- -class TestResultParsing: - def test_parses_list_of_dicts_with_line_switches(self): - pareto = [ - { - "overload_energy_n_1": 12.5, - "line_switches": [{"line_id": "LINE_A", "status": -1}], - }, - { - "overload_energy_n_1": 5.0, - "line_switches": [{"line_id": "LINE_B", "open": False}], - }, - ] - out = ToOpRecommender()._extract_line_switches(pareto, n=5) - # Sorted by score ascending (lower = better congestion). - assert out[0][0] == "LINE_B" - assert out[0][1] == 1 - assert out[1][0] == "LINE_A" - assert out[1][1] == -1 - - def test_truncates_to_top_n(self): - pareto = [ - {"overload_energy_n_1": float(i), "line_switches": [{"line_id": f"L{i}", "status": -1}]} - for i in range(10) - ] - out = ToOpRecommender()._extract_line_switches(pareto, n=3) - assert len(out) == 3 - assert [r[0] for r in out] == ["L0", "L1", "L2"] - - def test_dedupes_same_line_status_keeping_best_score(self): - pareto = [ - {"score": 9.0, "line_switches": [{"line_id": "LINE_A", "status": -1}]}, - {"score": 2.0, "line_switches": [{"line_id": "LINE_A", "status": -1}]}, - ] - out = ToOpRecommender()._extract_line_switches(pareto, n=5) - assert len(out) == 1 - assert out[0] == ("LINE_A", -1, 2.0) - - def test_accepts_object_with_solutions_attribute(self): - sol = MagicMock() - sol.score = 1.0 - sol.line_switches = [{"line_id": "LINE_X", "status": 1}] - container = MagicMock() - container.solutions = [sol] - out = ToOpRecommender()._extract_line_switches(container, n=5) - assert out == [("LINE_X", 1, 1.0)] - - def test_unknown_shape_returns_empty_and_logs(self, caplog): - with caplog.at_level("WARNING"): - out = ToOpRecommender()._extract_line_switches(object(), n=5) - assert out == [] - - def test_none_returns_empty(self): - assert ToOpRecommender()._extract_line_switches(None, n=5) == [] - - def test_skips_entries_with_missing_line_id_or_status(self): - pareto = [{ - "score": 1.0, - "line_switches": [ - {"status": -1}, # missing line_id - {"line_id": "LINE_Y"}, # missing status - {"line_id": "LINE_Z", "status": 99}, # invalid status - {"line_id": "LINE_OK", "status": -1}, - ], - }] - out = ToOpRecommender()._extract_line_switches(pareto, n=5) - assert out == [("LINE_OK", -1, 1.0)] - - -class TestCoerceScore: - def test_reads_overload_energy_n_1(self): - assert _coerce_score({"overload_energy_n_1": 3.5}) == 3.5 - - def test_falls_back_to_score_then_cost(self): - assert _coerce_score({"score": 2.0}) == 2.0 - assert _coerce_score({"cost": 4.0}) == 4.0 - - def test_returns_inf_when_nothing_recognised(self): - assert _coerce_score({}) == float("inf") - assert _coerce_score(object()) == float("inf") - - -class TestIterSwitches: - def test_dict_with_branch_switches_alias(self): - sol = {"branch_switches": [{"branch_id": "L1", "status": -1}]} - assert list(_iter_switches(sol)) == [("L1", -1)] - - def test_tuple_form(self): - sol = {"line_switches": [("L1", -1), ("L2", 1)]} - assert list(_iter_switches(sol)) == [("L1", -1), ("L2", 1)] - - def test_empty_when_no_switches(self): - assert list(_iter_switches({})) == [] - assert list(_iter_switches({"line_switches": []})) == [] +class TestIsLineOpen: + def test_both_terminals_connected_is_closed(self): + df = pd.DataFrame({"connected1": [True], "connected2": [True]}, index=["L"]) + assert _is_line_open(df, "L") is False + + def test_either_terminal_disconnected_is_open(self): + df = pd.DataFrame( + {"connected1": [True, False], "connected2": [False, True]}, + index=["L1", "L2"], + ) + assert _is_line_open(df, "L1") is True + assert _is_line_open(df, "L2") is True + + def test_missing_line_id_is_closed(self): + df = pd.DataFrame({"connected1": [True], "connected2": [True]}, index=["L"]) + assert _is_line_open(df, "GHOST") is False + + def test_missing_columns_defaults_closed(self): + df = pd.DataFrame({"other": [1]}, index=["L"]) + assert _is_line_open(df, "L") is False # --------------------------------------------------------------------- -# Action materialisation +# _merge_topology_content # --------------------------------------------------------------------- -class TestMaterialiseActions: - @pytest.fixture - def network_with_lines(self, mock_network): - return mock_network - - def test_prefers_existing_disco_entry_from_dict_action(self, network_with_lines): - env = MagicMock() - env.action_space.side_effect = lambda content: ("ACTION", content) - dict_action = { - "disco_LINE_A": { - "content": {"set_bus": {"lines_or_id": {"LINE_A": -1}, "lines_ex_id": {"LINE_A": -1}}}, - "description": "Open LINE_A", - }, - } - out, scores = ToOpRecommender()._materialise_actions( - switches=[("LINE_A", -1, 5.0)], - env=env, - dict_action=dict_action, - non_connected_reconnectable_lines=[], - network=network_with_lines, +class TestMergeTopologyContent: + @staticmethod + def _enrich_ok(raw, _network): + """Fake enrich_actions_lazy that resolves each VL entry to a content.""" + out = {} + for aid, entry in raw.items(): + e = dict(entry) + vl = entry["VoltageLevelId"] + e["content"] = { + "set_bus": { + "lines_or_id": {f"{vl}_LINE": 2}, + "lines_ex_id": {}, + "loads_id": {f"{vl}_LOAD": 1}, + "generators_id": {}, + "shunts_id": {}, + }, + "switches": entry["switches"], + } + out[aid] = e + return out + + def test_merges_line_toggles_only(self): + merged, constituents = ToOpRecommender()._merge_topology_content( + line_toggles={"LINE_A": -1, "LINE_B": 1}, + vl_switches={}, + enrich=None, + network=MagicMock(), ) - # Reused the operator's vocabulary instead of fabricating a toop_disco_*. - assert "disco_LINE_A" in out - assert "toop_disco_LINE_A" not in out - assert scores["disco_LINE_A"] == -5.0 - - def test_synthesises_toop_disco_when_dict_action_lacks_entry(self, network_with_lines): - env = MagicMock() - env.action_space.side_effect = lambda content: ("ACTION", content) - out, scores = ToOpRecommender()._materialise_actions( - switches=[("LINE_A", -1, 1.0)], - env=env, - dict_action={}, - non_connected_reconnectable_lines=[], - network=network_with_lines, + assert merged["set_bus"]["lines_or_id"] == {"LINE_A": -1, "LINE_B": 1} + assert merged["set_bus"]["lines_ex_id"] == {"LINE_A": -1, "LINE_B": 1} + assert "open LINE_A" in constituents + assert "close LINE_B" in constituents + + def test_merges_vl_splits_via_enrich(self): + merged, constituents = ToOpRecommender()._merge_topology_content( + line_toggles={}, + vl_switches={"VL_A": {"VL_A_sw1": True}, "VL_B": {"VL_B_sw2": False}}, + enrich=self._enrich_ok, + network=MagicMock(), ) - assert "toop_disco_LINE_A" in out - # Negated score = "higher is better" convention. - assert scores["toop_disco_LINE_A"] == -1.0 + # set_bus from both VLs is unioned. + assert merged["set_bus"]["lines_or_id"] == {"VL_A_LINE": 2, "VL_B_LINE": 2} + assert merged["set_bus"]["loads_id"] == {"VL_A_LOAD": 1, "VL_B_LOAD": 1} + # switches from both VLs are unioned. + assert merged["switches"] == {"VL_A_sw1": True, "VL_B_sw2": False} + assert set(constituents) == {"split VL_A", "split VL_B"} + + def test_line_toggle_wins_over_split_for_same_branch(self): + def enrich_sets_line(raw, _network): + out = {} + for aid, entry in raw.items(): + e = dict(entry) + e["content"] = { + "set_bus": { + "lines_or_id": {"SHARED_LINE": 2}, # split would set bus 2 + "lines_ex_id": {}, "loads_id": {}, + "generators_id": {}, "shunts_id": {}, + }, + "switches": entry["switches"], + } + out[aid] = e + return out - def test_reconnection_uses_toop_reco_prefix(self, network_with_lines): - env = MagicMock() - env.action_space.side_effect = lambda content: ("ACTION", content) - out, _ = ToOpRecommender()._materialise_actions( - switches=[("LINE_A", 1, 0.5)], - env=env, - dict_action={}, - non_connected_reconnectable_lines=["LINE_A"], - network=network_with_lines, + merged, _ = ToOpRecommender()._merge_topology_content( + line_toggles={"SHARED_LINE": -1}, # explicit open + vl_switches={"VL_A": {"VL_A_sw1": True}}, + enrich=enrich_sets_line, + network=MagicMock(), ) - assert "toop_reco_LINE_A" in out - - def test_drops_lines_not_on_loaded_network(self, network_with_lines): - env = MagicMock() - env.action_space.side_effect = lambda content: ("ACTION", content) - out, scores = ToOpRecommender()._materialise_actions( - switches=[("LINE_GHOST", -1, 1.0), ("LINE_A", -1, 2.0)], - env=env, - dict_action={}, - non_connected_reconnectable_lines=[], - network=network_with_lines, + # Line toggle applied last → -1 overrides the split's bus 2. + assert merged["set_bus"]["lines_or_id"]["SHARED_LINE"] == -1 + + def test_returns_none_when_nothing_simulable(self): + # enrich fails for the only VL and there are no line toggles. + def enrich_empty(raw, _network): + return {aid: {**e, "content": None} for aid, e in raw.items()} + + merged, constituents = ToOpRecommender()._merge_topology_content( + line_toggles={}, + vl_switches={"VL_A": {"VL_A_sw1": True}}, + enrich=enrich_empty, + network=MagicMock(), ) - # LINE_GHOST isn't in the mock network's lines DataFrame. - assert "toop_disco_LINE_GHOST" not in out - assert "toop_disco_LINE_A" in out - - def test_skips_actions_env_rejects(self, network_with_lines): - env = MagicMock() - env.action_space.side_effect = RuntimeError("invalid action") - out, _ = ToOpRecommender()._materialise_actions( - switches=[("LINE_A", -1, 1.0)], - env=env, - dict_action={}, - non_connected_reconnectable_lines=[], - network=network_with_lines, + assert merged is None + assert constituents == [] + + def test_enrich_unavailable_keeps_line_toggles(self): + merged, constituents = ToOpRecommender()._merge_topology_content( + line_toggles={"LINE_A": -1}, + vl_switches={"VL_A": {"VL_A_sw1": True}}, # dropped (no enrich) + enrich=None, + network=MagicMock(), ) - assert out == {} + assert merged["set_bus"]["lines_or_id"] == {"LINE_A": -1} + assert constituents == ["open LINE_A"] # --------------------------------------------------------------------- -# Busbar-split materialisation +# _build_topology_actions (uses the conftest pypowsybl mock) # --------------------------------------------------------------------- -class TestMaterialiseBusbarActions: - def _make_enrich_mock(self, set_content_for: set): - """Return a fake enrich_actions_lazy that fills `content` for the listed ids.""" - def fake_enrich(raw, _network): - out = {} - for aid, entry in raw.items(): - entry = dict(entry) - if aid in set_content_for: - entry["content"] = { - "set_bus": {"lines_or_id": {}, "lines_ex_id": {}, - "loads_id": {}, "generators_id": {}, - "shunts_id": {}}, - "switches": entry["switches"], - } - # else leave content as None to exercise the "didn't populate" path - out[aid] = entry - return out - return fake_enrich +class TestBuildTopologyActions: + def _mock_pn_load(self, orig_lines, orig_switches, mod_map): + """Return a fake pypowsybl.network.load resolving paths to networks. + + mod_map: {substring_in_path: (lines_df, switches_df)}. + """ + def load(path): + net = MagicMock() + for needle, (lines, switches) in mod_map.items(): + if needle in str(path): + net.get_lines.return_value = lines + net.get_switches.return_value = switches + return net + net.get_lines.return_value = orig_lines + net.get_switches.return_value = orig_switches + return net + return load + + def test_one_action_per_topology_with_line_toggle(self, tmp_path): + import pypowsybl.network as pn_mod + + orig_lines = pd.DataFrame( + {"connected1": [True, True], "connected2": [True, True]}, + index=["LINE_A", "LINE_B"], + ) + orig_switches = pd.DataFrame( + {"open": [False], "voltage_level_id": ["VL_A"]}, index=["sw1"], + ) + # topology_1 opens LINE_A. + mod_lines = pd.DataFrame( + {"connected1": [False, True], "connected2": [True, True]}, + index=["LINE_A", "LINE_B"], + ) + topo1 = tmp_path / "topology_1" + topo1.mkdir() + (topo1 / "modified_network.xiidm").write_text("") - def test_synthesizes_switch_action_via_enrich(self): env = MagicMock() env.action_space.side_effect = lambda c: ("ACTION", c) - network = MagicMock() - action_id = "toop_split_VL_A" - enrich = self._make_enrich_mock({action_id}) - with patch( - "expert_op4grid_recommender.data_loader.enrich_actions_lazy", - enrich, create=True, - ): - prioritized, scores = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_A", {"sw1": True, "sw2": False}, 0.0)], - env=env, network=network, + loader = self._mock_pn_load( + orig_lines, orig_switches, {"topology_1": (mod_lines, orig_switches)}, + ) + with patch.object(pn_mod, "load", loader): + prioritized, scores, groups, entries = ToOpRecommender()._build_topology_actions( + topology_paths=[topo1], + original_grid_file=tmp_path / "grid.xiidm", + env=env, + network=MagicMock(), + include_busbar_splits=True, + n=5, ) - assert list(prioritized.keys()) == [action_id] - assert scores[action_id] == 0.0 - # The synthesised action's switch ids are VL-prefixed. - action_tuple = prioritized[action_id] - assert action_tuple[0] == "ACTION" - assert set(action_tuple[1]["switches"]) == {"VL_A_sw1", "VL_A_sw2"} - - def test_already_prefixed_switch_id_not_double_prefixed(self): + assert list(prioritized.keys()) == ["toop_topology_1"] + assert scores["toop_topology_1"] == 0.0 + assert groups[0]["constituents"] == ["open LINE_A"] + assert groups[0]["line_count"] == 1 + # dict_entry content carries the line toggle for re-simulation. + content = entries["toop_topology_1"]["content"] + assert content["set_bus"]["lines_or_id"] == {"LINE_A": -1} + + def test_skips_topology_with_no_diff(self, tmp_path): + import pypowsybl.network as pn_mod + + orig_lines = pd.DataFrame( + {"connected1": [True], "connected2": [True]}, index=["LINE_A"], + ) + orig_switches = pd.DataFrame( + {"open": [False], "voltage_level_id": ["VL_A"]}, index=["sw1"], + ) + topo = tmp_path / "topology_1" + topo.mkdir() + (topo / "modified_network.xiidm").write_text("") + env = MagicMock() env.action_space.side_effect = lambda c: ("ACTION", c) - network = MagicMock() - action_id = "toop_split_VL_A" - enrich = self._make_enrich_mock({action_id}) - with patch( - "expert_op4grid_recommender.data_loader.enrich_actions_lazy", - enrich, create=True, - ): - prioritized, _ = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_A", {"VL_A_sw1": True}, 0.0)], - env=env, network=network, + # modified == original → no diff. + loader = self._mock_pn_load( + orig_lines, orig_switches, {"topology_1": (orig_lines, orig_switches)}, + ) + with patch.object(pn_mod, "load", loader): + prioritized, _, groups, _ = ToOpRecommender()._build_topology_actions( + topology_paths=[topo], + original_grid_file=tmp_path / "grid.xiidm", + env=env, network=MagicMock(), + include_busbar_splits=True, n=5, ) - action_tuple = prioritized[action_id] - # Only one prefix, not VL_A_VL_A_sw1. - assert set(action_tuple[1]["switches"]) == {"VL_A_sw1"} + assert prioritized == {} + assert groups == [] - def test_empty_switches_dict_skipped(self): - env = MagicMock() - network = MagicMock() - with patch( - "expert_op4grid_recommender.data_loader.enrich_actions_lazy", - self._make_enrich_mock(set()), create=True, - ): - prioritized, scores = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_A", {}, 0.0)], - env=env, network=network, - ) - assert prioritized == {} and scores == {} + def test_caps_at_n_topologies(self, tmp_path): + import pypowsybl.network as pn_mod + + orig_lines = pd.DataFrame( + {"connected1": [True], "connected2": [True]}, index=["LINE_A"], + ) + orig_switches = pd.DataFrame( + {"open": [False], "voltage_level_id": ["VL_A"]}, index=["sw1"], + ) + mod_lines = pd.DataFrame( + {"connected1": [False], "connected2": [True]}, index=["LINE_A"], + ) + topos = [] + for i in range(4): + d = tmp_path / f"topology_{i}" + d.mkdir() + (d / "modified_network.xiidm").write_text("") + topos.append(d) - def test_enrich_failure_yields_empty(self): env = MagicMock() - network = MagicMock() - def boom(_raw, _network): - raise RuntimeError("enrich blew up") - with patch( - "expert_op4grid_recommender.data_loader.enrich_actions_lazy", - boom, create=True, - ): - prioritized, _ = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_A", {"sw1": True}, 0.0)], - env=env, network=network, + env.action_space.side_effect = lambda c: ("ACTION", c) + # Every topology opens LINE_A. + loader = self._mock_pn_load( + orig_lines, orig_switches, {"topology_": (mod_lines, orig_switches)}, + ) + with patch.object(pn_mod, "load", loader): + prioritized, _, _, _ = ToOpRecommender()._build_topology_actions( + topology_paths=topos, + original_grid_file=tmp_path / "grid.xiidm", + env=env, network=MagicMock(), + include_busbar_splits=True, n=2, ) - assert prioritized == {} + assert len(prioritized) == 2 - def test_unpopulated_content_warned_and_skipped(self, caplog): - env = MagicMock() - network = MagicMock() - # enrich returns an entry whose content stays None (not populated). - with patch( - "expert_op4grid_recommender.data_loader.enrich_actions_lazy", - self._make_enrich_mock(set()), create=True, - ): - with caplog.at_level("WARNING"): - prioritized, _ = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_A", {"sw1": True}, 0.0)], - env=env, network=network, - ) - assert prioritized == {} - assert any( - "did not populate content" in r.getMessage() - for r in caplog.records + def test_action_space_rejection_skips_topology(self, tmp_path): + import pypowsybl.network as pn_mod + + orig_lines = pd.DataFrame( + {"connected1": [True], "connected2": [True]}, index=["LINE_A"], + ) + orig_switches = pd.DataFrame( + {"open": [False], "voltage_level_id": ["VL_A"]}, index=["sw1"], ) + mod_lines = pd.DataFrame( + {"connected1": [False], "connected2": [True]}, index=["LINE_A"], + ) + topo = tmp_path / "topology_1" + topo.mkdir() + (topo / "modified_network.xiidm").write_text("") - def test_env_rejection_skips_action(self): env = MagicMock() env.action_space.side_effect = RuntimeError("backend rejected") - network = MagicMock() - action_id = "toop_split_VL_A" - with patch( - "expert_op4grid_recommender.data_loader.enrich_actions_lazy", - self._make_enrich_mock({action_id}), create=True, - ): - prioritized, _ = ToOpRecommender()._materialise_busbar_actions( - splits=[("VL_A", {"sw1": True}, 0.0)], - env=env, network=network, + loader = self._mock_pn_load( + orig_lines, orig_switches, {"topology_1": (mod_lines, orig_switches)}, + ) + with patch.object(pn_mod, "load", loader): + prioritized, _, groups, _ = ToOpRecommender()._build_topology_actions( + topology_paths=[topo], + original_grid_file=tmp_path / "grid.xiidm", + env=env, network=MagicMock(), + include_busbar_splits=True, n=5, ) assert prioritized == {} - - -# --------------------------------------------------------------------- -# Light end-to-end with everything mocked — the happy path -# --------------------------------------------------------------------- -class TestRecommendHappyPath: - def test_full_flow_with_mocked_toop(self, mock_network): - fake_pareto = [ - { - "overload_energy_n_1": 7.0, - "line_switches": [{"line_id": "LINE_A", "status": -1}], - }, - ] - fake_run_pipeline = MagicMock(return_value=fake_pareto) - - inputs = MagicMock() - inputs.network = mock_network - inputs.env = MagicMock() - inputs.env.action_space.side_effect = lambda content: ("ACTION", content) - inputs.dict_action = {} - inputs.non_connected_reconnectable_lines = [] - - # Bypass real XIIDM export AND the ToOp config-building (PipelineConfig / - # prepare_importer_parameters / etc. only exist when ToOp is installed). - fake_grid_file = MagicMock() - with patch.object(toop_module, "_import_run_pipeline", return_value=fake_run_pipeline), \ - patch.object(toop_module, "_import_dictconfig", return_value=dict), \ - patch.object(ToOpRecommender, "_export_network", return_value=fake_grid_file), \ - patch.object(ToOpRecommender, "_run_toop", return_value=fake_pareto): - out = ToOpRecommender().recommend(inputs, {"n_prioritized_actions": 3}) - - assert "toop_disco_LINE_A" in out.prioritized_actions - assert out.action_scores["toop_disco_LINE_A"] == -7.0 - fake_run_pipeline.assert_called_once() + assert groups == [] From e648f0de660d2d2b555f53a82c465c4536f7a14b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 1 Jun 2026 16:14:07 +0000 Subject: [PATCH 10/14] feat(frontend): render ToOp candidate topologies as N-way combined cards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend now emits each ToOp candidate topology as one combined_actions entry (is_toop_topology=true) carrying a REAL simulated max_rho and the full constituent_ids list. The combined-actions UI assumed exactly 2-action superposition pairs; relax it for the N-way topology case: - types.ts: extend CombinedAction with is_toop_topology / constituent_ids / constituent_count / is_simulated / simulated_max_rho(_line) / rank. - CombinedActionsModal: seed a ToOp entry's simData from its carried rho (it is already really-simulated by the step-2 assessment), so the row shows the true combined loading without a manual Simulate click. ToOp rows bypass the per-constituent action-type ring (a topology is its own category) and are governed by severity / max-loading only. Pass is_toop_topology + constituent_ids through to the table row. - ComputedPairsTable: render a ToOp row as a single "🧩 ToOp topology #N" summary cell (colSpan 5) with the constituent moves as chips, the real simulated max-loading badge, and a Re-Simulate button that replays the whole combination by its topology id (which the backend registered in _dict_action). Legacy 2-up pair rows are untouched. Tests: 3 new cases for the topology row (title + chips, simulated badge, re-simulate-by-id). Full Vitest suite stays green (1448 passed). --- .../src/components/CombinedActionsModal.tsx | 25 +++++- .../components/ComputedPairsTable.test.tsx | 41 ++++++++++ .../src/components/ComputedPairsTable.tsx | 77 +++++++++++++++++++ frontend/src/types.ts | 14 ++++ 4 files changed, 155 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/CombinedActionsModal.tsx b/frontend/src/components/CombinedActionsModal.tsx index 8c3734b6..79775b6a 100644 --- a/frontend/src/components/CombinedActionsModal.tsx +++ b/frontend/src/components/CombinedActionsModal.tsx @@ -157,6 +157,20 @@ const CombinedActionsModal: React.FC = ({ const allPairs = [ ...combinedEntries.map(([id, ca]) => { const cId = canonicalizeId(id); + // ToOp candidate topologies arrive already really-simulated + // (the step-2 assessment simulated the whole combination), so + // seed simData from the carried rho rather than looking for a + // separate pair simulation. A later manual Re-Simulate still + // overrides via sessionSimResults below. + if (ca.is_toop_topology) { + const sessionResult = sessionSimResults[id] || sessionSimResults[cId]; + const simData: SimulationFeedback = sessionResult || { + max_rho: ca.simulated_max_rho ?? ca.max_rho ?? null, + max_rho_line: ca.simulated_max_rho_line ?? ca.max_rho_line ?? '', + is_rho_reduction: !!ca.is_rho_reduction, + }; + return { id, data: ca, simData }; + } const sessionResult = sessionSimResults[id] || sessionSimResults[cId]; const parentSimData = simulatedActions[id] || simulatedActions[cId]; const analysisSimData = analysisResult?.actions?.[id] || analysisResult?.actions?.[cId]; @@ -194,7 +208,9 @@ const CombinedActionsModal: React.FC = ({ isSimulated, simulated_max_rho: simMaxRho, simulated_max_rho_line: simMaxRhoLine, - simData: simData + simData: simData, + is_toop_topology: !!data.is_toop_topology, + constituent_ids: data.constituent_ids, }; }); }, [analysisResult, simulatedActions, sessionSimResults]); @@ -205,7 +221,12 @@ const CombinedActionsModal: React.FC = ({ // estimated value. const filteredComputedPairsList = useMemo(() => { return computedPairsList.filter(p => { - const typeOk = filters.actionType === 'all' + // ToOp candidate topologies are a category of their own (an N-way + // optimised combination), so the per-constituent action-type ring + // doesn't apply — they bypass it and are governed by severity / + // max-loading only. + const typeOk = p.is_toop_topology + || filters.actionType === 'all' || matchesActionTypeFilter(filters.actionType, p.action1, null, scoreTypeByActionId.get(p.action1) ?? null) || matchesActionTypeFilter(filters.actionType, p.action2, null, scoreTypeByActionId.get(p.action2) ?? null); if (!typeOk) return false; diff --git a/frontend/src/components/ComputedPairsTable.test.tsx b/frontend/src/components/ComputedPairsTable.test.tsx index 8b8e0bb8..42baab05 100644 --- a/frontend/src/components/ComputedPairsTable.test.tsx +++ b/frontend/src/components/ComputedPairsTable.test.tsx @@ -155,4 +155,45 @@ describe('ComputedPairsTable', () => { render(); expect(screen.getByTitle('Islanding detected')).toBeInTheDocument(); }); + + describe('ToOp candidate topology row', () => { + const topologyRow: ComputedPairEntry = { + id: 'toop_topology_3', + action1: 'split GROSNP7', + action2: 'split VIELMP6', + is_suspect: false, + isSimulated: true, + simulated_max_rho: 0.92, + simulated_max_rho_line: 'BEON L31CPVAN', + simData: { max_rho: 0.92, max_rho_line: 'BEON L31CPVAN', is_rho_reduction: true }, + is_toop_topology: true, + constituent_ids: ['split GROSNP7', 'split VIELMP6', 'open GROSNL71SSV.O'], + }; + + it('renders a friendly topology title and the constituent chips', () => { + render(); + expect(screen.getByText(/ToOp topology #3/)).toBeInTheDocument(); + expect(screen.getByText('split GROSNP7')).toBeInTheDocument(); + expect(screen.getByText('split VIELMP6')).toBeInTheDocument(); + expect(screen.getByText('open GROSNL71SSV.O')).toBeInTheDocument(); + }); + + it('shows the real simulated max-loading badge', () => { + render(); + expect(screen.getByText('92.0%')).toBeInTheDocument(); + }); + + it('re-simulates by topology id (not a + join)', () => { + const onSimulate = vi.fn(); + render( + , + ); + fireEvent.click(screen.getByText('Re-Simulate')); + expect(onSimulate).toHaveBeenCalledWith('toop_topology_3'); + }); + }); }); diff --git a/frontend/src/components/ComputedPairsTable.tsx b/frontend/src/components/ComputedPairsTable.tsx index a1066568..3d6e2e7b 100644 --- a/frontend/src/components/ComputedPairsTable.tsx +++ b/frontend/src/components/ComputedPairsTable.tsx @@ -32,6 +32,16 @@ export interface ComputedPairEntry { simulated_max_rho: number | null; simulated_max_rho_line: string | null; simData: ActionDetail | SimulationFeedback | null; + // ToOp N-way candidate topology: render the constituent list across the + // pair columns instead of the 2-up Action 1 / Action 2 / Betas / Est. + is_toop_topology?: boolean; + constituent_ids?: string[]; +} + +/** "toop_topology_3" → "ToOp topology #3"; falls back to the raw id. */ +function topologyTitle(id: string): string { + const m = /^toop_topology_(\d+)$/.exec(id); + return m ? `ToOp topology #${m[1]}` : id; } interface ComputedPairsTableProps { @@ -76,6 +86,73 @@ const ComputedPairsTable: React.FC = ({ const isSimulated = p.isSimulated; const simMaxRho = p.simulated_max_rho; + // ToOp candidate topology: one combined card per + // optimised topology. Render the constituent list + // across the pair columns and the real simulated + // loading on the right. Re-Simulate replays the whole + // combination by its topology id. + if (p.is_toop_topology) { + return ( + + +
+ 🧩 {topologyTitle(p.id)} +
+
+ {(p.constituent_ids || []).map((c, i) => ( + {c} + ))} +
+ + + {simMaxRho != null ? ( + monitoringFactor ? colors.dangerSoft : simMaxRho > (monitoringFactor - 0.05) ? colors.warningSoft : colors.successSoft, + color: simMaxRho > monitoringFactor ? colors.dangerText : simMaxRho > (monitoringFactor - 0.05) ? colors.warningText : colors.successStrong, + }}> + {(simMaxRho * 100).toFixed(1)}% + {(p.simData as ActionDetail | SimulationFeedback)?.is_islanded && ( + 🏝️ + )} + + ) : ( + — + )} + + + {p.simulated_max_rho_line ? displayName(p.simulated_max_rho_line) : '-'} + + + + + + ); + } + return ( {p.action1} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 295e4ef0..2d7308f6 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -131,6 +131,20 @@ export interface CombinedAction { target_max_rho?: number | null; target_max_rho_line?: string; error?: string; + // ToOp N-way candidate topology (ToOp recommender integration). When + // `is_toop_topology` is set this entry represents a whole optimised + // topology rather than a 2-action superposition pair: `constituent_ids` + // lists every elementary move it bundles, and the rho fields carry a + // REAL simulation of the full combination (`is_simulated` = true), not + // a superposition estimate. `action1_id` / `action2_id` are still + // populated with the first two constituents for legacy rendering. + is_toop_topology?: boolean; + constituent_ids?: string[]; + constituent_count?: number; + is_simulated?: boolean; + simulated_max_rho?: number | null; + simulated_max_rho_line?: string; + rank?: number; } export interface AnalysisResult { From 96166f9e93169547e0e80bcb208da0a4b5c7e140 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 1 Jun 2026 16:31:55 +0000 Subject: [PATCH 11/14] feat(toop): keep candidate topologies in the Suggested Actions feed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator feedback: routing ToOp candidate topologies into the Combine Actions modal is wrong — they should land directly in the Suggested Actions list, as one card per topology, alongside any other recommender output. Flip the routing: - `_service_integration`: don't move topology entries out of `enriched_actions` and into `combined_actions`. Instead, decorate each enriched topology entry IN PLACE with N-way metadata (`is_toop_topology`, `constituent_ids`, `constituent_count`, `rank`) so the frontend ActionCard can render the constituents. Still inject the synthesised merged contents into `_dict_action` so the card stays re-simulatable / session-saveable. - types.ts: move the N-way metadata fields from `CombinedAction` to `ActionDetail` (where action-card data lives). `CombinedAction` reverts to its previous 2-pair shape. - ActionCard: render a constituent-chip strip under the title when `is_toop_topology` is set. Native title tooltip explains the row ("really simulated as one combined action"). data-testid `action-card--constituents` for the regression test. - Revert the Combine-modal / ComputedPairsTable additions (no separate ToOp render path needed there anymore). - ActionCard tests: 3 new cases — chips render when is_toop_topology, no strip on non-ToOp cards, no strip when constituent_ids is empty. Full Vitest suite stays green (1448 passed). Backend lint + code- quality gate + step-2 integration tests (46) all pass. Resulting UX: a ToOp run on the same contingency now shows 4 cards in Suggested Actions named `toop_topology_1`…`toop_topology_4`, each with the elementary moves as chips and the real combined max_rho on the badge — instead of the 10 misleading single-move cards or a separate modal. --- .../recommenders/_service_integration.py | 54 ++++--------- frontend/src/components/ActionCard.test.tsx | 33 ++++++++ frontend/src/components/ActionCard.tsx | 25 ++++++ .../src/components/CombinedActionsModal.tsx | 25 +----- .../components/ComputedPairsTable.test.tsx | 41 ---------- .../src/components/ComputedPairsTable.tsx | 77 ------------------- frontend/src/types.ts | 23 +++--- 7 files changed, 86 insertions(+), 192 deletions(-) diff --git a/expert_backend/recommenders/_service_integration.py b/expert_backend/recommenders/_service_integration.py index adf99ff2..c3823f8d 100644 --- a/expert_backend/recommenders/_service_integration.py +++ b/expert_backend/recommenders/_service_integration.py @@ -280,53 +280,33 @@ def _run_analysis_step2_with_model( results.get("action_scores", {}) ) - # ToOp: reformat each candidate topology into ONE combined-action - # card. Each topology was returned as a single merged prioritized - # action and REALLY simulated by the assessment phase above, so - # ``enriched_actions`` already carries its true combined max_rho. - # Move those entries out of the main feed and into - # ``combined_actions`` (the channel the operator selected), and - # inject the merged contents into ``_dict_action`` so each card - # stays re-simulatable / session-saveable. + # ToOp: each candidate topology is a single prioritized action + # (one merged grid2op action object) that the assessment phase + # already REALLY simulated, so ``enriched_actions[topology_id]`` + # carries the true combined max_rho. Leave the entries in the + # main Suggested-Actions feed and decorate them in place with + # N-way metadata so the frontend ActionCard can render the + # constituent moves. Also inject the synthesised contents into + # ``_dict_action`` so the card stays re-simulatable / + # session-saveable. topo_groups = getattr(recommender, "_last_topology_groups", None) if topo_groups: - combined = results.get("combined_actions", {}) or {} topo_entries = getattr(recommender, "_last_topology_dict_entries", {}) or {} if self._dict_action is None: self._dict_action = {} self._dict_action.update(topo_entries) for grp in topo_groups: tid = grp["topology_id"] - enriched = enriched_actions.pop(tid, None) - if enriched is None: + entry = enriched_actions.get(tid) + if entry is None: continue constituents = grp.get("constituents", []) or [] - combined[tid] = { - # N-way combination metadata. action1_id/action2_id are - # kept (first two constituents) so the legacy 2-up pairs - # table still renders something before the N-way UI lands. - "constituent_ids": constituents, - "constituent_count": len(constituents), - "action1_id": constituents[0] if constituents else "", - "action2_id": constituents[1] if len(constituents) > 1 else "", - "description": "ToOp: " + ", ".join(constituents), - "betas": [], - "is_toop_topology": True, - "rank": grp.get("rank"), - # Real simulation results carried from the assessment. - "max_rho": enriched.get("max_rho"), - "max_rho_line": enriched.get("max_rho_line"), - "target_max_rho": enriched.get("max_rho"), - "target_max_rho_line": enriched.get("max_rho_line"), - "simulated_max_rho": enriched.get("max_rho"), - "simulated_max_rho_line": enriched.get("max_rho_line"), - "is_simulated": True, - "is_rho_reduction": enriched.get("is_rho_reduction"), - "rho_before": enriched.get("rho_before"), - "rho_after": enriched.get("rho_after"), - "lines_overloaded_after": enriched.get("lines_overloaded_after"), - } - results["combined_actions"] = combined + entry["is_toop_topology"] = True + entry["constituent_ids"] = constituents + entry["constituent_count"] = len(constituents) + entry["rank"] = grp.get("rank") + if not entry.get("description_unitaire"): + entry["description_unitaire"] = "ToOp: " + ", ".join(constituents) enrichment_time = time.time() - _t_enrich diff --git a/frontend/src/components/ActionCard.test.tsx b/frontend/src/components/ActionCard.test.tsx index 125e540c..7848e62d 100644 --- a/frontend/src/components/ActionCard.test.tsx +++ b/frontend/src/components/ActionCard.test.tsx @@ -472,4 +472,37 @@ describe('ActionCard', () => { expect(screen.queryByTestId('action-card-act_1-origin')).not.toBeInTheDocument(); }); }); + + describe('ToOp candidate topology constituents', () => { + const topologyDetails: ActionDetail = { + ...baseDetails, + is_toop_topology: true, + constituent_ids: ['split GROSNP7', 'split VIELMP6', 'open GROSNL71SSV.O'], + constituent_count: 3, + }; + + it('renders one chip per constituent move when the card is a ToOp topology', () => { + render(); + const strip = screen.getByTestId('action-card-toop_topology_1-constituents'); + expect(strip).toBeInTheDocument(); + expect(strip).toHaveTextContent('split GROSNP7'); + expect(strip).toHaveTextContent('split VIELMP6'); + expect(strip).toHaveTextContent('open GROSNL71SSV.O'); + }); + + it('does not render the constituent strip on non-ToOp action cards', () => { + render(); + expect(screen.queryByTestId('action-card-act_1-constituents')).not.toBeInTheDocument(); + }); + + it('does not render the strip when is_toop_topology is set but constituents are missing', () => { + const detailsNoConstituents: ActionDetail = { + ...baseDetails, + is_toop_topology: true, + constituent_ids: [], + }; + render(); + expect(screen.queryByTestId('action-card-toop_topology_2-constituents')).not.toBeInTheDocument(); + }); + }); }); diff --git a/frontend/src/components/ActionCard.tsx b/frontend/src/components/ActionCard.tsx index a3ec6119..ebd9271c 100644 --- a/frontend/src/components/ActionCard.tsx +++ b/frontend/src/components/ActionCard.tsx @@ -294,6 +294,31 @@ const ActionCard: React.FC = ({ + {details.is_toop_topology && (details.constituent_ids?.length ?? 0) > 0 && ( +
+ {(details.constituent_ids ?? []).map((c, i) => ( + {c} + ))} +
+ )} + {/* Compact at-rest body: max loading + target badges. The rail (⭐ / ❌) sits to the right and fades in on hover or when this card is being viewed. The row wraps diff --git a/frontend/src/components/CombinedActionsModal.tsx b/frontend/src/components/CombinedActionsModal.tsx index 79775b6a..8c3734b6 100644 --- a/frontend/src/components/CombinedActionsModal.tsx +++ b/frontend/src/components/CombinedActionsModal.tsx @@ -157,20 +157,6 @@ const CombinedActionsModal: React.FC = ({ const allPairs = [ ...combinedEntries.map(([id, ca]) => { const cId = canonicalizeId(id); - // ToOp candidate topologies arrive already really-simulated - // (the step-2 assessment simulated the whole combination), so - // seed simData from the carried rho rather than looking for a - // separate pair simulation. A later manual Re-Simulate still - // overrides via sessionSimResults below. - if (ca.is_toop_topology) { - const sessionResult = sessionSimResults[id] || sessionSimResults[cId]; - const simData: SimulationFeedback = sessionResult || { - max_rho: ca.simulated_max_rho ?? ca.max_rho ?? null, - max_rho_line: ca.simulated_max_rho_line ?? ca.max_rho_line ?? '', - is_rho_reduction: !!ca.is_rho_reduction, - }; - return { id, data: ca, simData }; - } const sessionResult = sessionSimResults[id] || sessionSimResults[cId]; const parentSimData = simulatedActions[id] || simulatedActions[cId]; const analysisSimData = analysisResult?.actions?.[id] || analysisResult?.actions?.[cId]; @@ -208,9 +194,7 @@ const CombinedActionsModal: React.FC = ({ isSimulated, simulated_max_rho: simMaxRho, simulated_max_rho_line: simMaxRhoLine, - simData: simData, - is_toop_topology: !!data.is_toop_topology, - constituent_ids: data.constituent_ids, + simData: simData }; }); }, [analysisResult, simulatedActions, sessionSimResults]); @@ -221,12 +205,7 @@ const CombinedActionsModal: React.FC = ({ // estimated value. const filteredComputedPairsList = useMemo(() => { return computedPairsList.filter(p => { - // ToOp candidate topologies are a category of their own (an N-way - // optimised combination), so the per-constituent action-type ring - // doesn't apply — they bypass it and are governed by severity / - // max-loading only. - const typeOk = p.is_toop_topology - || filters.actionType === 'all' + const typeOk = filters.actionType === 'all' || matchesActionTypeFilter(filters.actionType, p.action1, null, scoreTypeByActionId.get(p.action1) ?? null) || matchesActionTypeFilter(filters.actionType, p.action2, null, scoreTypeByActionId.get(p.action2) ?? null); if (!typeOk) return false; diff --git a/frontend/src/components/ComputedPairsTable.test.tsx b/frontend/src/components/ComputedPairsTable.test.tsx index 42baab05..8b8e0bb8 100644 --- a/frontend/src/components/ComputedPairsTable.test.tsx +++ b/frontend/src/components/ComputedPairsTable.test.tsx @@ -155,45 +155,4 @@ describe('ComputedPairsTable', () => { render(); expect(screen.getByTitle('Islanding detected')).toBeInTheDocument(); }); - - describe('ToOp candidate topology row', () => { - const topologyRow: ComputedPairEntry = { - id: 'toop_topology_3', - action1: 'split GROSNP7', - action2: 'split VIELMP6', - is_suspect: false, - isSimulated: true, - simulated_max_rho: 0.92, - simulated_max_rho_line: 'BEON L31CPVAN', - simData: { max_rho: 0.92, max_rho_line: 'BEON L31CPVAN', is_rho_reduction: true }, - is_toop_topology: true, - constituent_ids: ['split GROSNP7', 'split VIELMP6', 'open GROSNL71SSV.O'], - }; - - it('renders a friendly topology title and the constituent chips', () => { - render(); - expect(screen.getByText(/ToOp topology #3/)).toBeInTheDocument(); - expect(screen.getByText('split GROSNP7')).toBeInTheDocument(); - expect(screen.getByText('split VIELMP6')).toBeInTheDocument(); - expect(screen.getByText('open GROSNL71SSV.O')).toBeInTheDocument(); - }); - - it('shows the real simulated max-loading badge', () => { - render(); - expect(screen.getByText('92.0%')).toBeInTheDocument(); - }); - - it('re-simulates by topology id (not a + join)', () => { - const onSimulate = vi.fn(); - render( - , - ); - fireEvent.click(screen.getByText('Re-Simulate')); - expect(onSimulate).toHaveBeenCalledWith('toop_topology_3'); - }); - }); }); diff --git a/frontend/src/components/ComputedPairsTable.tsx b/frontend/src/components/ComputedPairsTable.tsx index 3d6e2e7b..a1066568 100644 --- a/frontend/src/components/ComputedPairsTable.tsx +++ b/frontend/src/components/ComputedPairsTable.tsx @@ -32,16 +32,6 @@ export interface ComputedPairEntry { simulated_max_rho: number | null; simulated_max_rho_line: string | null; simData: ActionDetail | SimulationFeedback | null; - // ToOp N-way candidate topology: render the constituent list across the - // pair columns instead of the 2-up Action 1 / Action 2 / Betas / Est. - is_toop_topology?: boolean; - constituent_ids?: string[]; -} - -/** "toop_topology_3" → "ToOp topology #3"; falls back to the raw id. */ -function topologyTitle(id: string): string { - const m = /^toop_topology_(\d+)$/.exec(id); - return m ? `ToOp topology #${m[1]}` : id; } interface ComputedPairsTableProps { @@ -86,73 +76,6 @@ const ComputedPairsTable: React.FC = ({ const isSimulated = p.isSimulated; const simMaxRho = p.simulated_max_rho; - // ToOp candidate topology: one combined card per - // optimised topology. Render the constituent list - // across the pair columns and the real simulated - // loading on the right. Re-Simulate replays the whole - // combination by its topology id. - if (p.is_toop_topology) { - return ( - - -
- 🧩 {topologyTitle(p.id)} -
-
- {(p.constituent_ids || []).map((c, i) => ( - {c} - ))} -
- - - {simMaxRho != null ? ( - monitoringFactor ? colors.dangerSoft : simMaxRho > (monitoringFactor - 0.05) ? colors.warningSoft : colors.successSoft, - color: simMaxRho > monitoringFactor ? colors.dangerText : simMaxRho > (monitoringFactor - 0.05) ? colors.warningText : colors.successStrong, - }}> - {(simMaxRho * 100).toFixed(1)}% - {(p.simData as ActionDetail | SimulationFeedback)?.is_islanded && ( - 🏝️ - )} - - ) : ( - — - )} - - - {p.simulated_max_rho_line ? displayName(p.simulated_max_rho_line) : '-'} - - - - - - ); - } - return ( {p.action1} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 2d7308f6..c46442f5 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -111,6 +111,15 @@ export interface ActionDetail { * reload (see `useSession.handleRestoreSession`). */ origin?: string; + // ToOp recommender N-way candidate topology. When `is_toop_topology` is + // set, this action card represents a whole optimised topology (multiple + // line toggles and/or busbar splits bundled into one merged action that + // was REALLY simulated). `constituent_ids` lists the elementary moves + // for display as chips on the card; `rank` is ToOp's order (0 = best). + is_toop_topology?: boolean; + constituent_ids?: string[]; + constituent_count?: number; + rank?: number; } export interface CombinedAction { @@ -131,20 +140,6 @@ export interface CombinedAction { target_max_rho?: number | null; target_max_rho_line?: string; error?: string; - // ToOp N-way candidate topology (ToOp recommender integration). When - // `is_toop_topology` is set this entry represents a whole optimised - // topology rather than a 2-action superposition pair: `constituent_ids` - // lists every elementary move it bundles, and the rho fields carry a - // REAL simulation of the full combination (`is_simulated` = true), not - // a superposition estimate. `action1_id` / `action2_id` are still - // populated with the first two constituents for legacy rendering. - is_toop_topology?: boolean; - constituent_ids?: string[]; - constituent_count?: number; - is_simulated?: boolean; - simulated_max_rho?: number | null; - simulated_max_rho_line?: string; - rank?: number; } export interface AnalysisResult { From f7d0aac9756b6ac1591e188ef84d0532e45cf38b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 1 Jun 2026 16:51:39 +0000 Subject: [PATCH 12/14] fix(toop): restore description fields + tolerant lazy-entry access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regression after the per-topology rework: on the same grid the prior synth path resolved (`synthesized busbar action toop_split_` for seven VLs), the new merge path fails on every VL ("could not enrich VL split"). Two diffs caused it: 1. The internal raw-action key was `_toop_vl_` instead of the `toop_split_` pattern the earlier run used. The library's enrich proxy keys off the action-id shape; a leading underscore + unfamiliar id silently fails to attach `content`. 2. The raw entries lost their `description` / `description_unitaire` fields — present in the per-VL synth code that previously worked, absent in the new merge code. Some lazy enrich paths require them to populate the content slot. Restore both: - Use the `toop_split_` id pattern for the temporary enrichment entries (purely internal to the merge — no conflict with the outer `toop_topology_` ids surfaced to the UI). - Populate `description` and `description_unitaire` on each raw entry before calling `enrich_actions_lazy`. Also harden the access pattern: the enriched value is a `LazyActionDict`-style proxy whose entries don't satisfy `isinstance(entry, dict)` even though they behave like one — so the old `entry.get("content") if isinstance(entry, dict) else None` guard sent everything to the silent-skip branch. Two new helpers, `_resolve_lazy_entry` and `_resolve_lazy_content`, try mapping access, attribute access, and ``.get`` in turn so populated content is found whichever proxy shape the library returns. Diagnostic logging upgraded: the skip-warning now prints `entry=, content=` so a future failure tells us exactly which proxy shape we got back, instead of just "could not enrich". --- expert_backend/recommenders/toop.py | 80 +++++++++++++++++++++++++---- 1 file changed, 70 insertions(+), 10 deletions(-) diff --git a/expert_backend/recommenders/toop.py b/expert_backend/recommenders/toop.py index f32e74ac..8958a50f 100644 --- a/expert_backend/recommenders/toop.py +++ b/expert_backend/recommenders/toop.py @@ -615,8 +615,19 @@ def _merge_topology_content( constituents: List[str] = [] if vl_switches and enrich is not None: + # Match the action-entry shape `enrich_actions_lazy` saw when the + # earlier per-VL synthesis worked end-to-end on the same grid: + # `description` + `description_unitaire` populated, and the + # action-id pattern uses the `toop_split_` prefix the library + # already recognises (a leading underscore + free-form id caused + # silent enrichment failure — no content was attached). raw = { - f"_toop_vl_{vl_id}": { + f"toop_split_{vl_id}": { + "description": ( + f"ToOp: substation reconfiguration at '{vl_id}' " + f"({len(switches)} switch operation(s))" + ), + "description_unitaire": f"ToOp split at {vl_id}", "VoltageLevelId": vl_id, "switches": dict(switches), "content": None, @@ -631,18 +642,16 @@ def _merge_topology_content( ) enriched = {} for vl_id in vl_switches: - key = f"_toop_vl_{vl_id}" - entry = None - try: - entry = enriched.get(key) if hasattr(enriched, "get") else enriched[key] - except Exception: - entry = None - content = entry.get("content") if isinstance(entry, dict) else None + key = f"toop_split_{vl_id}" + entry = _resolve_lazy_entry(enriched, key) + content = _resolve_lazy_content(entry) if not content: logger.warning( - "ToOpRecommender: could not enrich VL %s split (%d switch(es)); " - "excluded from this topology.", + "ToOpRecommender: could not enrich VL %s split (%d switch(es); " + "entry=%s, content=%s); excluded from this topology.", vl_id, len(vl_switches[vl_id]), + type(entry).__name__ if entry is not None else None, + type(content).__name__ if content is not None else None, ) continue set_bus = content.get("set_bus", {}) if isinstance(content, dict) else {} @@ -674,6 +683,57 @@ def _merge_topology_content( # --------------------------------------------------------------------- # Module-level helpers — kept module-level so tests can swap them out. # --------------------------------------------------------------------- +def _resolve_lazy_entry(enriched: Any, key: str) -> Any: + """Return ``enriched[key]`` regardless of whether enriched is a plain dict + or a ``LazyActionDict`` proxy. + + ``enrich_actions_lazy`` returns a proxy in production: ``.get(key)`` may + return ``None`` for keys whose entries haven't been touched yet, while + ``proxy[key]`` triggers the lazy resolution. We try both, in that order. + """ + if enriched is None: + return None + if hasattr(enriched, "get"): + try: + entry = enriched.get(key) + if entry is not None: + return entry + except Exception: + pass + try: + return enriched[key] + except Exception: + return None + + +def _resolve_lazy_content(entry: Any) -> Optional[Dict[str, Any]]: + """Read ``entry["content"]`` regardless of whether entry is a dict or proxy. + + LazyActionDict wraps its entries so ``isinstance(entry, dict)`` is False + even though they behave like one. We try mapping access AND attribute + access AND finally fall back to ``.get`` — whichever first returns a + populated dict-like value with the keys we need. + """ + if entry is None: + return None + candidates = [] + try: + candidates.append(entry["content"]) + except Exception: + pass + if hasattr(entry, "get"): + try: + candidates.append(entry.get("content")) + except Exception: + pass + if hasattr(entry, "content"): + candidates.append(getattr(entry, "content")) + for c in candidates: + if c and (isinstance(c, dict) or hasattr(c, "get")): + return c if isinstance(c, dict) else None + return None + + def _is_line_open(lines_df: Any, line_id: str) -> bool: """Return True when at least one terminal of the line is disconnected. From 2e05d8df2a257c3b86a6c3f690bf335fd907329d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 1 Jun 2026 18:46:14 +0000 Subject: [PATCH 13/14] fix(recommenders/toop): export post-contingency grid + restore N-1 topology actions - Export inputs.network_defaut (N-K state) instead of inputs.network (N), and explicitly disconnect inputs.lines_defaut on the exported copy, so ToOp's N-0 == the operator-selected contingency. Fixes overload_energy_n0=0.0 -> "No topologies found" -> 0 actions. - Port stash pipeline config: max_num_disconnections=4 (re-enables line switching), overload_energy_n_0 targeting via optimize_current_state_only, and _deflate_thermal_limits (align ToOp threshold with monitoring factor). - _merge_topology_content: fall back to raw switch flips when enrich_actions_lazy returns no content, instead of dropping the VL split. - Wrap action_scores in the category-keyed shape ({category:{scores,params}}) via _nest_scores; fixes AttributeError 'float' has no 'get' in propagate_non_convergence_to_scores. - Tests: params_spec knob, _run_toop config (n0/n1), _deflate_thermal_limits, _merge fallback, _apply_contingency_state, _nest_scores. --- expert_backend/recommenders/toop.py | 325 ++++++++++++++++-- expert_backend/tests/test_toop_recommender.py | 241 ++++++++++++- 2 files changed, 538 insertions(+), 28 deletions(-) diff --git a/expert_backend/recommenders/toop.py b/expert_backend/recommenders/toop.py index 8958a50f..25e7a14e 100644 --- a/expert_backend/recommenders/toop.py +++ b/expert_backend/recommenders/toop.py @@ -169,7 +169,21 @@ def params_spec(cls) -> List[ParamSpec]: max=20, description=( "Number of worst-N-1 contingencies ToOp evaluates " - "when scoring each topology candidate." + "when scoring each topology candidate. Ignored when " + "`optimize_current_state_only` is enabled." + ), + ), + ParamSpec( + "optimize_current_state_only", + "Optimize Current State Only", + "bool", + default=True, + description=( + "Score topologies on the operator-selected contingency " + "state only (ToOp's N-0). Ignore the secondary N-1 " + "outages ToOp would otherwise explore from that state " + "— those are N-2 cases from the operator's viewpoint. " + "Disable to also weigh subsequent contingencies." ), ), ] @@ -207,6 +221,7 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu runtime_seconds = int(params.get("runtime_seconds", 15)) n_worst = int(params.get("n_worst_contingencies", 2)) include_busbar_splits = bool(params.get("include_busbar_splits", True)) + current_state_only = bool(params.get("optimize_current_state_only", True)) env = inputs.env if env is None: @@ -221,15 +236,29 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu "ToOpRecommender: exporting pypowsybl network to XIIDM under %s", iteration_dir, ) - grid_file = self._export_network(inputs.network, iteration_dir) + # Export the POST-CONTINGENCY state, not the healthy N state. + # ``inputs.network_defaut`` is documented as the same network + # with the contingency variant active; we prefer it and then + # explicitly re-assert the contingency by disconnecting + # ``inputs.lines_defaut`` on the exported copy. This guarantees + # ToOp's N-0 == the operator-selected N-1 even if the working + # variant wasn't positioned — otherwise overload_energy_n_0 is + # 0.0 (no overload to optimise) and the GA returns 0 topologies. + export_source = inputs.network_defaut + if export_source is None: + export_source = inputs.network + contingency_lines = list(inputs.lines_defaut or []) + grid_file = self._export_network( + export_source, iteration_dir, contingency_lines=contingency_lines + ) if grid_file is None: logger.warning("ToOpRecommender: network export returned None — aborting.") return RecommenderOutput(prioritized_actions={}, action_scores={}) logger.warning( "ToOpRecommender: calling run_pipeline (runtime_seconds=%d, " - "n_worst_contingencies=%d) on %s", - runtime_seconds, n_worst, grid_file, + "n_worst_contingencies=%d, current_state_only=%s) on %s", + runtime_seconds, n_worst, current_state_only, grid_file, ) try: pareto = self._run_toop( @@ -239,6 +268,7 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu work_dir=tmp_path, runtime_seconds=runtime_seconds, n_worst_contingencies=n_worst, + current_state_only=current_state_only, ) except Exception: logger.exception("ToOpRecommender: run_pipeline failed; returning {}.") @@ -276,21 +306,63 @@ def recommend(self, inputs: RecommenderInputs, params: dict) -> RecommenderOutpu "ToOpRecommender: returning %d topology action(s): %s", len(prioritized), list(prioritized.keys()), ) + # The library's reassessment (``propagate_non_convergence_to_scores``) + # and the service's ``_compute_mw_start_for_scores`` both expect the + # category-keyed score shape + # ``{category: {"scores": {action_id: float}, "params": {}}}`` — the + # same shape the Expert model returns. ``_build_topology_actions`` + # yields a flat ``{topology_id: float}``; wrap it in a single + # "toop_topology" bucket so the downstream pipeline stops treating a + # bare float as a dict (``AttributeError: 'float' object has no + # attribute 'get'``). The "toop_topology" tag classifies as "other", + # so MW-at-start is left undefined (None) rather than mis-computed. return RecommenderOutput( prioritized_actions=prioritized, - action_scores=scores, + action_scores=self._nest_scores(scores), ) + @staticmethod + def _nest_scores(flat_scores: Dict[str, float]) -> Dict[str, Dict[str, Any]]: + """Wrap a flat ``{action_id: score}`` map in the category-keyed shape. + + The reassessment pipeline and ``_compute_mw_start_for_scores`` both + require ``{category: {"scores": {action_id: float}, "params": {}}}`` + (the Expert-model contract). Topology actions all live in a single + "toop_topology" bucket. Returns ``{}`` for an empty input so the + degraded path stays empty rather than carrying an empty bucket. + """ + if not flat_scores: + return {} + return {"toop_topology": {"scores": dict(flat_scores), "params": {}}} + # ------------------------------------------------------------------ # Network export + ToOp invocation # ------------------------------------------------------------------ - def _export_network(self, network: Any, iteration_dir: Path) -> Optional[Path]: + def _export_network( + self, + network: Any, + iteration_dir: Path, + contingency_lines: Optional[List[str]] = None, + ) -> Optional[Path]: """Save the live pypowsybl Network as XIIDM under ``iteration_dir``. ToOp's example notebooks consume XIIDM directly (``grid.xiidm``); no CGMES round-trip is necessary. Returns the absolute path of the saved file, or ``None`` on failure. The directory is already created by the caller. + + When ``contingency_lines`` is supplied, those branches are + disconnected on the exported copy so the grid handed to ToOp is + the operator-selected N-K state (ToOp's N-0). This is idempotent: + if the exported network was already post-contingency the flips + are no-ops. + + After saving, the file's CURRENT thermal limits are deflated by + Co-Study4Grid's ``MONITORING_FACTOR_THERMAL_LIMITS`` so ToOp's + overload-detection threshold (raw ρ ≥ 1.0) lines up with the + operator's effective threshold (ρ ≥ monitoring_factor). The + deflation is applied to a re-loaded copy — the live network the + rest of the backend uses is left untouched. """ if network is None: logger.warning("ToOpRecommender: inputs.network is None — cannot export.") @@ -315,13 +387,171 @@ def _export_network(self, network: Any, iteration_dir: Path) -> Optional[Path]: for ext in (".xiidm", ".iidm", ".xml"): hits = list(iteration_dir.glob(f"*{ext}")) if hits: - return hits[0] + out = hits[0] + break + else: + logger.warning( + "ToOpRecommender: XIIDM export produced no file under %s", + iteration_dir, + ) + return None + + if contingency_lines: + self._apply_contingency_state(out, contingency_lines) + self._deflate_thermal_limits(out) + return out + + def _apply_contingency_state(self, grid_file: Path, lines: List[str]) -> None: + """Disconnect ``lines`` in ``grid_file`` to bake in the N-K state. + + Re-opens the exported XIIDM and disconnects both terminals of + each contingency element so ToOp's base case (its N-0) matches + the operator-selected contingency. Handles lines and 2-winding + transformers; ids it cannot resolve are logged and skipped so a + single stale name doesn't abort the whole export. Operates on the + on-disk copy only — the live backend network is never mutated. + """ + try: + import pypowsybl.network as _pn + except Exception: + logger.exception( + "ToOpRecommender: pypowsybl import failed; cannot apply contingency " + "state — ToOp will run against the base (N) grid." + ) + return + + try: + net = _pn.load(str(grid_file)) + except Exception: + logger.exception( + "ToOpRecommender: failed to reload %s to apply contingency state.", + grid_file, + ) + return + + try: + line_ids = set(net.get_lines().index) + except Exception: + line_ids = set() + try: + twt_ids = set(net.get_2_windings_transformers().index) + except Exception: + twt_ids = set() + + applied = 0 + missing: List[str] = [] + for elem in lines: + try: + if elem in line_ids: + net.update_lines(id=elem, connected1=False, connected2=False) + applied += 1 + elif elem in twt_ids: + net.update_2_windings_transformers( + id=elem, connected1=False, connected2=False + ) + applied += 1 + else: + missing.append(elem) + except Exception: + logger.exception( + "ToOpRecommender: failed to disconnect contingency element %s.", + elem, + ) + missing.append(elem) + + if missing: logger.warning( - "ToOpRecommender: XIIDM export produced no file under %s", - iteration_dir, + "ToOpRecommender: could not resolve %d contingency element(s) in the " + "exported grid: %s", len(missing), missing, ) - return None - return out + if applied: + try: + net.save(str(grid_file), format="XIIDM") + except Exception: + logger.exception( + "ToOpRecommender: failed to save contingency state to %s.", + grid_file, + ) + return + logger.warning( + "ToOpRecommender: applied operator contingency (%d element(s)) to the " + "exported grid so ToOp optimises the N-K state.", applied, + ) + + def _deflate_thermal_limits(self, grid_file: Path) -> None: + """Multiply CURRENT thermal limits in ``grid_file`` by the monitoring factor. + + Aligns ToOp's overload threshold with Co-Study4Grid's effective + one: a line at ρ_raw = 0.97 with `monitoring_factor=0.95` becomes + ρ_toop = 0.97 / 0.95 ≈ 1.02 in the exported grid, so ToOp's GA + treats it as overloaded just like the operator's overload panel + does. Skips work when the factor is ≥ 1.0 (no deflation needed), + when the upstream library isn't importable, or when the network + has no operational limits. + """ + try: + from expert_op4grid_recommender import config as _config + except Exception as exc: + logger.debug( + "ToOpRecommender: expert_op4grid_recommender.config not importable " + "(%s: %s); skipping thermal-limit deflation.", + type(exc).__name__, exc, + ) + return + + factor = float(getattr(_config, "MONITORING_FACTOR_THERMAL_LIMITS", 1.0) or 1.0) + if factor >= 1.0 or factor <= 0.0: + return + + try: + import pypowsybl.network as _pn + except Exception: + logger.exception("ToOpRecommender: pypowsybl import failed; skipping deflation.") + return + + try: + net = _pn.load(str(grid_file)) + limits = net.get_operational_limits().reset_index() + except Exception: + logger.exception( + "ToOpRecommender: failed to read operational limits from %s — " + "ToOp will run against raw thermal limits.", + grid_file, + ) + return + + current_limits = limits[limits["type"] == "CURRENT"] + if current_limits.empty: + return + + # pypowsybl's update_operational_limits is strict about the + # element_id / side / type / group_name columns being plain + # strings — after reset_index() they come back as ``object`` + # dtype and `create_dataframe` rejects them. Pass via kwargs + # with explicit list[str] coercion to bypass the dtype check. + try: + net.update_operational_limits( + element_id=[str(v) for v in current_limits["element_id"]], + side=[str(v) for v in current_limits["side"]], + type=[str(v) for v in current_limits["type"]], + acceptable_duration=[int(v) for v in current_limits["acceptable_duration"]], + group_name=[str(v) for v in current_limits["group_name"]], + value=[float(v) * factor for v in current_limits["value"]], + ) + net.save(str(grid_file), format="XIIDM") + except Exception: + logger.exception( + "ToOpRecommender: failed to deflate thermal limits in %s; " + "ToOp will run against raw limits.", + grid_file, + ) + return + + logger.warning( + "ToOpRecommender: deflated %d CURRENT thermal limits by factor %.3f " + "to align ToOp's overload detection with Co-Study4Grid's monitoring factor.", + len(current_limits), factor, + ) def _run_toop( self, @@ -331,6 +561,7 @@ def _run_toop( work_dir: Path, runtime_seconds: int, n_worst_contingencies: int, + current_state_only: bool = True, ) -> Any: """Invoke ToOp's ``run_pipeline`` against an XIIDM grid file. @@ -383,7 +614,13 @@ def _run_toop( "omp_num_threads": 1, "xla_force_host_platform_device_count": None, "output_json": str(results_dir / "output.json"), - "lf_config": {"distributed": False}, + # `max_num_disconnections` defaults to 0 in ToOp's + # LoadflowSolverParameters, which disables the line-switching + # search entirely — without this override every solution is + # forced to be a busbar split (out of MVP scope). Allow up + # to 4 simultaneous disconnections so `best_topos[i].disconnections` + # is populated for the parser below to consume. + "lf_config": {"distributed": False, "max_num_disconnections": 4}, "ga_config": { "runtime_seconds": runtime_seconds, # `disconnected_branches` is ToOp's accepted metric name @@ -391,8 +628,25 @@ def _run_toop( # is pinned in BatchedMEParameters; a wrong name fails with # a pydantic ValidationError). "me_descriptors": [{"metric": "disconnected_branches", "num_cells": 4}], - "observed_metrics": ["overload_energy_n_1", "disconnected_branches"], - "n_worst_contingencies": n_worst_contingencies, + # The grid we hand ToOp is already the operator-selected + # contingency state — ToOp's N-0. When the operator wants + # to optimise that state only (no further N-1 from it, + # which would be N-2 from their viewpoint), target the + # n_0 overload metric and clamp n_worst_contingencies + # to the BatchedMEParameters minimum (it stays positive + # but stops driving the score). + "target_metrics": [ + [ + "overload_energy_n_0" if current_state_only else "overload_energy_n_1", + 1.0, + ], + ], + "observed_metrics": ( + ["overload_energy_n_0", "disconnected_branches"] + if current_state_only + else ["overload_energy_n_1", "disconnected_branches"] + ), + "n_worst_contingencies": 1 if current_state_only else n_worst_contingencies, }, }) ac_validation_cfg = DictConfig({ @@ -646,13 +900,31 @@ def _merge_topology_content( entry = _resolve_lazy_entry(enriched, key) content = _resolve_lazy_content(entry) if not content: - logger.warning( - "ToOpRecommender: could not enrich VL %s split (%d switch(es); " - "entry=%s, content=%s); excluded from this topology.", - vl_id, len(vl_switches[vl_id]), - type(entry).__name__ if entry is not None else None, - type(content).__name__ if content is not None else None, - ) + # enrich_actions_lazy could not resolve this VL's + # switch set into per-connectable set_bus assignments + # (e.g. the live network lacks the node-breaker detail + # the resolver needs). Rather than drop the split — + # which silently shrinks the topology and, when every + # VL fails, yields a 0-action result — fall back to the + # raw switch flips ToOp itself produced. They are still + # simulable: the action toggles the named switches even + # without the resolved bus assignments. + raw_switches = vl_switches[vl_id] + if raw_switches: + merged["switches"].update(raw_switches) + constituents.append(f"split {vl_id}") + logger.warning( + "ToOpRecommender: could not enrich VL %s split (%d switch(es); " + "entry=%s, content=%s); falling back to raw switch flips.", + vl_id, len(raw_switches), + type(entry).__name__ if entry is not None else None, + type(content).__name__ if content is not None else None, + ) + else: + logger.warning( + "ToOpRecommender: VL %s split has no switches to " + "synthesise; excluded from this topology.", vl_id, + ) continue set_bus = content.get("set_bus", {}) if isinstance(content, dict) else {} for sub_key in ("lines_or_id", "lines_ex_id", "loads_id", @@ -665,10 +937,17 @@ def _merge_topology_content( merged["switches"].update(sw) constituents.append(f"split {vl_id}") elif vl_switches and enrich is None: + # No enricher available — synthesise directly from the raw + # switch flips so the busbar splits still surface (matches the + # pre-rewrite behaviour) instead of being silently omitted. logger.warning( - "ToOpRecommender: enrich unavailable — %d busbar split(s) omitted " - "from this topology.", len(vl_switches), + "ToOpRecommender: enrich unavailable — synthesising %d busbar " + "split(s) from raw switch flips.", len(vl_switches), ) + for vl_id, switches in vl_switches.items(): + if switches: + merged["switches"].update(switches) + constituents.append(f"split {vl_id}") for line_id, status in line_toggles.items(): merged["set_bus"]["lines_or_id"][line_id] = status diff --git a/expert_backend/tests/test_toop_recommender.py b/expert_backend/tests/test_toop_recommender.py index 53ef4ba3..7a23c8b4 100644 --- a/expert_backend/tests/test_toop_recommender.py +++ b/expert_backend/tests/test_toop_recommender.py @@ -46,8 +46,17 @@ def test_params_spec_declares_expected_knobs(self): "include_busbar_splits", "runtime_seconds", "n_worst_contingencies", + "optimize_current_state_only", } <= names + def test_optimize_current_state_only_defaults_true(self): + descriptor = next(m for m in list_models() if m["name"] == "toop") + spec = next( + p for p in descriptor["params"] if p["name"] == "optimize_current_state_only" + ) + assert spec["kind"] == "bool" + assert spec["default"] is True + # --------------------------------------------------------------------- # Optional-install degradation @@ -191,8 +200,12 @@ def enrich_sets_line(raw, _network): # Line toggle applied last → -1 overrides the split's bus 2. assert merged["set_bus"]["lines_or_id"]["SHARED_LINE"] == -1 - def test_returns_none_when_nothing_simulable(self): - # enrich fails for the only VL and there are no line toggles. + def test_falls_back_to_raw_switches_when_enrich_fails(self): + # enrich fails to resolve content for the only VL and there are no + # line toggles. The split must NOT be dropped: we fall back to the + # raw switch flips ToOp produced so the topology stays simulable + # (otherwise every-VL-fails yields a 0-action result — see the + # "could not enrich VL …" regression). def enrich_empty(raw, _network): return {aid: {**e, "content": None} for aid, e in raw.items()} @@ -202,18 +215,236 @@ def enrich_empty(raw, _network): enrich=enrich_empty, network=MagicMock(), ) + assert merged is not None + assert merged["switches"] == {"VL_A_sw1": True} + assert constituents == ["split VL_A"] + + def test_returns_none_when_truly_empty(self): + # No line toggles and no VL switches → genuinely nothing simulable. + merged, constituents = ToOpRecommender()._merge_topology_content( + line_toggles={}, + vl_switches={}, + enrich=None, + network=MagicMock(), + ) assert merged is None assert constituents == [] - def test_enrich_unavailable_keeps_line_toggles(self): + def test_enrich_unavailable_synthesises_splits_from_raw_switches(self): + # With no enricher available, busbar splits are synthesised + # directly from the raw switch flips (matches the pre-rewrite + # behaviour) instead of being silently omitted. merged, constituents = ToOpRecommender()._merge_topology_content( line_toggles={"LINE_A": -1}, - vl_switches={"VL_A": {"VL_A_sw1": True}}, # dropped (no enrich) + vl_switches={"VL_A": {"VL_A_sw1": True}}, enrich=None, network=MagicMock(), ) assert merged["set_bus"]["lines_or_id"] == {"LINE_A": -1} - assert constituents == ["open LINE_A"] + assert merged["switches"] == {"VL_A_sw1": True} + assert set(constituents) == {"split VL_A", "open LINE_A"} + + +# --------------------------------------------------------------------- +# _nest_scores (category-keyed action_scores contract) +# --------------------------------------------------------------------- +class TestNestScores: + def test_wraps_flat_scores_in_category_bucket(self): + nested = ToOpRecommender._nest_scores( + {"toop_topology_1": 0.0, "toop_topology_2": -1.0} + ) + # Shape the reassessment pipeline depends on: + # {category: {"scores": {...}, "params": {}}}. + assert set(nested) == {"toop_topology"} + bucket = nested["toop_topology"] + assert bucket["scores"] == {"toop_topology_1": 0.0, "toop_topology_2": -1.0} + assert bucket["params"] == {} + # Every bucket value must be a dict exposing .get("scores") — this is + # exactly what propagate_non_convergence_to_scores calls. + for category in nested: + assert nested[category].get("scores", {}) is not None + + def test_empty_scores_stay_empty(self): + assert ToOpRecommender._nest_scores({}) == {} + + +# --------------------------------------------------------------------- +# _apply_contingency_state (post-contingency export — uses conftest mock) +# --------------------------------------------------------------------- +class TestApplyContingencyState: + @staticmethod + def _net(line_ids, twt_ids, recorder): + net = MagicMock() + net.get_lines.return_value = pd.DataFrame(index=list(line_ids)) + net.get_2_windings_transformers.return_value = pd.DataFrame(index=list(twt_ids)) + net.update_lines.side_effect = lambda **kw: recorder["lines"].append(kw) + net.update_2_windings_transformers.side_effect = ( + lambda **kw: recorder["twt"].append(kw) + ) + net.save.side_effect = lambda p, format=None: recorder["save"].append(str(p)) + return net + + def test_disconnects_lines_and_transformers(self, tmp_path): + import pypowsybl.network as pn_mod + + rec = {"lines": [], "twt": [], "save": []} + net = self._net(["P.SAOL31RONCI", "OTHER"], ["TWT_1"], rec) + grid = tmp_path / "grid.xiidm" + grid.write_text("") + with patch.object(pn_mod, "load", lambda p: net): + ToOpRecommender()._apply_contingency_state( + grid, ["P.SAOL31RONCI", "TWT_1"], + ) + assert rec["lines"] == [ + {"id": "P.SAOL31RONCI", "connected1": False, "connected2": False} + ] + assert rec["twt"] == [ + {"id": "TWT_1", "connected1": False, "connected2": False} + ] + assert len(rec["save"]) == 1 # saved once because elements were applied + + def test_unknown_ids_are_skipped_without_saving(self, tmp_path): + import pypowsybl.network as pn_mod + + rec = {"lines": [], "twt": [], "save": []} + net = self._net(["KNOWN"], [], rec) + grid = tmp_path / "grid.xiidm" + grid.write_text("") + with patch.object(pn_mod, "load", lambda p: net): + ToOpRecommender()._apply_contingency_state(grid, ["GHOST"]) + # Nothing resolved → no update, no save (grid left as base export). + assert rec["lines"] == [] + assert rec["save"] == [] + + +# --------------------------------------------------------------------- +# _run_toop pipeline config (ToOp engine mocked into sys.modules) +# --------------------------------------------------------------------- +class TestRunToopConfig: + @staticmethod + def _install_fake_engine(tmp_path, grid_file): + """Inject a fake toop_engine_topology_optimizer.benchmark.benchmark_utils.""" + import sys as _sys + + bu = MagicMock() + + class PipelineConfig: + static_info_relpath = "static_information.hdf5" + + def __init__(self, **kw): + self.__dict__.update(kw) + + bu.PipelineConfig = PipelineConfig + bu.PreprocessParameters = lambda **kw: MagicMock() + bu.get_paths = lambda cfg: (tmp_path, grid_file, tmp_path, tmp_path) + bu.prepare_importer_parameters = lambda fp, df: MagicMock() + mods = { + "toop_engine_topology_optimizer": MagicMock(), + "toop_engine_topology_optimizer.benchmark": MagicMock(), + "toop_engine_topology_optimizer.benchmark.benchmark_utils": bu, + } + return patch.dict(_sys.modules, mods) + + def _run(self, tmp_path, current_state_only): + grid_file = tmp_path / "grid.xiidm" + grid_file.write_text("") + captured = {} + + def fake_run_pipeline(**kw): + captured.update(kw) + return "pareto" + + with self._install_fake_engine(tmp_path, grid_file): + ToOpRecommender()._run_toop( + run_pipeline=fake_run_pipeline, + DictConfig=lambda d: d, # identity so we can inspect the dict + grid_file=grid_file, + work_dir=tmp_path, + runtime_seconds=15, + n_worst_contingencies=2, + current_state_only=current_state_only, + ) + return captured["dc_optim_config"] + + def test_line_switching_is_enabled(self, tmp_path): + # max_num_disconnections must be > 0 or ToOp produces only busbar + # splits (it defaults to 0, disabling line-switching entirely). + cfg = self._run(tmp_path, current_state_only=True) + assert cfg["lf_config"]["max_num_disconnections"] == 4 + + def test_targets_n0_when_current_state_only(self, tmp_path): + cfg = self._run(tmp_path, current_state_only=True) + ga = cfg["ga_config"] + assert ga["target_metrics"] == [["overload_energy_n_0", 1.0]] + assert ga["observed_metrics"] == ["overload_energy_n_0", "disconnected_branches"] + assert ga["n_worst_contingencies"] == 1 + + def test_targets_n1_when_not_current_state_only(self, tmp_path): + cfg = self._run(tmp_path, current_state_only=False) + ga = cfg["ga_config"] + assert ga["target_metrics"] == [["overload_energy_n_1", 1.0]] + assert ga["observed_metrics"] == ["overload_energy_n_1", "disconnected_branches"] + assert ga["n_worst_contingencies"] == 2 # passes through unchanged + + +# --------------------------------------------------------------------- +# _deflate_thermal_limits (aligns ToOp threshold with monitoring factor) +# --------------------------------------------------------------------- +class TestDeflateThermalLimits: + @staticmethod + def _net_with_limits(recorder): + net = MagicMock() + limits = pd.DataFrame( + { + "element_id": ["LINE_A", "LINE_A", "LINE_B"], + "side": ["ONE", "ONE", "TWO"], + "type": ["CURRENT", "ACTIVE_POWER", "CURRENT"], + "acceptable_duration": [-1, -1, 600], + "group_name": ["DEFAULT", "DEFAULT", "DEFAULT"], + "value": [1000.0, 50.0, 800.0], + } + ) + # get_operational_limits().reset_index() is what the code calls; + # our frame already has the columns, so reset_index is a no-op-ish. + net.get_operational_limits.return_value = limits + net.update_operational_limits.side_effect = ( + lambda **kw: recorder.update({"update": kw}) + ) + net.save.side_effect = lambda p, format=None: recorder.setdefault( + "saves", [] + ).append(str(p)) + return net + + def test_deflates_only_current_limits_by_factor(self, tmp_path): + import pypowsybl.network as pn_mod + + rec: dict = {} + net = self._net_with_limits(rec) + grid = tmp_path / "grid.xiidm" + grid.write_text("") + # conftest sets MONITORING_FACTOR_THERMAL_LIMITS = 0.95. + with patch.object(pn_mod, "load", lambda p: net): + ToOpRecommender()._deflate_thermal_limits(grid) + + # Only the two CURRENT limits are deflated (ACTIVE_POWER untouched). + assert rec["update"]["value"] == [1000.0 * 0.95, 800.0 * 0.95] + assert rec["update"]["element_id"] == ["LINE_A", "LINE_B"] + assert rec["saves"] == [str(grid)] + + def test_noop_when_factor_at_least_one(self, tmp_path): + import pypowsybl.network as pn_mod + from expert_op4grid_recommender import config as _config + + rec: dict = {} + net = self._net_with_limits(rec) + grid = tmp_path / "grid.xiidm" + grid.write_text("") + _config.MONITORING_FACTOR_THERMAL_LIMITS = 1.0 # restored by reset_config fixture + with patch.object(pn_mod, "load", lambda p: net): + ToOpRecommender()._deflate_thermal_limits(grid) + # No deflation, no save when the factor disables deflation. + assert "update" not in rec + assert "saves" not in rec # --------------------------------------------------------------------- From 2383ccc95bc91ae18b10dd55bf00bf5f716a45db Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 1 Jun 2026 21:38:37 +0200 Subject: [PATCH 14/14] feat(frontend/toop): clickable VL targets, contained badge row, neutral analysis bg --- frontend/src/components/ActionCard.tsx | 12 +++++- .../components/VisualizationPanel.test.tsx | 13 ++++--- .../src/components/VisualizationPanel.tsx | 7 +++- frontend/src/utils/svg/highlights.ts | 12 ++++++ frontend/src/utils/svgUtils.test.ts | 39 +++++++++++++++++++ 5 files changed, 73 insertions(+), 10 deletions(-) diff --git a/frontend/src/components/ActionCard.tsx b/frontend/src/components/ActionCard.tsx index ebd9271c..a6b16312 100644 --- a/frontend/src/components/ActionCard.tsx +++ b/frontend/src/components/ActionCard.tsx @@ -217,8 +217,13 @@ const ActionCard: React.FC = ({ }); } + if (badges.length === 0) return null; + // Full-width wrapping row of its own so a long target list flows + // onto multiple lines INSIDE the card instead of overflowing the + // metric row and getting clipped (ToOp topologies can target many + // voltage levels + branches at once). return ( -
+
{badges}
); @@ -344,7 +349,6 @@ const ActionCard: React.FC = ({
No loading metric
)}
- {renderBadges()}
{!isSelected && (
+ {/* Clickable target badges (VLs + branches) on their own + full-width row so long lists wrap inside the card. */} + {renderBadges()} + {/* Fault states (divergent / islanded) are primary signals and stay visible regardless of the viewing-state — they replace the missing max-loading indicator. */} diff --git a/frontend/src/components/VisualizationPanel.test.tsx b/frontend/src/components/VisualizationPanel.test.tsx index 3312db72..cbe5064e 100644 --- a/frontend/src/components/VisualizationPanel.test.tsx +++ b/frontend/src/components/VisualizationPanel.test.tsx @@ -471,7 +471,7 @@ describe('VisualizationPanel', () => { expect(screen.getAllByText('LINE_B').length).toBeGreaterThan(0); }); - it('shows analysis loading text in overflow tab with yellow theme', () => { + it('shows analysis loading text in overflow tab on a neutral (non-yellow) background', () => { render( { expect(screen.getByText('Processing Analysis...')).toBeInTheDocument(); const placeholder = screen.getByText('Processing Analysis...').parentElement; if (placeholder) { - // jsdom doesn't expand the `background:` shorthand into - // `backgroundColor` longhand for var(--…) values, so read - // the shorthand directly. - expect(placeholder.style.background).toContain('var(--color-warning-soft)'); - expect(placeholder.style.color).toContain('var(--color-warning-text)'); + // The diagram window stays neutral while the recommender runs — + // no full-window yellow flood. The spinner + text carry the + // busy signal instead. + expect(placeholder.style.background).toBe('white'); + expect(placeholder.style.background).not.toContain('var(--color-warning-soft)'); + expect(placeholder.style.color).not.toContain('var(--color-warning-text)'); } }); diff --git a/frontend/src/components/VisualizationPanel.tsx b/frontend/src/components/VisualizationPanel.tsx index 7e4914f4..793befc2 100644 --- a/frontend/src/components/VisualizationPanel.tsx +++ b/frontend/src/components/VisualizationPanel.tsx @@ -998,8 +998,11 @@ const VisualizationPanel: React.FC = ({ {!result?.pdf_url && (
diff --git a/frontend/src/utils/svg/highlights.ts b/frontend/src/utils/svg/highlights.ts index f5126fab..b5b6a1db 100644 --- a/frontend/src/utils/svg/highlights.ts +++ b/frontend/src/utils/svg/highlights.ts @@ -214,6 +214,18 @@ export const getActionTargetVoltageLevels = ( if (topo?.voltage_level_id && nodesByEquipmentId.has(topo.voltage_level_id)) { targets.add(topo.voltage_level_id); } + + // ToOp candidate topologies expose their elementary moves as + // ``constituent_ids`` labels such as "split GROSNP6" / "merge VIELMP6". + // Surface the busbar-split voltage levels so they become clickable + // card badges AND get highlighted on the diagram (this helper feeds + // both). Branch toggles like "open …"/"close …" are intentionally + // left to getActionTargetLines. + actionDetail?.constituent_ids?.forEach(c => { + const m = /^\s*(?:split|merge)\s+(.+?)\s*$/i.exec(c); + if (m && nodesByEquipmentId.has(m[1])) targets.add(m[1]); + }); + if (desc && desc !== 'No description available') { // Try all quoted strings — any might be the VL name const quotedMatches = desc.match(/'([^']+)'/g); diff --git a/frontend/src/utils/svgUtils.test.ts b/frontend/src/utils/svgUtils.test.ts index 2bdaa6d4..de622637 100644 --- a/frontend/src/utils/svgUtils.test.ts +++ b/frontend/src/utils/svgUtils.test.ts @@ -543,6 +543,45 @@ describe('getActionTargetVoltageLevels', () => { expect(result).toEqual(['VL1']); }); + it('extracts VL splits from ToOp topology constituent_ids', () => { + // ToOp candidate topology with an opaque id + no description, but + // its elementary moves name the busbar-split voltage levels. + const detail: ActionDetail = { + description_unitaire: 'No description available', + rho_before: null, + rho_after: null, + max_rho: null, + max_rho_line: '', + is_rho_reduction: false, + is_toop_topology: true, + constituent_ids: ['split GROSNP6', 'merge VIELMP6', 'open CHALOL61GROSN'], + }; + + const result = getActionTargetVoltageLevels( + detail, 'toop_topology_2', makeNodeMap('GROSNP6', 'VIELMP6'), + ); + // Both VL splits surface; the "open …" branch toggle is ignored + // (it belongs to getActionTargetLines), and only VLs that exist + // in the diagram metadata are kept. + expect(result.sort()).toEqual(['GROSNP6', 'VIELMP6']); + }); + + it('ignores constituent VL splits absent from the diagram metadata', () => { + const detail: ActionDetail = { + description_unitaire: 'No description available', + rho_before: null, + rho_after: null, + max_rho: null, + max_rho_line: '', + is_rho_reduction: false, + is_toop_topology: true, + constituent_ids: ['split GHOST_VL'], + }; + + const result = getActionTargetVoltageLevels(detail, 'toop_topology_1', makeNodeMap('GROSNP6')); + expect(result).toEqual([]); + }); + it('returns empty array when no match found', () => { const detail: ActionDetail = { description_unitaire: 'Some generic action',