From 4ca8a10f7702f51a39548e294b3efd8e0b6b3f18 Mon Sep 17 00:00:00 2001 From: Erik Alvarez Date: Fri, 12 Jun 2026 23:33:35 +0200 Subject: [PATCH 1/4] fix sentinel ens_mwh units: vENS is in GW, convert GW*h to MWh with factor 1e3 --- openTEPES/openTEPES.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openTEPES/openTEPES.py b/openTEPES/openTEPES.py index 4bbf3779..af2b03c0 100644 --- a/openTEPES/openTEPES.py +++ b/openTEPES/openTEPES.py @@ -351,8 +351,9 @@ def openTEPES_run(DirName, CaseName, SolverName, pIndOutputResults, pIndLogConso else getattr(mTEPES, "vTotalSCost", lambda: float("nan"))()) except Exception: _TotalCost = float("nan") - # ENS in MWh and HUE in hours — system-wide totals. Cheap to compute (one - # pass over vENS); skipped silently if the variable / index set is missing. + # ENS in MWh and HUE in hours — system-wide totals. vENS is in GW (the model's internal power unit), so + # GW x h is converted to MWh with a factor 1e3. Cheap to compute (one pass over vENS); skipped silently + # if the variable / index set is missing. _EnsMwh = float("nan") _HueH = float("nan") try: @@ -364,7 +365,7 @@ def openTEPES_run(DirName, CaseName, SolverName, pIndOutputResults, pIndLogConso _hue_h = 0.0 for (p, sc, n), val in _ens_psn.items(): _dur = float(mTEPES.pLoadLevelDuration[p, sc, n]()) - _ens_mwh += val * _dur + _ens_mwh += val * _dur * 1e3 if val > 0: _hue_h += _dur _EnsMwh = round(_ens_mwh, 4) From b70006451120b9c369b23d7626a5672a5f29903d Mon Sep 17 00:00:00 2001 From: Erik Alvarez Date: Mon, 15 Jun 2026 20:22:41 +0200 Subject: [PATCH 2/4] fix(solver): tolerate locked HiGHS log file on Windows re-solve The HiGHS interface keeps its log file open for the life of the highspy.Highs object, and reference cycles delay its collection. On Windows an open file cannot be deleted, so the stale-log os.remove in setup_solver crashed with WinError 32 the first time a second in-process solve reused the same log path (the Mode C 9n re-solve test). Force a GC pass to release any orphaned handle, then make the removal best-effort. --- openTEPES/openTEPES_ProblemSolvingPersistent.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/openTEPES/openTEPES_ProblemSolvingPersistent.py b/openTEPES/openTEPES_ProblemSolvingPersistent.py index 67cca4a7..dfa356b4 100644 --- a/openTEPES/openTEPES_ProblemSolvingPersistent.py +++ b/openTEPES/openTEPES_ProblemSolvingPersistent.py @@ -12,6 +12,7 @@ """ from __future__ import annotations +import gc import os import pyomo.environ as pyo @@ -28,7 +29,17 @@ def setup_solver(OptModel, SolverName: str, FileName: str, ncall: int, mTEPES): if SolverName != "appsi_gurobi" and SolverName != "gurobi_persistent": Solver = SolverFactory(SolverName) if os.path.exists(FileName): - os.remove(FileName) + # A solver instance from an earlier in-process solve (e.g. the HiGHS interface, which keeps its + # log file open for the lifetime of the underlying highspy.Highs object) can still hold this log + # file open. The interface creates reference cycles, so that prior object is not freed until a GC + # pass runs. On Windows an open file cannot be deleted, so force collection first to release any + # orphaned handle. Removal is then best-effort: if the file is still locked we leave it in place + # rather than failing the solve, since the solver truncates the log when it re-opens it. + gc.collect() + try: + os.remove(FileName) + except OSError: + pass return Solver if SolverName == "appsi_gurobi": From 3274e0e5d73c322755463fded4ddbe857c087c45 Mon Sep 17 00:00:00 2001 From: Erik Alvarez Date: Mon, 15 Jun 2026 20:37:10 +0200 Subject: [PATCH 3/4] fix: replace deprecated datetime.utcnow() with timezone-aware UTC datetime.utcnow() is deprecated in Python 3.12. Use now(timezone.utc).replace(tzinfo=None) so the emitted timestamp stays byte-identical (naive ISO string with a trailing Z, no +00:00 offset). --- openTEPES/openTEPES.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openTEPES/openTEPES.py b/openTEPES/openTEPES.py index af2b03c0..798fc620 100644 --- a/openTEPES/openTEPES.py +++ b/openTEPES/openTEPES.py @@ -138,7 +138,7 @@ def openTEPES_run(DirName, CaseName, SolverName, pIndOutputResults, pIndLogConso """ InitialTime = time.time() - _RunStartedUtc = datetime.datetime.utcnow().isoformat(timespec="seconds") + "Z" + _RunStartedUtc = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" # If the caller pointed at a .duckdb file (CaseName carries the # suffix), open it as a DuckDBSource; InputData reads tables from @@ -387,7 +387,7 @@ def openTEPES_run(DirName, CaseName, SolverName, pIndOutputResults, pIndLogConso "backend": getattr(mTEPES, "pOutputBackend", "csv"), "opentepes_version": "4.18.17RC", "run_started_utc": _RunStartedUtc, - "run_finished_utc": datetime.datetime.utcnow().isoformat(timespec="seconds") + "Z", + "run_finished_utc": datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None).isoformat(timespec="seconds") + "Z", "outputs_enabled": [k for k, v in _flags.items() if v], "gzip_patterns": list(gzip_patterns) if gzip_patterns else None, "gzip_files": _GzipFiles, From b0baf2b1029684f92f9b09e341d23da137681366 Mon Sep 17 00:00:00 2001 From: Erik Alvarez Date: Mon, 15 Jun 2026 20:46:07 +0200 Subject: [PATCH 4/4] docs: trim Mode C prose and add changelog entries (parco) --- CHANGELOG.md | 4 +++- .../openTEPES_ProblemSolvingPersistent.py | 9 +++----- openTEPES/openTEPES_ProblemSolvingResolve.py | 23 ++++++++----------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e62ff485..ebc83423 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,9 @@ ## [4.18.17RC] - 2026-06-11 Unreleased in PyPI -- [ADDED] Mode C post-build hot-swap re-solve (`openTEPES_ProblemSolvingResolve.py`). `resolve(OptModel, SolverName, overlays)` re-solves an already-built model once per parameter overlay without rebuilding it, so a sweep, sensitivity study or Monte-Carlo reuses the one slow build; `overlay_scaled` builds a scale-by-a-factor overlay. Overlays are applied relative to the baseline, not cumulatively, and the baseline is restored at the end. Only `mutable=True` Params can be hot-swapped, so the operational set is promoted to mutable: `pDemandElec`, `pENSCost`, `pLinearVarCost`, `pEFOR`, `pReserveMargin`, `pRESEnergy`. Structural and topology Params stay immutable because they are read in build-time `if` statements, where a mutable Param raises instead of returning its value (RFC Modes A / B). The six promoted Params are now read by call (`pFoo[idx]()`) in the build-time guards and in the cost and demand sums; reads inside constraints and the objective stay symbolic so the swap reaches the solver. No behaviour change: case 9n solves bit-identical (164.382043867 MEUR, `PYTHONHASHSEED=0`) and the solve suite passes with the same locked costs. New test `test_mode_c_resolve_demand_hot_swap` builds 9n once, then checks an identity overlay reproduces the cost, a +10 % demand overlay raises it, and restoring the baseline returns it. +- [FIXED] `setup_solver` no longer crashes on Windows when an earlier in-process solve still holds the stale log file open. +- [FIXED] deprecated `datetime.utcnow()` replaced with timezone-aware UTC; emitted timestamp unchanged. +- [ADDED] Mode C post-build hot-swap re-solve (`openTEPES_ProblemSolvingResolve.py`): `resolve(OptModel, SolverName, overlays)` re-solves a built model once per parameter overlay without rebuilding, so a sweep reuses the single slow build; `overlay_scaled` makes a scale-factor overlay. Overlays apply relative to the baseline, which is restored at the end. Only `mutable=True` Params hot-swap, so the operational set (`pDemandElec`, `pENSCost`, `pLinearVarCost`, `pEFOR`, `pReserveMargin`, `pRESEnergy`) is promoted to mutable and read by call in the build-time guards; structural and topology Params stay immutable. No behaviour change: 9n solves bit-identical (164.382043867 MEUR, `PYTHONHASHSEED=0`). New test `test_mode_c_resolve_demand_hot_swap`. - [CHANGED] split the ~2900-line `openTEPES_InputData.py` into three modules at the package root, one per model-build step — the input-side counterpart of the earlier Formulation, Solver and Output splits, and the last of the three mega-file splits. `openTEPES_InputData.py` keeps `InputData`, which reads the raw sets and parameters from the case source (a CSV directory or a DuckDB file). The new `openTEPES_DataConfiguration.py` holds `DataConfiguration`, which builds the derived and instrumental sets and the flag-driven branches (hydro topology, hydrogen, heat, PTDF). The new `openTEPES_SettingUpVariables.py` holds `SettingUpVariables`, which creates the decision variables and their bounds, fixes the generators' commitment, relaxes or forbids investment conditions, zeroes out epsilon values, and screens for infeasibilities. `openTEPES.py` imports each function from its own module, and `__init__.py` re-exports all three, so a user who imported `DataConfiguration` or `SettingUpVariables` from `openTEPES.openTEPES_InputData` moves to the matching module (e.g. `...DataConfiguration`). This is a pure move — the three functions are byte-identical to before, checked function by function, and none of them calls another — so results do not change: the whole solve test suite passes with the same locked costs. The two new modules import only pandas and Pyomo (no sibling `openTEPES` modules), so they need no relative-import guard; `tests/test_direct_run.py` still runs them as scripts like every other module. This finishes the Layer-3 model-build split from the architecture RFC. Separating set construction from parameter construction inside `InputData` (groundwork for the planned overlay and mutable-parameter work) is function surgery rather than a pure move and is left for its own later change. - [CHANGED] document every bundled case on the Download & Installation page (`doc/md/Download.md`). The list now covers all ten cases under `openTEPES/cases/`, not just seven: the three 9-node variants that were missing are added — `9n_PTDF` (the 9-node case solved with the PTDF network formulation instead of the angle-based DC power flow), `9n_heat` (the 9-node case coupled with a heat network), and `9n_H2` (the 9-node case coupled with a hydrogen network) — and the `sSEP` entry now notes it also includes a hydrogen network. Each entry says in one line what makes the case different, so a new user can pick the right starting point. - [CHANGED] split the ~1770-line `openTEPES_ModelFormulation.py` into six per-concern modules at the package root — the formulation-side counterpart of the earlier Input, Solver and Output splits. The two cross-sector concerns are `openTEPES_ModelFormulationObjective.py` (the total-cost objective and the per-stage operation-cost accumulation) and `openTEPES_ModelFormulationInvestment.py` (the four investment builders plus the installed-capacity, adequacy-reserve-margin and emission / RES-energy limits). Each energy sector then gets its own module: `openTEPES_ModelFormulationElectricity.py` (demand balance, operating reserves and inertia, storage, unit commitment, ramping, line switching, DC network operation and the cycle constraints), `openTEPES_ModelFormulationHydro.py` (reservoir water balance), `openTEPES_ModelFormulationHydrogen.py` (H2 network) and `openTEPES_ModelFormulationHeat.py` (heat network). The old `openTEPES_ModelFormulation.py` is removed: `openTEPES.py` imports the objective and investment builders from their modules, `openTEPES_ProblemSolvingStageIter.py` imports the per-stage operation builders, and `__init__.py` re-exports all six (so `from openTEPES.openTEPES_ModelFormulation import ...` users move to the concern module — e.g. `...Investment`). This is a pure move — the 19 builder functions are byte-identical to before, checked function by function — so results do not change: the whole solve test suite passes with the same locked costs. Keeping each sector in one file, with its existing granular per-constraint functions inside, is the groundwork for selectable formulations later (for example a DC vs AC network build, or unit commitment on or off) chosen by which functions the stage driver calls. The new modules use the descriptive-docstring convention of the other split modules; the standalone test `tests/test_heat_investment_typo.py` now imports `InvestmentHeatModelFormulation` from `...Investment`. diff --git a/openTEPES/openTEPES_ProblemSolvingPersistent.py b/openTEPES/openTEPES_ProblemSolvingPersistent.py index dfa356b4..90fc0120 100644 --- a/openTEPES/openTEPES_ProblemSolvingPersistent.py +++ b/openTEPES/openTEPES_ProblemSolvingPersistent.py @@ -29,12 +29,9 @@ def setup_solver(OptModel, SolverName: str, FileName: str, ncall: int, mTEPES): if SolverName != "appsi_gurobi" and SolverName != "gurobi_persistent": Solver = SolverFactory(SolverName) if os.path.exists(FileName): - # A solver instance from an earlier in-process solve (e.g. the HiGHS interface, which keeps its - # log file open for the lifetime of the underlying highspy.Highs object) can still hold this log - # file open. The interface creates reference cycles, so that prior object is not freed until a GC - # pass runs. On Windows an open file cannot be deleted, so force collection first to release any - # orphaned handle. Removal is then best-effort: if the file is still locked we leave it in place - # rather than failing the solve, since the solver truncates the log when it re-opens it. + # A prior in-process solve may still hold this log open (the HiGHS interface keeps it open until + # GC frees the highspy object); on Windows an open file cannot be deleted. Collect first, then + # remove best-effort: a still-locked log is left in place, since the solver truncates it on re-open. gc.collect() try: os.remove(FileName) diff --git a/openTEPES/openTEPES_ProblemSolvingResolve.py b/openTEPES/openTEPES_ProblemSolvingResolve.py index c9b846fa..8ad4cb7c 100644 --- a/openTEPES/openTEPES_ProblemSolvingResolve.py +++ b/openTEPES/openTEPES_ProblemSolvingResolve.py @@ -3,9 +3,8 @@ openTEPES.openTEPES_ProblemSolvingResolve — Mode C post-build hot-swap re-solve. -Re-solve an already-built model once per parameter overlay without rebuilding it, so a -sweep, sensitivity study or Monte-Carlo reuses the one slow build. Only ``mutable=True`` -Params can be hot-swapped: the operational set ``pDemandElec``, ``pENSCost``, +Re-solve a built model once per parameter overlay without rebuilding, so a sweep reuses the +one slow build. Only ``mutable=True`` Params hot-swap: ``pDemandElec``, ``pENSCost``, ``pLinearVarCost``, ``pEFOR``, ``pReserveMargin``, ``pRESEnergy``. See ``resolve`` for the overlay format. """ @@ -16,9 +15,9 @@ def overlay_scaled(OptModel, param_name: str, factor: float) -> dict: - """Return an overlay that scales every current value of ``param_name`` by ``factor``. + """Return an overlay scaling every current value of ``param_name`` by ``factor``. - Example: ``overlay_scaled(model, "pDemandElec", 1.10)`` for a 10 % demand uplift. + E.g. ``overlay_scaled(model, "pDemandElec", 1.10)`` for a 10 % demand uplift. """ param = getattr(OptModel, param_name) return {param_name: {idx: val * factor for idx, val in param.extract_values().items()}} @@ -27,18 +26,16 @@ def overlay_scaled(OptModel, param_name: str, factor: float) -> dict: def resolve(OptModel, SolverName: str, overlays, *, restore: bool = True, tee: bool = False): """Re-solve ``OptModel`` once per overlay, swapping mutable Param values in place. - ``overlays`` is a list of dicts, each mapping a mutable Param name to new values (a - dict ``index -> value`` for an indexed Param, or a scalar). Each overlay is applied - relative to the baseline (values at call time), not cumulatively; ``{}`` re-solves the - baseline. With ``restore`` (default) the baseline is restored at the end. ``SolverName`` - must be a non-persistent solver. Returns one dict per overlay with the termination + ``overlays`` is a list of dicts mapping a mutable Param name to new values (``index -> + value``, or a scalar). Each overlay applies relative to the baseline, not cumulatively; + ``{}`` re-solves the baseline. ``restore`` (default) restores the baseline at the end. + ``SolverName`` must be non-persistent. Returns one dict per overlay with the termination condition and total system cost (None if not optimal). """ overlays = list(overlays) - # Snapshot the baseline values of every Param any overlay touches. The snapshot serves - # two purposes: resetting the touched Params before each overlay (so overlays are - # independent, not cumulative) and the optional restore at the end of the sweep. + # Snapshot the baseline of every touched Param: used to reset before each overlay (so + # overlays are independent, not cumulative) and for the optional restore at the end. touched = set() for ov in overlays: touched.update(ov.keys())