From ef51b6468a8d23daabbc0f0fe7f18f413baa5525 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Tue, 28 Apr 2026 13:56:15 +0200 Subject: [PATCH 01/15] fix energy mix --- packages/control/chargelog/chargelog_test.py | 9 +- .../data_migration/data_migration.py | 3 +- .../measurement_logging/conftest.py | 382 ++++++++---------- .../measurement_logging/process_log.py | 222 ++++------ .../process_log_integration_test.py | 26 ++ .../measurement_logging/process_log_test.py | 79 ---- .../process_log_testdata.py | 139 +++---- .../process_log_unit_test.py | 341 ++++++++++++++++ .../test_data_analyse_percentage_totals.json | 79 ++++ .../helpermodules/utils/precision_math.py | 42 ++ 10 files changed, 798 insertions(+), 524 deletions(-) create mode 100644 packages/helpermodules/measurement_logging/process_log_integration_test.py delete mode 100644 packages/helpermodules/measurement_logging/process_log_test.py create mode 100644 packages/helpermodules/measurement_logging/process_log_unit_test.py create mode 100644 packages/helpermodules/measurement_logging/test_data_analyse_percentage_totals.json create mode 100644 packages/helpermodules/utils/precision_math.py diff --git a/packages/control/chargelog/chargelog_test.py b/packages/control/chargelog/chargelog_test.py index 774bfb260a..1dbae9aa11 100644 --- a/packages/control/chargelog/chargelog_test.py +++ b/packages/control/chargelog/chargelog_test.py @@ -62,7 +62,7 @@ def test_calc_charge_cost_reference_middle(mock_data, monkeypatch): calc_energy_costs(cp) assert cp.data.set.log.charged_energy_by_source == { - 'grid': 1243, 'pv': 386, 'bat': 671, 'cp': 0.0} + 'grid': 1242.8, 'pv': 385.8, 'bat': 671.4, 'cp': 0.0} assert round(cp.data.set.log.costs, 5) == 0.5 @@ -77,8 +77,7 @@ def test_calc_charge_cost_reference_start(mock_data, monkeypatch): with patch("builtins.open", mock_open(read_data=json.dumps(daily_log))): calc_energy_costs(cp) - assert cp.data.set.log.charged_energy_by_source == { - 'bat': 28.549999999999997, 'cp': 0.0, 'grid': 57.15, 'pv': 14.299999999999999} + assert cp.data.set.log.charged_energy_by_source == {'bat': 28.57, 'cp': 0.0, 'grid': 57.14, 'pv': 14.29} assert round(cp.data.set.log.costs, 5) == 0.025 @@ -93,7 +92,7 @@ def test_calc_charge_cost_reference_end(mock_data, monkeypatch): with patch("builtins.open", mock_open(read_data=json.dumps(daily_log))): calc_energy_costs(cp, True) - assert cp.data.set.log.charged_energy_by_source == {'bat': 699.55, 'cp': 0.0, 'grid': 1300.15, 'pv': 400.3} + assert cp.data.set.log.charged_energy_by_source == {'bat': 699.57, 'cp': 0.0, 'grid': 1300.14, 'pv': 400.29} assert round(cp.data.set.log.costs, 5) == 0.025 @@ -144,5 +143,5 @@ def test_calc_charge_cost_reference_middle_day_change(mock_data, monkeypatch): calc_energy_costs(cp) assert cp.data.set.log.charged_energy_by_source == { - 'grid': 1243, 'pv': 386, 'bat': 671, 'cp': 0.0} + 'grid': 1242.8, 'pv': 385.8, 'bat': 671.4, 'cp': 0.0} assert round(cp.data.set.log.costs, 5) == 0.5 diff --git a/packages/helpermodules/data_migration/data_migration.py b/packages/helpermodules/data_migration/data_migration.py index e80a61d2f3..f5f441a75c 100644 --- a/packages/helpermodules/data_migration/data_migration.py +++ b/packages/helpermodules/data_migration/data_migration.py @@ -26,10 +26,11 @@ from helpermodules.broker import BrokerClient from helpermodules.data_migration.id_mapping import MapId from helpermodules.hardware_configuration import update_hardware_configuration -from helpermodules.measurement_logging.process_log import get_totals, string_to_float, string_to_int +from helpermodules.measurement_logging.process_log import get_totals from helpermodules.measurement_logging.write_log import LegacySmartHomeLogData, get_names from helpermodules.timecheck import convert_timedelta_to_time_string, get_difference from helpermodules.utils import joined_thread_handler +from helpermodules.utils.precision_math import string_to_float, string_to_int from helpermodules.utils.topic_parser import get_index from helpermodules.pub import Pub from helpermodules.utils.json_file_handler import write_and_check diff --git a/packages/helpermodules/measurement_logging/conftest.py b/packages/helpermodules/measurement_logging/conftest.py index 17e3209584..f510057202 100644 --- a/packages/helpermodules/measurement_logging/conftest.py +++ b/packages/helpermodules/measurement_logging/conftest.py @@ -68,247 +68,197 @@ def daily_log_sample(): @pytest.fixture() def daily_log_totals(): - return {'bat': {'all': {'energy_exported': 550.0, 'energy_imported': 0.0}, - 'bat2': {'energy_exported': 550.0, 'energy_imported': 0.0}}, - 'counter': {'counter0': {'energy_exported': 0.0, 'energy_imported': 1492.0, 'grid': True}}, - 'cp': {'all': {'energy_exported': 0.0, 'energy_imported': 1920.0}, - 'cp3': {'energy_exported': 0.0, 'energy_imported': 1152.0}, - 'cp4': {'energy_exported': 0.0, 'energy_imported': 384.0}, - 'cp5': {'energy_exported': 0.0, 'energy_imported': 192.0}, + return {'bat': {'all': {'energy_exported': 550.857, 'energy_imported': 0.0}, + 'bat2': {'energy_exported': 550.857, 'energy_imported': 0.0}}, + 'counter': {'counter0': {'energy_exported': 0.0, 'energy_imported': 1492.011, 'grid': True}}, + 'cp': {'all': {'energy_exported': 0.0, 'energy_imported': 1919.626}, + 'cp3': {'energy_exported': 0.0, 'energy_imported': 1151.52}, + 'cp4': {'energy_exported': 0.0, 'energy_imported': 383.942}, + 'cp5': {'energy_exported': 0.0, 'energy_imported': 191.928}, 'cp6': {'energy_exported': 0.0, 'energy_imported': 0}}, 'pv': {'all': {'energy_exported': 251.0}, 'pv1': {'energy_exported': 251.0}}, - "sh": {"sh1": {"energy_imported": 0.0, "energy_exported": 0.0}}, + "sh": {"sh1": {"energy_imported": 0.3, "energy_exported": 0.0}}, "hc": {"all": {"energy_imported": 20.0}}} @pytest.fixture() -def daily_log_entry_kw(): - return {"timestamp": 1690529761, - "date": "09:35", - "cp": { - "cp3": { - "imported": 3620.971, - "exported": 0, - "power_average": 6.932, - "power_imported": 6.932, - "power_exported": 0.0, - "energy_imported": 0.576, - "energy_exported": 0.0 - }, - "cp5": { - "imported": 1208.646, - "exported": 0, - "power_average": 2.311, - "power_imported": 2.311, - "power_exported": 0.0, - "energy_imported": 0.192, - "energy_exported": 0.0 - }, - "cp4": { - "imported": 1198.566, - "exported": 0, - "power_average": 2.313, - "power_imported": 2.313, - "power_exported": 0.0, - "energy_imported": 0.192, - "energy_exported": 0.0 - }, - "all": { - "imported": 6028.183, - "exported": 0, - "power_average": 11.556, - "power_imported": 11.556, - "power_exported": 0.0, - "energy_imported": 0.96, - "energy_exported": 0.0 - } - }, "ev": { - "ev0": { - "soc": 0 - } - }, - "counter": { - "counter0": { - "imported": 4686.054, - "exported": 2.396, - "grid": True, - "power_average": 8.983, - "power_imported": 8.983, - "power_exported": 0.0, - "energy_imported": 0.746, - "energy_exported": 0.0 - } - }, - "pv": { - "pv1": { - "exported": 804, - "power_average": -1.517, - "power_imported": 0.0, - "power_exported": 1.517, - "energy_imported": 0.0, - "energy_exported": 0.126 - }, - "all": { - "exported": 804, - "power_average": -1.517, - "power_imported": 0.0, - "power_exported": 1.517, - "energy_imported": 0.0, - "energy_exported": 0.126 - } - }, - "bat": { - "bat2": { - "imported": 2.42, - "exported": 1742.135, - "soc": 15, - "power_average": -3.316, - "power_imported": 0.0, - "power_exported": 3.316, - "energy_imported": 0.0, - "energy_exported": 0.275 - }, - "all": { - "imported": 2.42, - "exported": 1742.135, - "soc": 15, - "power_average": -3.316, - "power_imported": 0.0, - "power_exported": 3.316, - "energy_imported": 0.0, - "energy_exported": 0.275 - } - }, - "sh": { - "sh1": { - "temp0": 300, - "temp1": 300, - "temp2": 300, - "imported": 0.1, - "exported": 0, - "power_average": 0.001, - "power_imported": 0.001, - "power_exported": 0.0, - "energy_imported": 0.0, - "energy_exported": 0.0 - } - }, +def daily_log_entry_processed(): + return {'bat': {'all': {'energy_exported': 275.434, + 'energy_imported': 0.0, + 'exported': 1742.135, + 'imported': 2.42, + 'power_average': -3316.262, + 'power_exported': 3316.262, + 'power_imported': 0, + 'soc': 15}, + 'bat2': {'energy_exported': 275.434, + 'energy_imported': 0.0, + 'exported': 1742.135, + 'imported': 2.42, + 'power_average': -3316.262, + 'power_exported': 3316.262, + 'power_imported': 0, + 'soc': 15}}, + 'counter': {'counter0': {'energy_exported': 0.0, + 'energy_imported': 746.123, + 'exported': 2.396, + 'grid': True, + 'imported': 4686.054, + 'power_average': 8983.421, + 'power_exported': 0, + 'power_imported': 8983.421}}, + 'cp': {'all': {'energy_exported': 0.0, + 'energy_imported': 959.813, + 'exported': 0, + 'imported': 6028.183, + 'power_average': 11556.277, + 'power_exported': 0, + 'power_imported': 11556.277}, + 'cp3': {'energy_exported': 0.0, + 'energy_imported': 575.766, + 'exported': 0, + 'imported': 3620.971, + 'power_average': 6932.3, + 'power_exported': 0, + 'power_imported': 6932.3}, + 'cp4': {'energy_exported': 0.0, + 'energy_imported': 192.119, + 'exported': 0, + 'imported': 1198.566, + 'power_average': 2313.138, + 'power_exported': 0, + 'power_imported': 2313.138}, + 'cp5': {'energy_exported': 0.0, + 'energy_imported': 191.928, + 'exported': 0, + 'imported': 1208.646, + 'power_average': 2310.839, + 'power_exported': 0, + 'power_imported': 2310.839}}, + 'date': '09:35', + 'ev': {'ev0': {'soc': 0}}, 'hc': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.01, + 'energy_imported': 10.0, 'imported': 100, - 'power_average': 0.12, - 'power_exported': 0.0, - 'power_imported': 0.12}}} + 'power_average': 120.401, + 'power_exported': 0, + 'power_imported': 120.401}}, + 'pv': {'all': {'energy_exported': 126.0, + 'energy_imported': 0.0, + 'exported': 804, + 'power_average': -1517.057, + 'power_exported': 1517.057, + 'power_imported': 0}, + 'pv1': {'energy_exported': 126.0, + 'energy_imported': 0.0, + 'exported': 804, + 'power_average': -1517.057, + 'power_exported': 1517.057, + 'power_imported': 0}}, + 'sh': {'sh1': {'energy_exported': 0.0, + 'energy_imported': 0.1, + 'exported': 0, + 'imported': 0.1, + 'power_average': 1.204, + 'power_exported': 0, + 'power_imported': 1.204, + 'temp0': 300, + 'temp1': 300, + 'temp2': 300}}, + 'timestamp': 1690529761} @pytest.fixture() -def daily_log_entry_kw_percentage(): +def daily_log_entry_percentage(): return { 'bat': {'all': {'energy_exported': 0.275, - 'energy_imported': 0.0, - 'exported': 1742.135, - 'imported': 2.42, - 'power_average': -3.316, - 'power_exported': 3.316, - 'power_imported': 0.0, - 'soc': 15}, + 'energy_imported': 0.0}, 'bat2': {'energy_exported': 0.275, - 'energy_imported': 0.0, - 'exported': 1742.135, - 'imported': 2.42, - 'power_average': -3.316, - 'power_exported': 3.316, - 'power_imported': 0.0, - 'soc': 15}}, + 'energy_imported': 0.0}}, 'counter': {'counter0': {'energy_exported': 0.0, 'energy_imported': 0.746, - 'exported': 2.396, - 'grid': True, - 'imported': 4686.054, - 'power_average': 8.983, - 'power_exported': 0.0, - 'power_imported': 8.983}}, + 'grid': True}}, 'cp': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.96, - 'energy_imported_bat': 0.23, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.624, - 'energy_imported_pv': 0.105, - 'exported': 0, - 'imported': 6028.183, - 'power_average': 11.556, - 'power_exported': 0.0, - 'power_imported': 11.556}, + 'energy_imported': 0.96}, 'cp3': {'energy_exported': 0.0, - 'energy_imported': 0.576, - 'energy_imported_bat': 0.138, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.375, - 'energy_imported_pv': 0.063, - 'exported': 0, - 'imported': 3620.971, - 'power_average': 6.932, - 'power_exported': 0.0, - 'power_imported': 6.932}, + 'energy_imported': 0.5762}, 'cp4': {'energy_exported': 0.0, - 'energy_imported': 0.192, - 'energy_imported_bat': 0.046, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.125, - 'energy_imported_pv': 0.021, - 'exported': 0, - 'imported': 1198.566, - 'power_average': 2.313, - 'power_exported': 0.0, - 'power_imported': 2.313}, + 'energy_imported': 0.192}, 'cp5': {'energy_exported': 0.0, - 'energy_imported': 0.192, - 'energy_imported_bat': 0.046, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.125, - 'energy_imported_pv': 0.021, - 'exported': 0, - 'imported': 1208.646, - 'power_average': 2.311, - 'power_exported': 0.0, - 'power_imported': 2.311}}, + 'energy_imported': 0.192}}, 'date': '09:35', - 'energy_source': {'bat': 0.2398, - 'cp': 0.0, - 'grid': 0.6504, - 'pv': 0.1098}, 'ev': {'ev0': {'soc': 0}}, 'hc': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.01, - 'energy_imported_bat': 0.002, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.007, - 'energy_imported_pv': 0.001, - 'imported': 100, - 'power_average': 0.12, - 'power_exported': 0.0, - 'power_imported': 0.12}}, + 'energy_imported': 0.01}}, 'pv': {'all': {'energy_exported': 0.126, - 'energy_imported': 0.0, - 'exported': 804, - 'power_average': -1.517, - 'power_exported': 1.517, - 'power_imported': 0.0}, + 'energy_imported': 0.0}, 'pv1': {'energy_exported': 0.126, - 'energy_imported': 0.0, - 'exported': 804, - 'power_average': -1.517, - 'power_exported': 1.517, - 'power_imported': 0.0}}, + 'energy_imported': 0.0}}, 'sh': {'sh1': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'exported': 0, - 'imported': 0.1, - 'power_average': 0.001, - 'power_exported': 0.0, - 'power_imported': 0.001, - 'temp0': 300, - 'temp1': 300, - 'temp2': 300}}, + 'energy_imported': 0.0}}, + 'timestamp': 1690529761, + } + + +@pytest.fixture() +def daily_log_entry_percentage_negative_consumption(): + return { + 'bat': {'all': {'energy_exported': 0.275, + 'energy_imported': 0.0}, + 'bat2': {'energy_exported': 0.275, + 'energy_imported': 0.0}}, + 'counter': {'counter0': {'energy_exported': 2, + 'energy_imported': 0.746, + 'grid': True}}, + 'cp': {'all': {'energy_exported': 0.0, + 'energy_imported': 0.96}, + 'cp3': {'energy_exported': 0.0, + 'energy_imported': 0.5762}, + 'cp4': {'energy_exported': 0.0, + 'energy_imported': 0.192}, + 'cp5': {'energy_exported': 0.0, + 'energy_imported': 0.192}}, + 'date': '09:35', + 'ev': {'ev0': {'soc': 0}}, + 'hc': {'all': {'energy_exported': 0.0, + 'energy_imported': 0.01}}, + 'pv': {'all': {'energy_exported': 0.15, + 'energy_imported': 0.0}, + 'pv1': {'energy_exported': 0.15, + 'energy_imported': 0.0}}, + 'sh': {'sh1': {'energy_exported': 0.0, + 'energy_imported': 0.0}}, + 'timestamp': 1690529761, + } + + +@pytest.fixture() +def daily_log_entry_percentage_cp_discharge(): + return { + 'bat': {'all': {'energy_exported': 0.275, + 'energy_imported': 0.0}, + 'bat2': {'energy_exported': 0.275, + 'energy_imported': 0.0}}, + 'counter': {'counter0': {'energy_exported': 0.0, + 'energy_imported': 0.746, + 'grid': True}}, + 'cp': {'all': {'energy_exported': 0.2, + 'energy_imported': 0.96}, + 'cp3': {'energy_exported': 0.0, + 'energy_imported': 0.5762}, + 'cp4': {'energy_exported': 0.0, + 'energy_imported': 0.192}, + 'cp5': {'energy_exported': 0.0, + 'energy_imported': 0.392}}, + 'date': '09:35', + 'ev': {'ev0': {'soc': 0}}, + 'hc': {'all': {'energy_exported': 0.0, + 'energy_imported': 0.01}}, + 'pv': {'all': {'energy_exported': 0.15, + 'energy_imported': 0.0}, + 'pv1': {'energy_exported': 0.15, + 'energy_imported': 0.0}}, + 'sh': {'sh1': {'energy_exported': 0.0, + 'energy_imported': 0.0}}, 'timestamp': 1690529761, } diff --git a/packages/helpermodules/measurement_logging/process_log.py b/packages/helpermodules/measurement_logging/process_log.py index d54c4da024..5b742f6490 100644 --- a/packages/helpermodules/measurement_logging/process_log.py +++ b/packages/helpermodules/measurement_logging/process_log.py @@ -1,5 +1,3 @@ -import datetime -from decimal import Decimal from enum import Enum import json import logging @@ -10,6 +8,7 @@ from helpermodules.measurement_logging.write_log import (LegacySmartHomeLogData, LogType, create_entry, get_previous_entry) from helpermodules.messaging import MessageType, pub_system_message +from helpermodules.utils.precision_math import decimal_add, decimal_multiply, decimal_subtract log = logging.getLogger(__name__) @@ -48,18 +47,14 @@ def get_default_charge_log_columns() -> Dict: } -def string_to_float(value: str, default: float = 0) -> float: - try: - return float(value) - except ValueError: - return default - - -def string_to_int(value: str, default: int = 0) -> int: - try: - return int(value) - except ValueError: - return default +def safe_get_nested(data: Dict, *keys, default: Union[int, float] = 0) -> Union[int, float]: + current = data + for key in keys: + if isinstance(current, dict) and key in current: + current = current[key] + else: + return default + return current if isinstance(current, (int, float)) else default def get_totals(entries: List, process_entries: bool = True) -> Dict: @@ -88,13 +83,9 @@ def get_totals(entries: List, process_entries: bool = True) -> Dict: for entry_module_key, entry_module_value in entry[totals_group][entry_module].items(): if "grid" != entry_module_key and entry_module_key in totals[totals_group][entry_module]: # avoid floating point issues with using Decimal - value = (Decimal(str(totals[totals_group][entry_module][entry_module_key])) - + Decimal(str(entry_module_value * 1000))) # totals in Wh! - value.quantize(Decimal('0.001')) - value = f'{value: f}' - # remove trailing zeros - totals[totals_group][entry_module][entry_module_key] = string_to_float( - value) if "." in value else string_to_int(value) + current_total = totals[totals_group][entry_module][entry_module_key] + totals[totals_group][entry_module][entry_module_key] = decimal_add( + current_total, entry_module_value) # totals in Wh! except Exception: log.exception(f"Fehler beim Berechnen der Summe von {entry_module}; " @@ -194,7 +185,7 @@ def _collect_daily_log_data(date: str): except FILE_ERRORS: pass except FILE_ERRORS: - log_data = {"entries": [], "totals": {}, "names": {}} + log_data = {"entries": [], "names": {}} return log_data @@ -233,7 +224,7 @@ def _collect_monthly_log_data(date: str): except FILE_ERRORS: pass except FILE_ERRORS: - log_data = {"entries": [], "totals": {}, "names": {}} + log_data = {"entries": [], "names": {}} return log_data @@ -245,66 +236,6 @@ def get_yearly_log(year: str): return data -def get_log_from_date_until_now(timestamp: int): - data = {} - try: - entries = _collect_log_data_from_date_until_now(timestamp) - data["entries"] = _process_entries(entries, CalculationType.ENERGY) - data["totals"] = get_totals(data["entries"], False) - data = _analyse_energy_source(data) - except Exception: - log.exception(f"Fehler beim Zusammenstellen der Logdaten von {timestamp}") - finally: - return data - - -def _collect_log_data_from_date_until_now(timestamp: int): - def add_to_list(log_data: List, data: Union[Dict, List]): - if isinstance(data, list): - log_data.extend(data) - else: - log_data.append(data) - return log_data - log_data = [] - try: - date = datetime.datetime.fromtimestamp(timestamp).strftime("%Y%m%d") - try: - with open(f"{_get_data_folder_path()}/daily_log/{date}.json", "r") as jsonFile: - entries = json.load(jsonFile)["entries"] - except FILE_ERRORS: - pass - for index, entry in enumerate(entries): - if entry["timestamp"] > timestamp: - log_data = add_to_list(log_data, entries[index:]) - break - else: - try: - # Wenn der Ladevorgang nicht über volle 5 Minuten ging, wurde während dem Laden kein Eintrag ins - # daily-log geschrieben. - log_data = add_to_list(log_data, entries[-1]) - except KeyError: - log.exception(f"Fehler beim Zusammenstellen der Logdaten. Bitte Logdatei daily_log/{date}.json prüfen.") - # Das Teillog vom ersten Tag wurde bereits ermittelt. - start_date = datetime.datetime.fromtimestamp(timestamp) + datetime.timedelta(days=1) - end_date = datetime.datetime.now() - current_date = start_date - date_list = [] - while current_date <= end_date: - date_list.append(current_date.strftime('%Y%m%d')) - current_date += datetime.timedelta(days=1) - for date_str in date_list: - try: - with open(f"{_get_data_folder_path()}/daily_log/{date_str}.json", "r") as jsonFile: - log_data = add_to_list(log_data, json.load(jsonFile)["entries"]) - except FILE_ERRORS: - pass - log_data = add_to_list(log_data, create_entry(LogType.DAILY, LegacySmartHomeLogData(), log_data[-1])) - except Exception: - log.exception(f"Fehler beim Zusammenstellen der Logdaten von {timestamp}") - finally: - return log_data - - def _collect_yearly_log_data(year: str): def add_monthly_log(month: str, check_next_month: bool = False) -> None: monthly_log_path = Path(__file__).resolve().parents[3]/"data"/"monthly_log" @@ -382,6 +313,7 @@ def _analyse_energy_source(data) -> Dict: try: for i in range(0, len(data["entries"])): data["entries"][i] = analyse_percentage(data["entries"][i]) + data["entries"][i] = calc_energy_imported_by_source(data["entries"][i]) data["totals"] = analyse_percentage_totals(data["entries"], data["totals"]) except Exception: pub_system_message({}, "Fehler beim Berechnen des Strom-Mix", MessageType.ERROR) @@ -398,65 +330,57 @@ def get_grid_from(entry) -> Tuple[float, float]: raise KeyError(f"Kein Zähler für das Netz gefunden in Eintrag '{entry['timestamp']}'.") return sum(grid["energy_imported"] for grid in grids), sum(grid["energy_exported"] for grid in grids) - def calc_energy_imported_by_source(energy_imported, energy_source): - value = (Decimal(str(energy_imported)) * - Decimal(str(energy_source))).quantize(Decimal('0.001')) # limit precision - value = f'{value: f}' - value = string_to_float(value) if "." in value else string_to_int(value) - return value - try: - bat_imported = entry["bat"]["all"]["energy_imported"] if "all" in entry["bat"].keys() else 0 - bat_exported = entry["bat"]["all"]["energy_exported"] if "all" in entry["bat"].keys() else 0 - cp_exported = entry["cp"]["all"]["energy_exported"] if "all" in entry["cp"].keys() else 0 - pv = entry["pv"]["all"]["energy_exported"] if "all" in entry["pv"].keys() else 0 + bat_imported = safe_get_nested(entry, "bat", "all", "energy_imported") + bat_exported = safe_get_nested(entry, "bat", "all", "energy_exported") + cp_exported = safe_get_nested(entry, "cp", "all", "energy_exported") + pv_exported = safe_get_nested(entry, "pv", "all", "energy_exported") grid_imported, grid_exported = get_grid_from(entry) - consumption = grid_imported - grid_exported + pv + bat_exported - bat_imported + cp_exported - for type in ("bat", "cp"): - if entry[type]["all"]["energy_imported"] > consumption: - consumption += entry[type]["all"]["energy_imported"] - consumption - grid_imported += entry[type]["all"]["energy_imported"] - grid_imported - log.debug(f"Angepasste Verbrauchswerte für {type} um " - f"{entry[type]['all']['energy_imported'] - consumption} kWh") - for counter in entry["counter"].values(): - if counter["grid"] is False: - if counter["energy_imported"] > consumption: - consumption += counter["energy_imported"] - consumption - grid_imported += counter["energy_imported"] - grid_imported - log.debug(f"Angepasste Verbrauchswerte für {type} um " - f"{entry[type]['all']['energy_imported'] - consumption} kWh") + consumption = grid_imported - grid_exported + pv_exported + bat_exported - bat_imported + cp_exported + if consumption < 0: + consumption = 0 + try: - if grid_exported > pv: - # Ins Netz eingespeiste Leistung kam nicht von der PV-Anlage sondern aus dem Speicher - consumption += grid_exported - pv - elif bat_imported > pv: - # Die geladene Energie des Speichers kam nicht von der PV-Anlage sondern aus dem Netz - consumption += bat_imported - pv - grid_energy_source = format(grid_imported / consumption) - cp_energy_source = format(cp_exported/consumption) - bat_energy_source = format(bat_exported/consumption) - pv_energy_source = format(1 - grid_energy_source - bat_energy_source - cp_energy_source) + pv_direct = min(pv_exported, consumption) + remaining = consumption - pv_direct + + bat_direct = min(bat_exported, remaining) + remaining -= bat_direct + + cp_direct = min(cp_exported, remaining) + remaining -= cp_direct + + grid_direct = min(grid_imported, remaining) + entry["energy_source"] = { - "grid": grid_energy_source, - "pv": pv_energy_source, - "bat": bat_energy_source, - "cp": cp_energy_source} + "grid": format(grid_direct / consumption), + "pv": format(pv_direct / consumption), + "bat": format(bat_direct / consumption), + "cp": format(cp_direct / consumption)} except ZeroDivisionError: entry["energy_source"] = {"grid": 0, "pv": 0, "bat": 0, "cp": 0} + except Exception: + log.exception(f"Fehler beim Berechnen des Strom-Mix von {entry['timestamp']}") + finally: + return entry + + +def calc_energy_imported_by_source(entry): + + try: for source in ("grid", "pv", "bat", "cp"): if "all" in entry["hc"].keys(): - entry["hc"]["all"][f"energy_imported_{source}"] = calc_energy_imported_by_source( + entry["hc"]["all"][f"energy_imported_{source}"] = decimal_multiply( entry["hc"]["all"]["energy_imported"], entry["energy_source"][source]) for key in entry["cp"].keys(): - entry["cp"][key][f"energy_imported_{source}"] = calc_energy_imported_by_source( + entry["cp"][key][f"energy_imported_{source}"] = decimal_multiply( entry["cp"][key]["energy_imported"], entry["energy_source"][source]) for counter in entry["counter"].values(): if counter["grid"] is False: - counter[f"energy_imported_{source}"] = calc_energy_imported_by_source( + counter[f"energy_imported_{source}"] = decimal_multiply( counter["energy_imported"], entry["energy_source"][source]) - except Exception: - log.exception(f"Fehler beim Berechnen des Strom-Mix von {entry['timestamp']}") + log.exception(f"Fehler beim Berechnen der Engergie-Anteile aus dem Strom-Mix von {entry['timestamp']}") finally: return entry @@ -469,18 +393,26 @@ def analyse_percentage_totals(entries, totals): totals["hc"]["all"].update({f"energy_imported_{source}": 0}) for entry in entries: if "hc" in entry.keys() and "all" in entry["hc"].keys(): - totals["hc"]["all"][f"energy_imported_{source}"] += entry["hc"]["all"].get( - f"energy_imported_{source}", 0)*1000 + current_value = totals["hc"]["all"][f"energy_imported_{source}"] + add_value = entry["hc"]["all"].get(f"energy_imported_{source}", 0) + totals["hc"]["all"][f"energy_imported_{source}"] = decimal_add( + current_value, add_value) for key in entry["cp"].keys(): if f"energy_imported_{source}" in entry["cp"][key].keys(): if totals["cp"][key].get(f"energy_imported_{source}") is None: totals["cp"][key].update({f"energy_imported_{source}": 0}) - totals["cp"][key][f"energy_imported_{source}"] += entry["cp"][key][f"energy_imported_{source}"]*1000 + current_value = totals["cp"][key][f"energy_imported_{source}"] + add_value = entry["cp"][key][f"energy_imported_{source}"] + totals["cp"][key][f"energy_imported_{source}"] = decimal_add( + current_value, add_value) for key, counter in entry["counter"].items(): if counter["grid"] is False: if totals["counter"][key].get(f"energy_imported_{source}") is None: totals["counter"][key].update({f"energy_imported_{source}": 0}) - totals["counter"][key][f"energy_imported_{source}"] += counter[f"energy_imported_{source}"]*1000 + current_value = totals["counter"][key][f"energy_imported_{source}"] + add_value = counter[f"energy_imported_{source}"] + totals["counter"][key][f"energy_imported_{source}"] = decimal_add( + current_value, add_value) return totals @@ -536,8 +468,8 @@ def get_single_value(source: dict, default: int = 0) -> float: average_power = 0 else: average_power = _calculate_average_power( - time_diff, value_imported / 1000, next_value_imported / 1000, - value_exported / 1000, next_value_exported / 1000) + time_diff, value_imported, next_value_imported, + value_exported, next_value_exported) new_data.update({ "power_average": average_power, "power_imported": average_power if average_power >= 0 else 0, @@ -548,14 +480,14 @@ def get_single_value(source: dict, default: int = 0) -> float: # do not calculate as we have a backwards jump in our meter value! energy_imported = 0 else: - energy_imported = _calculate_energy_difference(value_imported / 1000, - next_value_imported / 1000) + energy_imported = decimal_subtract(next_value_imported, + value_imported) if next_value_exported < value_exported: # do not calculate as we have a backwards jump in our meter value! energy_exported = 0 else: - energy_exported = _calculate_energy_difference(value_exported / 1000, - next_value_exported / 1000) + energy_exported = decimal_subtract(next_value_exported, + value_exported) new_data.update({ "energy_imported": energy_imported, "energy_exported": energy_exported @@ -579,20 +511,12 @@ def get_single_value(source: dict, default: int = 0) -> float: return entry -def _calculate_energy_difference(current_value: float, next_value: float) -> float: - value = (Decimal(str(next_value)) - Decimal(str(current_value))) - value = value.quantize(Decimal('0.001')) # limit precision - value = f'{value: f}' - return string_to_float(value) if "." in value else string_to_int(value) - - def _calculate_average_power(time_diff: float, current_imported: float = 0, next_imported: float = 0, current_exported: float = 0, next_exported: float = 0) -> float: - power = (Decimal(str(next_imported)) - Decimal(str(current_imported)) - - (Decimal(str(next_exported)) - Decimal(str(current_exported)))) * Decimal(str(3600 / time_diff)) # Ws - power = power.quantize(Decimal('0.001')) # limit precision - power = f'{power: f}' - return string_to_float(power) if "." in power else string_to_int(power) + imported_diff = decimal_subtract(next_imported, current_imported) + exported_diff = decimal_subtract(next_exported, current_exported) + energy_diff = decimal_subtract(imported_diff, exported_diff) + return decimal_multiply(energy_diff, 3600 / time_diff) # Ws -> W def _get_data_folder_path() -> str: diff --git a/packages/helpermodules/measurement_logging/process_log_integration_test.py b/packages/helpermodules/measurement_logging/process_log_integration_test.py new file mode 100644 index 0000000000..540740ef2b --- /dev/null +++ b/packages/helpermodules/measurement_logging/process_log_integration_test.py @@ -0,0 +1,26 @@ +from pprint import pprint +from unittest.mock import Mock +import pytest + +from helpermodules.measurement_logging import process_log + +from helpermodules.measurement_logging.process_log_testdata import (counter_jumps_forward, + counter_jumps_forward_processed, + regular_daily_log_entry, + regular_daily_log_entry_processed) + + +@pytest.mark.parametrize("data, expected", [ + pytest.param(counter_jumps_forward, counter_jumps_forward_processed, id="counter jumps forward"), + pytest.param(regular_daily_log_entry, regular_daily_log_entry_processed, id="regular daily log entry") +]) +def test_get_daily_log(data, expected, monkeypatch): + # setup + collect_daily_log_data_mock = Mock(return_value=data) + monkeypatch.setattr(process_log, "_collect_daily_log_data", collect_daily_log_data_mock) + + # execution + daily_log_processed = process_log.get_daily_log("20250616") + pprint(daily_log_processed) + # evaluation + assert daily_log_processed == expected diff --git a/packages/helpermodules/measurement_logging/process_log_test.py b/packages/helpermodules/measurement_logging/process_log_test.py deleted file mode 100644 index 0bf4e53d52..0000000000 --- a/packages/helpermodules/measurement_logging/process_log_test.py +++ /dev/null @@ -1,79 +0,0 @@ -from copy import deepcopy -from unittest.mock import Mock -import pytest - -from helpermodules.measurement_logging import process_log -from helpermodules.measurement_logging.process_log import ( - analyse_percentage, - _calculate_average_power, - process_entry, - get_totals, - CalculationType) - -from helpermodules.measurement_logging.process_log_testdata import (counter_jumps_forward, - counter_jumps_forward_processed, - regular_daily_log_entry, - regular_daily_log_entry_processed) - - -def test_get_totals(daily_log_sample, daily_log_totals): - # setup and execution - entries = deepcopy(daily_log_sample) - totals = get_totals(entries) - - # evaluation - assert totals == daily_log_totals - - -def test_analyse_percentage(daily_log_entry_kw_percentage): - # setup - expected = deepcopy(daily_log_entry_kw_percentage) - expected.update({"energy_source": {'bat': 0.2398, 'cp': 0.0, 'grid': 0.6504, 'pv': 0.1098}}) - expected["cp"]["all"].update({ - "energy_imported_bat": 0.23, - "energy_imported_cp": 0.0, - "energy_imported_grid": 0.624, - "energy_imported_pv": 0.105}) - expected["hc"]["all"].update({ - "energy_imported_bat": 0.002, - "energy_imported_cp": 0.0, - "energy_imported_grid": 0.007, - "energy_imported_pv": 0.001}) - - # execution - entry = analyse_percentage(daily_log_entry_kw_percentage) - - # evaluation - assert entry == expected - - -def test_convert_value_to_kW(): - # setup and execution - power = _calculate_average_power(100, 250, 300) - - # evaluation - assert power == 1800 - - -def test_convert(daily_log_entry_kw, daily_log_sample): - # setup and execution - entry = process_entry(daily_log_sample[0], daily_log_sample[1], CalculationType.ALL) - - # evaluation - assert entry == daily_log_entry_kw - - -@pytest.mark.parametrize("data, expected", [ - pytest.param(counter_jumps_forward, counter_jumps_forward_processed, id="counter jumps forward"), - pytest.param(regular_daily_log_entry, regular_daily_log_entry_processed, id="regular daily log entry") -]) -def test_get_daily_log(data, expected, monkeypatch): - # setup - collect_daily_log_data_mock = Mock(return_value=data) - monkeypatch.setattr(process_log, "_collect_daily_log_data", collect_daily_log_data_mock) - - # execution - daily_log_processed = process_log.get_daily_log("20250616") - - # evaluation - assert daily_log_processed == expected diff --git a/packages/helpermodules/measurement_logging/process_log_testdata.py b/packages/helpermodules/measurement_logging/process_log_testdata.py index 147b6f3765..37d05a3c7f 100644 --- a/packages/helpermodules/measurement_logging/process_log_testdata.py +++ b/packages/helpermodules/measurement_logging/process_log_testdata.py @@ -1,4 +1,3 @@ -# Wenn ein Zwischenzähler nicht auslesbar war, soll beim Sprung der Anteil auf das Netz gerechnet werden. counter_jumps_forward = {'entries': [{'bat': {'all': {'exported': 3195.13, 'imported': 629.37, 'soc': 48}, @@ -7,7 +6,7 @@ 'soc': 48}}, 'counter': {'counter0': {'exported': 26029.945, 'grid': True, - 'imported': 2728.572}, + 'imported': 0}, 'counter2': {'exported': 26029.945, 'grid': False, 'imported': 0}}, @@ -30,12 +29,12 @@ 'soc': 48}}, 'counter': {'counter0': {'exported': 26802.355, 'grid': True, - 'imported': 2728.572}, + 'imported': 0}, 'counter2': {'exported': 26029.945, - 'grid': True, - 'imported': 2728.572}}, - 'cp': {'all': {'exported': 0, 'imported': 12639.11}, - 'cp3': {'exported': 0, 'imported': 12639.11}, + 'grid': False, + 'imported': 0}}, + 'cp': {'all': {'exported': 0, 'imported': 20639.11}, + 'cp3': {'exported': 0, 'imported': 20639.11}, 'cp4': {'exported': 0, 'imported': 0}, 'cp5': {'exported': 0, 'imported': 0}}, 'date': '14:30', @@ -70,48 +69,48 @@ 'power_exported': 0, 'power_imported': 0.0, 'soc': 48}}, - 'counter': {'counter0': {'energy_exported': 0.772, + 'counter': {'counter0': {'energy_exported': 772.41, 'energy_imported': 0.0, 'exported': 26029.945, 'grid': True, - 'imported': 2728.572, - 'power_average': -9.3, - 'power_exported': 9.3, + 'imported': 0, + 'power_average': -9299.92, + 'power_exported': 9299.92, 'power_imported': 0}, 'counter2': {'energy_exported': 0.0, - 'energy_imported': 2.729, + 'energy_imported': 0.0, 'energy_imported_bat': 0.0, 'energy_imported_cp': 0.0, - 'energy_imported_grid': 2.729, + 'energy_imported_grid': 0.0, 'energy_imported_pv': 0.0, 'exported': 26029.945, 'grid': False, 'imported': 0, - 'power_average': 32.852, + 'power_average': 0.0, 'power_exported': 0, - 'power_imported': 32.852}}, + 'power_imported': 0.0}}, 'cp': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.0, + 'energy_imported': 8000.0, 'energy_imported_bat': 0.0, 'energy_imported_cp': 0.0, 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0, + 'energy_imported_pv': 8000.0, 'exported': 0, 'imported': 12639.11, - 'power_average': 0.0, + 'power_average': 96321.07, 'power_exported': 0, - 'power_imported': 0.0}, + 'power_imported': 96321.07}, 'cp3': {'energy_exported': 0.0, - 'energy_imported': 0.0, + 'energy_imported': 8000.0, 'energy_imported_bat': 0.0, 'energy_imported_cp': 0.0, 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0, + 'energy_imported_pv': 8000.0, 'exported': 0, 'imported': 12639.11, - 'power_average': 0.0, + 'power_average': 96321.07, 'power_exported': 0, - 'power_imported': 0.0}, + 'power_imported': 96321.07}, 'cp4': {'energy_exported': 0.0, 'energy_imported': 0.0, 'energy_imported_bat': 0.0, @@ -135,30 +134,30 @@ 'power_exported': 0, 'power_imported': 0.0}}, 'date': '14:25', - 'energy_source': {'bat': 0.0, 'cp': 0.0, 'grid': 1.0, 'pv': 0.0}, + 'energy_source': {'bat': 0.0, 'cp': 0.0, 'grid': 0.0, 'pv': 1.0}, 'ev': {'ev0': {'soc': None}}, 'hc': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.037, + 'energy_imported': 37.177, 'energy_imported_bat': 0.0, 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.037, - 'energy_imported_pv': 0.0, + 'energy_imported_grid': 0.0, + 'energy_imported_pv': 37.177, 'imported': 2324.001611140539, - 'power_average': 0.448, + 'power_average': 447.616, 'power_exported': 0, - 'power_imported': 0.448}}, + 'power_imported': 447.616}}, 'prices': {'bat': 0.0002, 'grid': 0.0003, 'pv': 0.00015}, - 'pv': {'all': {'energy_exported': 0.809, + 'pv': {'all': {'energy_exported': 809.0, 'energy_imported': 0.0, 'exported': 35827, - 'power_average': -9.74, - 'power_exported': 9.74, + 'power_average': -9740.468, + 'power_exported': 9740.468, 'power_imported': 0}, - 'pv1': {'energy_exported': 0.809, + 'pv1': {'energy_exported': 809.0, 'energy_imported': 0.0, 'exported': 35827, - 'power_average': -9.74, - 'power_exported': 9.74, + 'power_average': -9740.468, + 'power_exported': 9740.468, 'power_imported': 0}}, 'sh': {}, 'timestamp': 1750767902}], @@ -172,28 +171,28 @@ 'pv1': 'MQTT-Wechselrichter'}, 'totals': {'bat': {'all': {'energy_exported': 0.0, 'energy_imported': 0.0}, 'bat2': {'energy_exported': 0.0, 'energy_imported': 0.0}}, - 'counter': {'counter0': {'energy_exported': 772.0, + 'counter': {'counter0': {'energy_exported': 772.41, 'energy_imported': 0.0, 'grid': True}, 'counter2': {'energy_exported': 0.0, - 'energy_imported': 2729.0, + 'energy_imported': 0.0, 'energy_imported_bat': 0.0, 'energy_imported_cp': 0.0, - 'energy_imported_grid': 2729.0, + 'energy_imported_grid': 0.0, 'energy_imported_pv': 0.0, 'grid': False}}, 'cp': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.0, + 'energy_imported': 8000.0, 'energy_imported_bat': 0.0, 'energy_imported_cp': 0.0, 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0}, + 'energy_imported_pv': 8000.0}, 'cp3': {'energy_exported': 0.0, - 'energy_imported': 0.0, + 'energy_imported': 8000.0, 'energy_imported_bat': 0.0, 'energy_imported_cp': 0.0, 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0}, + 'energy_imported_pv': 8000.0}, 'cp4': {'energy_exported': 0.0, 'energy_imported': 0.0, 'energy_imported_bat': 0.0, @@ -206,11 +205,11 @@ 'energy_imported_cp': 0.0, 'energy_imported_grid': 0.0, 'energy_imported_pv': 0.0}}, - 'hc': {'all': {'energy_imported': 37.0, + 'hc': {'all': {'energy_imported': 37.177, 'energy_imported_bat': 0.0, 'energy_imported_cp': 0.0, - 'energy_imported_grid': 37.0, - 'energy_imported_pv': 0.0}}, + 'energy_imported_grid': 0.0, + 'energy_imported_pv': 37.177}}, 'pv': {'all': {'energy_exported': 809.0}, 'pv1': {'energy_exported': 809.0}}, 'sh': {}}} @@ -280,13 +279,13 @@ 'power_exported': 0, 'power_imported': 0.0, 'soc': 48}}, - 'counter': {'counter0': {'energy_exported': 0.772, + 'counter': {'counter0': {'energy_exported': 772.41, 'energy_imported': 0.0, 'exported': 26029.945, 'grid': True, 'imported': 2728.572, - 'power_average': -9.3, - 'power_exported': 9.3, + 'power_average': -9299.92, + 'power_exported': 9299.92, 'power_imported': 0}}, 'cp': {'all': {'energy_exported': 0.0, 'energy_imported': 0.0, @@ -333,35 +332,30 @@ 'power_exported': 0, 'power_imported': 0.0}}, 'date': '14:25', - 'energy_source': {'bat': 0.0, - 'cp': 0.0, - 'grid': 0.0, - 'pv': 1.0}, + 'energy_source': {'bat': 0.0, 'cp': 0.0, 'grid': 0.0, 'pv': 1.0}, 'ev': {'ev0': {'soc': None}}, 'hc': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.037, + 'energy_imported': 37.177, 'energy_imported_bat': 0.0, 'energy_imported_cp': 0.0, 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.037, + 'energy_imported_pv': 37.177, 'imported': 2324.001611140539, - 'power_average': 0.448, + 'power_average': 447.616, 'power_exported': 0, - 'power_imported': 0.448}}, - 'prices': {'bat': 0.0002, - 'grid': 0.0003, - 'pv': 0.00015}, - 'pv': {'all': {'energy_exported': 0.809, + 'power_imported': 447.616}}, + 'prices': {'bat': 0.0002, 'grid': 0.0003, 'pv': 0.00015}, + 'pv': {'all': {'energy_exported': 809.0, 'energy_imported': 0.0, 'exported': 35827, - 'power_average': -9.74, - 'power_exported': 9.74, + 'power_average': -9740.468, + 'power_exported': 9740.468, 'power_imported': 0}, - 'pv1': {'energy_exported': 0.809, + 'pv1': {'energy_exported': 809.0, 'energy_imported': 0.0, 'exported': 35827, - 'power_average': -9.74, - 'power_exported': 9.74, + 'power_average': -9740.468, + 'power_exported': 9740.468, 'power_imported': 0}}, 'sh': {}, 'timestamp': 1750767902}], @@ -373,11 +367,9 @@ 'cp5': 'MQTT-Ladepunkt', 'ev0': 'Standard-Fahrzeug', 'pv1': 'MQTT-Wechselrichter'}, - 'totals': {'bat': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.0}, - 'bat2': {'energy_exported': 0.0, - 'energy_imported': 0.0}}, - 'counter': {'counter0': {'energy_exported': 772.0, + 'totals': {'bat': {'all': {'energy_exported': 0.0, 'energy_imported': 0.0}, + 'bat2': {'energy_exported': 0.0, 'energy_imported': 0.0}}, + 'counter': {'counter0': {'energy_exported': 772.41, 'energy_imported': 0.0, 'grid': True}}, 'cp': {'all': {'energy_exported': 0.0, @@ -404,12 +396,11 @@ 'energy_imported_cp': 0.0, 'energy_imported_grid': 0.0, 'energy_imported_pv': 0.0}}, - 'hc': {'all': {'energy_imported': 37.0, + 'hc': {'all': {'energy_imported': 37.177, 'energy_imported_bat': 0.0, 'energy_imported_cp': 0.0, 'energy_imported_grid': 0.0, - 'energy_imported_pv': 37.0}}, + 'energy_imported_pv': 37.177}}, 'pv': {'all': {'energy_exported': 809.0}, 'pv1': {'energy_exported': 809.0}}, - 'sh': {}}, - } + 'sh': {}}} diff --git a/packages/helpermodules/measurement_logging/process_log_unit_test.py b/packages/helpermodules/measurement_logging/process_log_unit_test.py new file mode 100644 index 0000000000..ab25b4ccd5 --- /dev/null +++ b/packages/helpermodules/measurement_logging/process_log_unit_test.py @@ -0,0 +1,341 @@ +from copy import deepcopy +import json +import os +from pprint import pprint +from typing import Dict +from unittest.mock import Mock, mock_open +import pytest + +from helpermodules.measurement_logging.process_log import ( + analyse_percentage, + _calculate_average_power, + process_entry, + get_totals, + _collect_daily_log_data, + calc_energy_imported_by_source, + analyse_percentage_totals, + CalculationType) + + +def test_get_totals(daily_log_sample, daily_log_totals): + # setup and execution + entries = deepcopy(daily_log_sample) + totals = get_totals(entries) + + # evaluation + assert totals == daily_log_totals + + +@pytest.mark.parametrize("name", + ["regular", + "negative consumption", + "cp export"]) +def test_analyse_percentage(name: str, + daily_log_entry_percentage: Dict, + daily_log_entry_percentage_negative_consumption: Dict, + daily_log_entry_percentage_cp_discharge: Dict): + # setup + if name == "regular": + data = daily_log_entry_percentage + expected = deepcopy(data) + expected.update({"energy_source": {'bat': 0.2398, 'cp': 0.0, 'grid': 0.6504, 'pv': 0.1099}}) + elif name == "negative consumption": + data = daily_log_entry_percentage_negative_consumption + expected = deepcopy(data) + expected.update({"energy_source": {'bat': 0.0, 'cp': 0.0, 'grid': 0.0, 'pv': 0.0}}) + elif name == "cp export": + data = daily_log_entry_percentage_cp_discharge + expected = deepcopy(data) + expected.update({"energy_source": {'bat': 0.2006, 'cp': 0.1459, 'grid': 0.5441, 'pv': 0.1094}}) + + # execution + entry = analyse_percentage(data) + + # evaluation + assert entry == expected + + +@pytest.mark.parametrize("test_case, entry_data, expected_energy_source, should_be_unchanged", [ + ( + "zero_consumption", + { + "timestamp": 1234567890, + "bat": {"all": {"energy_imported": 5.0, "energy_exported": 5.0}}, + "cp": {"all": {"energy_exported": 0.0}}, + "pv": {"all": {"energy_exported": 0.0}}, + "counter": {"counter0": {"grid": True, "energy_imported": 5.0, "energy_exported": 5.0}} + }, + {"grid": 0, "pv": 0, "bat": 0, "cp": 0}, + False + ), + ( + "missing_sections", + { + "timestamp": 1234567890, + "counter": {"counter0": {"grid": True, "energy_imported": 10.0, "energy_exported": 2.0}} + }, + {"grid": 1.0, "pv": 0.0, "bat": 0.0, "cp": 0.0}, + False + ), + ( + "no_grid_counter", + { + "timestamp": 1234567890, + "bat": {"all": {"energy_imported": 0.0, "energy_exported": 5.0}}, + "counter": {"counter0": {"grid": False, "energy_imported": 10.0, "energy_exported": 2.0}} + }, + None, + True + ) +]) +def test_analyse_percentage_edge_cases(test_case, entry_data, expected_energy_source, should_be_unchanged): + # execution + result = analyse_percentage(entry_data) + + # evaluation + if should_be_unchanged: + # Entry should be unchanged, no energy_source added due to error + assert result["timestamp"] == entry_data["timestamp"] + assert "energy_source" not in result or result.get("energy_source") is None + else: + # Energy source should be calculated correctly + assert result["energy_source"] == expected_energy_source + + +def test_convert_value_to_kW(): + # setup and execution + power = _calculate_average_power(100, 250, 300) + + # evaluation + assert power == 1800 + + +def test_calc_energy_imported_by_source(): + # setup + entry = { + "timestamp": 1234567890, + "energy_source": {"grid": 0.6523, "pv": 0.2487, "bat": 0.0789, "cp": 0.0201}, + "hc": {"all": {"energy_imported": 2345.6}}, # Wh + "cp": { + "cp1": {"energy_imported": 15723.4}, # Wh + "cp2": {"energy_imported": 22108.7} # Wh + }, + "counter": { + "counter0": {"grid": True, "energy_imported": 45892.3}, # Wh + "counter1": {"grid": False, "energy_imported": 8956.7} # Wh + } + } + + # execution + result = calc_energy_imported_by_source(entry) + + # evaluation - realistic Wh values with decimal precision + assert result["hc"]["all"]["energy_imported_grid"] == 1530.035 # 2345.6 * 0.6523 + assert result["hc"]["all"]["energy_imported_pv"] == 583.351 # 2345.6 * 0.2487 + assert result["hc"]["all"]["energy_imported_bat"] == 185.068 # 2345.6 * 0.0789 + assert result["hc"]["all"]["energy_imported_cp"] == 47.147 # 2345.6 * 0.0201 + + assert result["cp"]["cp1"]["energy_imported_grid"] == 10256.374 # 15723.4 * 0.6523 + assert result["cp"]["cp1"]["energy_imported_pv"] == 3910.41 # 15723.4 * 0.2487 + assert result["cp"]["cp1"]["energy_imported_bat"] == 1240.576 # 15723.4 * 0.0789 + assert result["cp"]["cp1"]["energy_imported_cp"] == 316.04 # 15723.4 * 0.0201 + + assert result["cp"]["cp2"]["energy_imported_grid"] == 14421.505 # 22108.7 * 0.6523 + assert result["cp"]["cp2"]["energy_imported_pv"] == 5498.434 # 22108.7 * 0.2487 + assert result["cp"]["cp2"]["energy_imported_bat"] == 1744.376 # 22108.7 * 0.0789 + assert result["cp"]["cp2"]["energy_imported_cp"] == 444.385 # 22108.7 * 0.0201 + + assert result["counter"]["counter1"]["energy_imported_grid"] == 5842.455 # 8956.7 * 0.6523 + assert result["counter"]["counter1"]["energy_imported_pv"] == 2227.531 # 8956.7 * 0.2487 + assert result["counter"]["counter1"]["energy_imported_bat"] == 706.684 # 8956.7 * 0.0789 + assert result["counter"]["counter1"]["energy_imported_cp"] == 180.03 # 8956.7 * 0.0201 + # counter0 should not have these fields as it's a grid counter + assert "energy_imported_grid" not in result["counter"]["counter0"] + assert "energy_imported_pv" not in result["counter"]["counter0"] + assert "energy_imported_bat" not in result["counter"]["counter0"] + assert "energy_imported_cp" not in result["counter"]["counter0"] + + +def test_analyse_percentage_totals(): + # setup + current_dir = os.path.dirname(os.path.abspath(__file__)) + json_path = os.path.join(current_dir, 'test_data_analyse_percentage_totals.json') + with open(json_path, 'r') as f: + entries = json.load(f) + + totals = { + "hc": {"all": {"energy_imported": 3500}}, # realistic household total in Wh + "cp": { + "cp1": {"energy_imported": 18500}, # realistic EV charging total in Wh + "cp2": {"energy_imported": 29000}, # another EV total in Wh + "cp3": {"energy_imported": 11433} # another EV total in Wh + }, + "counter": { + "counter0": {"grid": True, "energy_imported": 45892}, # grid counter total in Wh + "counter1": {"grid": False, "energy_imported": 10000}, # sub-meter total in Wh + "counter2": {"grid": False, "energy_imported": 8735} # another sub-meter total in Wh + } + } + + # execution + result = analyse_percentage_totals(entries, totals) + + # evaluation + # Check hc totals (sum of both entries in Wh) + assert result["hc"]["all"]["energy_imported_grid"] == 7909 # (4820.5 + 3087.9) Wh = 7908.4 Wh + assert result["hc"]["all"]["energy_imported_pv"] == 2980 # (1734.2 + 1245.6) Wh = 2979.8 Wh + assert result["hc"]["all"]["energy_imported_bat"] == 912 # (287.8 + 623.7) Wh = 911.5 Wh + assert result["hc"]["all"]["energy_imported_cp"] == 273 # (95.3 + 178.4) Wh = 273.7 Wh + + # Check cp totals (in Wh) + assert result["cp"]["cp1"]["energy_imported_grid"] == 22222 # (12345.7 + 9876.2) Wh = 22221.9 Wh + assert result["cp"]["cp1"]["energy_imported_pv"] == 8354 # (4632.1 + 3721.8) Wh = 8353.9 Wh + assert result["cp"]["cp1"]["energy_imported_bat"] == 3300 # (1876.4 + 1423.5) Wh = 3299.9 Wh + assert result["cp"]["cp1"]["energy_imported_cp"] == 802 # (234.6 + 567.1) Wh = 801.7 Wh + + assert result["cp"]["cp2"]["energy_imported_grid"] == 18721 # 18721.3 Wh (only in first entry) + assert result["cp"]["cp2"]["energy_imported_pv"] == 7124 # 7123.8 Wh + assert result["cp"]["cp2"]["energy_imported_bat"] == 2955 # 2954.7 Wh + assert result["cp"]["cp2"]["energy_imported_cp"] == 313 # 312.9 Wh + + # 11432.6 Wh (only in second entry) + assert result["cp"]["cp3"]["energy_imported_grid"] == 11433 + assert result["cp"]["cp3"]["energy_imported_pv"] == 4824 # 4823.9 Wh + assert result["cp"]["cp3"]["energy_imported_bat"] == 2135 # 2134.7 Wh + assert result["cp"]["cp3"]["energy_imported_cp"] == 689 # 689.2 Wh + + # Check counter totals (in Wh, only non-grid counters) + assert result["counter"]["counter1"]["energy_imported_grid"] == 12158 # (6234.8 + 5923.1) Wh = 12157.9 Wh + assert result["counter"]["counter1"]["energy_imported_pv"] == 5123 # (2387.5 + 2734.8) Wh = 5122.3 Wh + assert result["counter"]["counter1"]["energy_imported_bat"] == 2011 # (923.4 + 1087.6) Wh = 2011.0 Wh + assert result["counter"]["counter1"]["energy_imported_cp"] == 744 # (445.7 + 298.3) Wh = 744.0 Wh + + # 8734.5 Wh (only in second entry) + assert result["counter"]["counter2"]["energy_imported_grid"] == 8735 + assert result["counter"]["counter2"]["energy_imported_pv"] == 3290 # 3289.7 Wh + assert result["counter"]["counter2"]["energy_imported_bat"] == 1634 # 1634.2 Wh + assert result["counter"]["counter2"]["energy_imported_cp"] == 824 # 823.6 Wh + + +def test_convert(daily_log_entry_processed, daily_log_sample): + # setup and execution + entry = process_entry(daily_log_sample[0], daily_log_sample[1], CalculationType.ALL) + pprint(entry) + # evaluation + assert entry == daily_log_entry_processed + + +def test_collect_daily_log_data_current_day(monkeypatch): + # setup + test_date = "20240422" + mock_log_data = { + "entries": [{"timestamp": 1234567890, "data": "test"}], + "names": {} + } + mock_current_entry = {"timestamp": 1234567999, "data": "current"} + + mock_timecheck = Mock() + mock_timecheck.create_timestamp_YYYYMMDD.return_value = test_date + monkeypatch.setattr('helpermodules.measurement_logging.process_log.timecheck', mock_timecheck) + + mock_json_load = Mock(return_value=mock_log_data) + monkeypatch.setattr('helpermodules.measurement_logging.process_log.json.load', mock_json_load) + + mock_create_entry = Mock(return_value=mock_current_entry) + monkeypatch.setattr('helpermodules.measurement_logging.process_log.create_entry', mock_create_entry) + + mock_get_previous_entry = Mock(return_value={"timestamp": 1234567800, "data": "previous"}) + monkeypatch.setattr('helpermodules.measurement_logging.process_log.get_previous_entry', mock_get_previous_entry) + + monkeypatch.setattr('builtins.open', mock_open(read_data=json.dumps(mock_log_data))) + + # execution + result = _collect_daily_log_data(test_date) + + # evaluation + expected_result = { + "entries": [ + {"timestamp": 1234567890, "data": "test"}, + {"timestamp": 1234567999, "data": "current"} + ], + "names": {} + } + assert result == expected_result + + +def test_collect_daily_log_data_past_date_with_next_day(monkeypatch): + # setup + test_date = "20240422" + next_date = "20240423" + mock_current_log_data = { + "entries": [{"timestamp": 1234567890, "data": "test"}], + "names": {} + } + mock_next_log_data = { + "entries": [{"timestamp": 1234567999, "data": "next_day"}] + } + + mock_timecheck = Mock() + mock_timecheck.create_timestamp_YYYYMMDD.return_value = "20240425" + mock_timecheck.get_relative_date_string.return_value = next_date + monkeypatch.setattr('helpermodules.measurement_logging.process_log.timecheck', mock_timecheck) + + mock_json_load = Mock(side_effect=[mock_current_log_data, mock_next_log_data]) + monkeypatch.setattr('helpermodules.measurement_logging.process_log.json.load', mock_json_load) + + monkeypatch.setattr('builtins.open', mock_open()) + + # execution + result = _collect_daily_log_data(test_date) + + # evaluation + expected_result = { + "entries": [ + {"timestamp": 1234567890, "data": "test"}, + {"timestamp": 1234567999, "data": "next_day"} + ], + "names": {} + } + assert result == expected_result + + +def test_collect_daily_log_data_file_not_found(monkeypatch): + # setup + test_date = "20240422" + + mock_timecheck = Mock() + mock_timecheck.create_timestamp_YYYYMMDD.return_value = "20240425" + monkeypatch.setattr('helpermodules.measurement_logging.process_log.timecheck', mock_timecheck) + + def mock_open_side_effect(*args, **kwargs): + raise FileNotFoundError() + + monkeypatch.setattr('builtins.open', mock_open_side_effect) + + # execution + result = _collect_daily_log_data(test_date) + + # evaluation + expected_result = {"entries": [], "names": {}} + assert result == expected_result + + +def test_collect_daily_log_data_json_decode_error(monkeypatch): + # setup + test_date = "20240422" + + mock_timecheck = Mock() + mock_timecheck.create_timestamp_YYYYMMDD.return_value = "20240425" + monkeypatch.setattr('helpermodules.measurement_logging.process_log.timecheck', mock_timecheck) + + mock_json_load = Mock(side_effect=json.JSONDecodeError("msg", "doc", 1)) + monkeypatch.setattr('helpermodules.measurement_logging.process_log.json.load', mock_json_load) + + monkeypatch.setattr('builtins.open', mock_open(read_data="invalid json")) + + # execution + result = _collect_daily_log_data(test_date) + + # evaluation + expected_result = {"entries": [], "names": {}} + assert result == expected_result diff --git a/packages/helpermodules/measurement_logging/test_data_analyse_percentage_totals.json b/packages/helpermodules/measurement_logging/test_data_analyse_percentage_totals.json new file mode 100644 index 0000000000..1e57d2b268 --- /dev/null +++ b/packages/helpermodules/measurement_logging/test_data_analyse_percentage_totals.json @@ -0,0 +1,79 @@ +[ + { + "hc": { + "all": { + "energy_imported_grid": 4821, + "energy_imported_pv": 1734, + "energy_imported_bat": 288, + "energy_imported_cp": 95 + } + }, + "cp": { + "cp1": { + "energy_imported_grid": 12346, + "energy_imported_pv": 4632, + "energy_imported_bat": 1876, + "energy_imported_cp": 235 + }, + "cp2": { + "energy_imported_grid": 18721, + "energy_imported_pv": 7124, + "energy_imported_bat": 2955, + "energy_imported_cp": 313 + } + }, + "counter": { + "counter0": { + "grid": true, + "energy_imported": 48367 + }, + "counter1": { + "grid": false, + "energy_imported_grid": 6235, + "energy_imported_pv": 2388, + "energy_imported_bat": 923, + "energy_imported_cp": 446 + } + } + }, + { + "hc": { + "all": { + "energy_imported_grid": 3088, + "energy_imported_pv": 1246, + "energy_imported_bat": 624, + "energy_imported_cp": 178 + } + }, + "cp": { + "cp1": { + "energy_imported_grid": 9876, + "energy_imported_pv": 3722, + "energy_imported_bat": 1424, + "energy_imported_cp": 567 + }, + "cp3": { + "energy_imported_grid": 11433, + "energy_imported_pv": 4824, + "energy_imported_bat": 2135, + "energy_imported_cp": 689 + } + }, + "counter": { + "counter1": { + "grid": false, + "energy_imported_grid": 5923, + "energy_imported_pv": 2735, + "energy_imported_bat": 1088, + "energy_imported_cp": 298 + }, + "counter2": { + "grid": false, + "energy_imported_grid": 8735, + "energy_imported_pv": 3290, + "energy_imported_bat": 1634, + "energy_imported_cp": 824 + } + } + } +] \ No newline at end of file diff --git a/packages/helpermodules/utils/precision_math.py b/packages/helpermodules/utils/precision_math.py new file mode 100644 index 0000000000..6e335c6a1e --- /dev/null +++ b/packages/helpermodules/utils/precision_math.py @@ -0,0 +1,42 @@ +from decimal import Decimal +from typing import Union + + +def string_to_float(value: str, default: float = 0) -> float: + """Convert string to float with fallback to default value.""" + try: + return float(value) + except ValueError: + return default + + +def string_to_int(value: str, default: int = 0) -> int: + """Convert string to int with fallback to default value.""" + try: + return int(value) + except ValueError: + return default + + +def _decimal_to_number(decimal_value: Decimal) -> Union[int, float]: + """Convert Decimal to int or float, removing trailing zeros.""" + value_str = f'{decimal_value: f}' + return string_to_float(value_str) if "." in value_str else string_to_int(value_str) + + +def decimal_add(current_value: Union[int, float], add_value: Union[int, float]) -> Union[int, float]: + """Add two values using Decimal to avoid floating point issues.""" + result = (Decimal(str(current_value)) + Decimal(str(add_value))).quantize(Decimal('0.001')) + return _decimal_to_number(result) + + +def decimal_multiply(value1: Union[int, float], value2: Union[int, float]) -> Union[int, float]: + """Multiply two values using Decimal to avoid floating point issues.""" + result = (Decimal(str(value1)) * Decimal(str(value2))).quantize(Decimal('0.001')) + return _decimal_to_number(result) + + +def decimal_subtract(value1: Union[int, float], value2: Union[int, float]) -> Union[int, float]: + """Subtract two values using Decimal to avoid floating point issues.""" + result = (Decimal(str(value1)) - Decimal(str(value2))).quantize(Decimal('0.001')) + return _decimal_to_number(result) From b3a374bb2569a05d1e50a9ffa6b2593b7f7fcd01 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Tue, 28 Apr 2026 14:51:48 +0200 Subject: [PATCH 02/15] review --- packages/helpermodules/measurement_logging/process_log.py | 5 +++-- .../measurement_logging/process_log_integration_test.py | 3 +-- .../measurement_logging/process_log_unit_test.py | 3 +-- packages/helpermodules/utils/precision_math.py | 6 ++++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/helpermodules/measurement_logging/process_log.py b/packages/helpermodules/measurement_logging/process_log.py index 5b742f6490..85989427df 100644 --- a/packages/helpermodules/measurement_logging/process_log.py +++ b/packages/helpermodules/measurement_logging/process_log.py @@ -366,8 +366,9 @@ def get_grid_from(entry) -> Tuple[float, float]: def calc_energy_imported_by_source(entry): - try: + if "energy_source" not in entry.keys(): + return for source in ("grid", "pv", "bat", "cp"): if "all" in entry["hc"].keys(): entry["hc"]["all"][f"energy_imported_{source}"] = decimal_multiply( @@ -380,7 +381,7 @@ def calc_energy_imported_by_source(entry): counter[f"energy_imported_{source}"] = decimal_multiply( counter["energy_imported"], entry["energy_source"][source]) except Exception: - log.exception(f"Fehler beim Berechnen der Engergie-Anteile aus dem Strom-Mix von {entry['timestamp']}") + log.exception(f"Fehler beim Berechnen der Energie-Anteile aus dem Strom-Mix von {entry['timestamp']}") finally: return entry diff --git a/packages/helpermodules/measurement_logging/process_log_integration_test.py b/packages/helpermodules/measurement_logging/process_log_integration_test.py index 540740ef2b..3bb1156e0a 100644 --- a/packages/helpermodules/measurement_logging/process_log_integration_test.py +++ b/packages/helpermodules/measurement_logging/process_log_integration_test.py @@ -1,4 +1,3 @@ -from pprint import pprint from unittest.mock import Mock import pytest @@ -21,6 +20,6 @@ def test_get_daily_log(data, expected, monkeypatch): # execution daily_log_processed = process_log.get_daily_log("20250616") - pprint(daily_log_processed) + # evaluation assert daily_log_processed == expected diff --git a/packages/helpermodules/measurement_logging/process_log_unit_test.py b/packages/helpermodules/measurement_logging/process_log_unit_test.py index ab25b4ccd5..537e79a1b6 100644 --- a/packages/helpermodules/measurement_logging/process_log_unit_test.py +++ b/packages/helpermodules/measurement_logging/process_log_unit_test.py @@ -1,7 +1,6 @@ from copy import deepcopy import json import os -from pprint import pprint from typing import Dict from unittest.mock import Mock, mock_open import pytest @@ -220,7 +219,7 @@ def test_analyse_percentage_totals(): def test_convert(daily_log_entry_processed, daily_log_sample): # setup and execution entry = process_entry(daily_log_sample[0], daily_log_sample[1], CalculationType.ALL) - pprint(entry) + # evaluation assert entry == daily_log_entry_processed diff --git a/packages/helpermodules/utils/precision_math.py b/packages/helpermodules/utils/precision_math.py index 6e335c6a1e..97017a3e32 100644 --- a/packages/helpermodules/utils/precision_math.py +++ b/packages/helpermodules/utils/precision_math.py @@ -20,8 +20,10 @@ def string_to_int(value: str, default: int = 0) -> int: def _decimal_to_number(decimal_value: Decimal) -> Union[int, float]: """Convert Decimal to int or float, removing trailing zeros.""" - value_str = f'{decimal_value: f}' - return string_to_float(value_str) if "." in value_str else string_to_int(value_str) + normalized_value = decimal_value.normalize() + value_str = format(normalized_value, 'f') + return (string_to_int(value_str) if normalized_value == normalized_value.to_integral() + else string_to_float(value_str)) def decimal_add(current_value: Union[int, float], add_value: Union[int, float]) -> Union[int, float]: From 2eb8117751dabbad70ef366563453c82bd6cc823 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 29 Apr 2026 08:50:48 +0200 Subject: [PATCH 03/15] review --- .../measurement_logging/process_log.py | 37 ++++-- .../process_log_unit_test.py | 108 +++++++++--------- .../helpermodules/utils/precision_math.py | 4 +- 3 files changed, 82 insertions(+), 67 deletions(-) diff --git a/packages/helpermodules/measurement_logging/process_log.py b/packages/helpermodules/measurement_logging/process_log.py index 85989427df..b512bff035 100644 --- a/packages/helpermodules/measurement_logging/process_log.py +++ b/packages/helpermodules/measurement_logging/process_log.py @@ -368,18 +368,33 @@ def get_grid_from(entry) -> Tuple[float, float]: def calc_energy_imported_by_source(entry): try: if "energy_source" not in entry.keys(): - return + return entry + + energy_source = entry["energy_source"] for source in ("grid", "pv", "bat", "cp"): - if "all" in entry["hc"].keys(): - entry["hc"]["all"][f"energy_imported_{source}"] = decimal_multiply( - entry["hc"]["all"]["energy_imported"], entry["energy_source"][source]) - for key in entry["cp"].keys(): - entry["cp"][key][f"energy_imported_{source}"] = decimal_multiply( - entry["cp"][key]["energy_imported"], entry["energy_source"][source]) - for counter in entry["counter"].values(): - if counter["grid"] is False: - counter[f"energy_imported_{source}"] = decimal_multiply( - counter["energy_imported"], entry["energy_source"][source]) + # Handle hc section + hc_section = entry.get("hc") + if isinstance(hc_section, dict) and "all" in hc_section: + hc_all = hc_section["all"] + if isinstance(hc_all, dict) and "energy_imported" in hc_all: + hc_all[f"energy_imported_{source}"] = decimal_multiply( + hc_all["energy_imported"], energy_source[source]) + + # Handle cp section + cp_section = entry.get("cp") + if isinstance(cp_section, dict): + for cp_data in cp_section.values(): + if isinstance(cp_data, dict) and "energy_imported" in cp_data: + cp_data[f"energy_imported_{source}"] = decimal_multiply( + cp_data["energy_imported"], energy_source[source]) + + # Handle counter section + counter_section = entry.get("counter") + if isinstance(counter_section, dict): + for counter in counter_section.values(): + if isinstance(counter, dict) and counter.get("grid") is False and "energy_imported" in counter: + counter[f"energy_imported_{source}"] = decimal_multiply( + counter["energy_imported"], energy_source[source]) except Exception: log.exception(f"Fehler beim Berechnen der Energie-Anteile aus dem Strom-Mix von {entry['timestamp']}") finally: diff --git a/packages/helpermodules/measurement_logging/process_log_unit_test.py b/packages/helpermodules/measurement_logging/process_log_unit_test.py index 537e79a1b6..d8bdac6446 100644 --- a/packages/helpermodules/measurement_logging/process_log_unit_test.py +++ b/packages/helpermodules/measurement_logging/process_log_unit_test.py @@ -101,7 +101,7 @@ def test_analyse_percentage_edge_cases(test_case, entry_data, expected_energy_so assert result["energy_source"] == expected_energy_source -def test_convert_value_to_kW(): +def test_calculate_average_power(): # setup and execution power = _calculate_average_power(100, 250, 300) @@ -114,14 +114,14 @@ def test_calc_energy_imported_by_source(): entry = { "timestamp": 1234567890, "energy_source": {"grid": 0.6523, "pv": 0.2487, "bat": 0.0789, "cp": 0.0201}, - "hc": {"all": {"energy_imported": 2345.6}}, # Wh + "hc": {"all": {"energy_imported": 2345.6}}, "cp": { - "cp1": {"energy_imported": 15723.4}, # Wh - "cp2": {"energy_imported": 22108.7} # Wh + "cp1": {"energy_imported": 15723.4}, + "cp2": {"energy_imported": 22108.7} }, "counter": { - "counter0": {"grid": True, "energy_imported": 45892.3}, # Wh - "counter1": {"grid": False, "energy_imported": 8956.7} # Wh + "counter0": {"grid": True, "energy_imported": 45892.3}, + "counter1": {"grid": False, "energy_imported": 8956.7} } } @@ -129,25 +129,25 @@ def test_calc_energy_imported_by_source(): result = calc_energy_imported_by_source(entry) # evaluation - realistic Wh values with decimal precision - assert result["hc"]["all"]["energy_imported_grid"] == 1530.035 # 2345.6 * 0.6523 - assert result["hc"]["all"]["energy_imported_pv"] == 583.351 # 2345.6 * 0.2487 - assert result["hc"]["all"]["energy_imported_bat"] == 185.068 # 2345.6 * 0.0789 - assert result["hc"]["all"]["energy_imported_cp"] == 47.147 # 2345.6 * 0.0201 - - assert result["cp"]["cp1"]["energy_imported_grid"] == 10256.374 # 15723.4 * 0.6523 - assert result["cp"]["cp1"]["energy_imported_pv"] == 3910.41 # 15723.4 * 0.2487 - assert result["cp"]["cp1"]["energy_imported_bat"] == 1240.576 # 15723.4 * 0.0789 - assert result["cp"]["cp1"]["energy_imported_cp"] == 316.04 # 15723.4 * 0.0201 - - assert result["cp"]["cp2"]["energy_imported_grid"] == 14421.505 # 22108.7 * 0.6523 - assert result["cp"]["cp2"]["energy_imported_pv"] == 5498.434 # 22108.7 * 0.2487 - assert result["cp"]["cp2"]["energy_imported_bat"] == 1744.376 # 22108.7 * 0.0789 - assert result["cp"]["cp2"]["energy_imported_cp"] == 444.385 # 22108.7 * 0.0201 - - assert result["counter"]["counter1"]["energy_imported_grid"] == 5842.455 # 8956.7 * 0.6523 - assert result["counter"]["counter1"]["energy_imported_pv"] == 2227.531 # 8956.7 * 0.2487 - assert result["counter"]["counter1"]["energy_imported_bat"] == 706.684 # 8956.7 * 0.0789 - assert result["counter"]["counter1"]["energy_imported_cp"] == 180.03 # 8956.7 * 0.0201 + assert result["hc"]["all"]["energy_imported_grid"] == 1530.035 + assert result["hc"]["all"]["energy_imported_pv"] == 583.351 + assert result["hc"]["all"]["energy_imported_bat"] == 185.068 + assert result["hc"]["all"]["energy_imported_cp"] == 47.147 + + assert result["cp"]["cp1"]["energy_imported_grid"] == 10256.374 + assert result["cp"]["cp1"]["energy_imported_pv"] == 3910.41 + assert result["cp"]["cp1"]["energy_imported_bat"] == 1240.576 + assert result["cp"]["cp1"]["energy_imported_cp"] == 316.04 + + assert result["cp"]["cp2"]["energy_imported_grid"] == 14421.505 + assert result["cp"]["cp2"]["energy_imported_pv"] == 5498.434 + assert result["cp"]["cp2"]["energy_imported_bat"] == 1744.376 + assert result["cp"]["cp2"]["energy_imported_cp"] == 444.385 + + assert result["counter"]["counter1"]["energy_imported_grid"] == 5842.455 + assert result["counter"]["counter1"]["energy_imported_pv"] == 2227.531 + assert result["counter"]["counter1"]["energy_imported_bat"] == 706.684 + assert result["counter"]["counter1"]["energy_imported_cp"] == 180.03 # counter0 should not have these fields as it's a grid counter assert "energy_imported_grid" not in result["counter"]["counter0"] assert "energy_imported_pv" not in result["counter"]["counter0"] @@ -163,16 +163,16 @@ def test_analyse_percentage_totals(): entries = json.load(f) totals = { - "hc": {"all": {"energy_imported": 3500}}, # realistic household total in Wh + "hc": {"all": {"energy_imported": 3500}}, "cp": { - "cp1": {"energy_imported": 18500}, # realistic EV charging total in Wh - "cp2": {"energy_imported": 29000}, # another EV total in Wh - "cp3": {"energy_imported": 11433} # another EV total in Wh + "cp1": {"energy_imported": 18500}, + "cp2": {"energy_imported": 29000}, + "cp3": {"energy_imported": 11433} }, "counter": { - "counter0": {"grid": True, "energy_imported": 45892}, # grid counter total in Wh - "counter1": {"grid": False, "energy_imported": 10000}, # sub-meter total in Wh - "counter2": {"grid": False, "energy_imported": 8735} # another sub-meter total in Wh + "counter0": {"grid": True, "energy_imported": 45892}, + "counter1": {"grid": False, "energy_imported": 10000}, + "counter2": {"grid": False, "energy_imported": 8735} } } @@ -181,39 +181,39 @@ def test_analyse_percentage_totals(): # evaluation # Check hc totals (sum of both entries in Wh) - assert result["hc"]["all"]["energy_imported_grid"] == 7909 # (4820.5 + 3087.9) Wh = 7908.4 Wh - assert result["hc"]["all"]["energy_imported_pv"] == 2980 # (1734.2 + 1245.6) Wh = 2979.8 Wh - assert result["hc"]["all"]["energy_imported_bat"] == 912 # (287.8 + 623.7) Wh = 911.5 Wh - assert result["hc"]["all"]["energy_imported_cp"] == 273 # (95.3 + 178.4) Wh = 273.7 Wh + assert result["hc"]["all"]["energy_imported_grid"] == 7909 + assert result["hc"]["all"]["energy_imported_pv"] == 2980 + assert result["hc"]["all"]["energy_imported_bat"] == 912 + assert result["hc"]["all"]["energy_imported_cp"] == 273 # Check cp totals (in Wh) - assert result["cp"]["cp1"]["energy_imported_grid"] == 22222 # (12345.7 + 9876.2) Wh = 22221.9 Wh - assert result["cp"]["cp1"]["energy_imported_pv"] == 8354 # (4632.1 + 3721.8) Wh = 8353.9 Wh - assert result["cp"]["cp1"]["energy_imported_bat"] == 3300 # (1876.4 + 1423.5) Wh = 3299.9 Wh - assert result["cp"]["cp1"]["energy_imported_cp"] == 802 # (234.6 + 567.1) Wh = 801.7 Wh + assert result["cp"]["cp1"]["energy_imported_grid"] == 22222 + assert result["cp"]["cp1"]["energy_imported_pv"] == 8354 + assert result["cp"]["cp1"]["energy_imported_bat"] == 3300 + assert result["cp"]["cp1"]["energy_imported_cp"] == 802 - assert result["cp"]["cp2"]["energy_imported_grid"] == 18721 # 18721.3 Wh (only in first entry) - assert result["cp"]["cp2"]["energy_imported_pv"] == 7124 # 7123.8 Wh - assert result["cp"]["cp2"]["energy_imported_bat"] == 2955 # 2954.7 Wh - assert result["cp"]["cp2"]["energy_imported_cp"] == 313 # 312.9 Wh + assert result["cp"]["cp2"]["energy_imported_grid"] == 18721 + assert result["cp"]["cp2"]["energy_imported_pv"] == 7124 + assert result["cp"]["cp2"]["energy_imported_bat"] == 2955 + assert result["cp"]["cp2"]["energy_imported_cp"] == 313 # 11432.6 Wh (only in second entry) assert result["cp"]["cp3"]["energy_imported_grid"] == 11433 - assert result["cp"]["cp3"]["energy_imported_pv"] == 4824 # 4823.9 Wh - assert result["cp"]["cp3"]["energy_imported_bat"] == 2135 # 2134.7 Wh - assert result["cp"]["cp3"]["energy_imported_cp"] == 689 # 689.2 Wh + assert result["cp"]["cp3"]["energy_imported_pv"] == 4824 + assert result["cp"]["cp3"]["energy_imported_bat"] == 2135 + assert result["cp"]["cp3"]["energy_imported_cp"] == 689 # Check counter totals (in Wh, only non-grid counters) - assert result["counter"]["counter1"]["energy_imported_grid"] == 12158 # (6234.8 + 5923.1) Wh = 12157.9 Wh - assert result["counter"]["counter1"]["energy_imported_pv"] == 5123 # (2387.5 + 2734.8) Wh = 5122.3 Wh - assert result["counter"]["counter1"]["energy_imported_bat"] == 2011 # (923.4 + 1087.6) Wh = 2011.0 Wh - assert result["counter"]["counter1"]["energy_imported_cp"] == 744 # (445.7 + 298.3) Wh = 744.0 Wh + assert result["counter"]["counter1"]["energy_imported_grid"] == 12158 + assert result["counter"]["counter1"]["energy_imported_pv"] == 5123 + assert result["counter"]["counter1"]["energy_imported_bat"] == 2011 + assert result["counter"]["counter1"]["energy_imported_cp"] == 744 # 8734.5 Wh (only in second entry) assert result["counter"]["counter2"]["energy_imported_grid"] == 8735 - assert result["counter"]["counter2"]["energy_imported_pv"] == 3290 # 3289.7 Wh - assert result["counter"]["counter2"]["energy_imported_bat"] == 1634 # 1634.2 Wh - assert result["counter"]["counter2"]["energy_imported_cp"] == 824 # 823.6 Wh + assert result["counter"]["counter2"]["energy_imported_pv"] == 3290 + assert result["counter"]["counter2"]["energy_imported_bat"] == 1634 + assert result["counter"]["counter2"]["energy_imported_cp"] == 824 def test_convert(daily_log_entry_processed, daily_log_sample): diff --git a/packages/helpermodules/utils/precision_math.py b/packages/helpermodules/utils/precision_math.py index 97017a3e32..9d79c774e8 100644 --- a/packages/helpermodules/utils/precision_math.py +++ b/packages/helpermodules/utils/precision_math.py @@ -6,7 +6,7 @@ def string_to_float(value: str, default: float = 0) -> float: """Convert string to float with fallback to default value.""" try: return float(value) - except ValueError: + except (ValueError, TypeError): return default @@ -14,7 +14,7 @@ def string_to_int(value: str, default: int = 0) -> int: """Convert string to int with fallback to default value.""" try: return int(value) - except ValueError: + except (ValueError, TypeError): return default From 13514446eb38a11bdd82cf55e9c14246ca28043b Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 29 Apr 2026 08:52:07 +0200 Subject: [PATCH 04/15] clean up --- packages/helpermodules/measurement_logging/process_log.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/helpermodules/measurement_logging/process_log.py b/packages/helpermodules/measurement_logging/process_log.py index b512bff035..b1c9bc9f4c 100644 --- a/packages/helpermodules/measurement_logging/process_log.py +++ b/packages/helpermodules/measurement_logging/process_log.py @@ -372,7 +372,6 @@ def calc_energy_imported_by_source(entry): energy_source = entry["energy_source"] for source in ("grid", "pv", "bat", "cp"): - # Handle hc section hc_section = entry.get("hc") if isinstance(hc_section, dict) and "all" in hc_section: hc_all = hc_section["all"] @@ -380,7 +379,6 @@ def calc_energy_imported_by_source(entry): hc_all[f"energy_imported_{source}"] = decimal_multiply( hc_all["energy_imported"], energy_source[source]) - # Handle cp section cp_section = entry.get("cp") if isinstance(cp_section, dict): for cp_data in cp_section.values(): @@ -388,7 +386,6 @@ def calc_energy_imported_by_source(entry): cp_data[f"energy_imported_{source}"] = decimal_multiply( cp_data["energy_imported"], energy_source[source]) - # Handle counter section counter_section = entry.get("counter") if isinstance(counter_section, dict): for counter in counter_section.values(): From 297fd25b9efe5ac1c690db5bc1eef65930bc5407 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 29 Apr 2026 16:01:42 +0200 Subject: [PATCH 05/15] error handling --- packages/control/chargelog/chargelog.py | 6 +- .../measurement_logging/conftest.py | 99 ++-- .../measurement_logging/process_log.py | 184 +++++-- .../process_log_integration_test.py | 3 +- .../process_log_testdata.py | 469 ++++++++++-------- .../process_log_unit_test.py | 40 +- .../measurement_logging/write_log.py | 29 +- packages/helpermodules/update_config.py | 32 ++ packages/helpermodules/update_config_test.py | 64 +++ 9 files changed, 596 insertions(+), 330 deletions(-) diff --git a/packages/control/chargelog/chargelog.py b/packages/control/chargelog/chargelog.py index bdb40e7ace..1c9c8bd7aa 100644 --- a/packages/control/chargelog/chargelog.py +++ b/packages/control/chargelog/chargelog.py @@ -327,7 +327,7 @@ def write_new_entry(new_entry): def calc_energy_costs(cp, create_log_entry: bool = False): try: if cp.data.set.log.imported_since_plugged != 0 and cp.data.set.log.imported_since_mode_switch != 0: - processed_entries, reference_entries = _get_reference_entries() + processed_entries, reference_entries = _get_reference_entries(cp) charged_energy_by_source = calculate_charged_energy_by_source( cp, processed_entries, reference_entries, create_log_entry) _add_charged_energy_by_source(cp, charged_energy_by_source) @@ -396,7 +396,7 @@ def _get_reference_position(cp, create_log_entry: bool) -> ReferenceTime: return ReferenceTime.MIDDLE -def _get_reference_entries() -> Tuple[List[Dict], List]: +def _get_reference_entries(cp) -> Tuple[List[Dict], List]: processed_entries = {} reference_entries = [] try: @@ -410,7 +410,7 @@ def _get_reference_entries() -> Tuple[List[Dict], List]: processed_entries["entries"] = copy.deepcopy(reference_entries) processed_entries["entries"] = _process_entries(processed_entries["entries"], CalculationType.ENERGY) processed_entries["totals"] = get_totals(processed_entries["entries"], False) - processed_entries = _analyse_energy_source(processed_entries) + processed_entries = _analyse_energy_source(processed_entries, f"cp{cp.num}") except Exception: log.exception("Fehler beim Zusammenstellen der zwei letzten Logeinträge") finally: diff --git a/packages/helpermodules/measurement_logging/conftest.py b/packages/helpermodules/measurement_logging/conftest.py index f510057202..9cb2b90153 100644 --- a/packages/helpermodules/measurement_logging/conftest.py +++ b/packages/helpermodules/measurement_logging/conftest.py @@ -172,28 +172,39 @@ def daily_log_entry_processed(): def daily_log_entry_percentage(): return { 'bat': {'all': {'energy_exported': 0.275, - 'energy_imported': 0.0}, + 'energy_imported': 0.0, + 'fault_state': 0}, 'bat2': {'energy_exported': 0.275, - 'energy_imported': 0.0}}, + 'energy_imported': 0.0, + 'fault_state': 0}}, 'counter': {'counter0': {'energy_exported': 0.0, 'energy_imported': 0.746, - 'grid': True}}, + 'grid': True, + 'fault_state': 0}}, 'cp': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.96}, + 'energy_imported': 0.96, + 'fault_state': 0}, 'cp3': {'energy_exported': 0.0, - 'energy_imported': 0.5762}, + 'energy_imported': 0.5762, + 'fault_state': 0}, 'cp4': {'energy_exported': 0.0, - 'energy_imported': 0.192}, + 'energy_imported': 0.192, + 'fault_state': 0}, 'cp5': {'energy_exported': 0.0, - 'energy_imported': 0.192}}, + 'energy_imported': 0.192, + 'fault_state': 0}}, 'date': '09:35', - 'ev': {'ev0': {'soc': 0}}, + 'ev': {'ev0': {'soc': 0, + 'fault_state': 0}}, 'hc': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.01}}, + 'energy_imported': 0.01, + 'fault_state': 0}}, 'pv': {'all': {'energy_exported': 0.126, - 'energy_imported': 0.0}, + 'energy_imported': 0.0, + 'fault_state': 0}, 'pv1': {'energy_exported': 0.126, - 'energy_imported': 0.0}}, + 'energy_imported': 0.0, + 'fault_state': 0}}, 'sh': {'sh1': {'energy_exported': 0.0, 'energy_imported': 0.0}}, 'timestamp': 1690529761, @@ -204,28 +215,39 @@ def daily_log_entry_percentage(): def daily_log_entry_percentage_negative_consumption(): return { 'bat': {'all': {'energy_exported': 0.275, - 'energy_imported': 0.0}, + 'energy_imported': 0.0, + 'fault_state': 0}, 'bat2': {'energy_exported': 0.275, - 'energy_imported': 0.0}}, + 'energy_imported': 0.0, + 'fault_state': 0}}, 'counter': {'counter0': {'energy_exported': 2, 'energy_imported': 0.746, - 'grid': True}}, + 'grid': True, + 'fault_state': 0}}, 'cp': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.96}, + 'energy_imported': 0.96, + 'fault_state': 0}, 'cp3': {'energy_exported': 0.0, - 'energy_imported': 0.5762}, + 'energy_imported': 0.5762, + 'fault_state': 0}, 'cp4': {'energy_exported': 0.0, - 'energy_imported': 0.192}, + 'energy_imported': 0.192, + 'fault_state': 0}, 'cp5': {'energy_exported': 0.0, - 'energy_imported': 0.192}}, + 'energy_imported': 0.192, + 'fault_state': 0}}, 'date': '09:35', - 'ev': {'ev0': {'soc': 0}}, + 'ev': {'ev0': {'soc': 0, + 'fault_state': 0}}, 'hc': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.01}}, + 'energy_imported': 0.01, + 'fault_state': 0}}, 'pv': {'all': {'energy_exported': 0.15, - 'energy_imported': 0.0}, + 'energy_imported': 0.0, + 'fault_state': 0}, 'pv1': {'energy_exported': 0.15, - 'energy_imported': 0.0}}, + 'energy_imported': 0.0, + 'fault_state': 0}}, 'sh': {'sh1': {'energy_exported': 0.0, 'energy_imported': 0.0}}, 'timestamp': 1690529761, @@ -236,28 +258,39 @@ def daily_log_entry_percentage_negative_consumption(): def daily_log_entry_percentage_cp_discharge(): return { 'bat': {'all': {'energy_exported': 0.275, - 'energy_imported': 0.0}, + 'energy_imported': 0.0, + 'fault_state': 0}, 'bat2': {'energy_exported': 0.275, - 'energy_imported': 0.0}}, + 'energy_imported': 0.0, + 'fault_state': 0}}, 'counter': {'counter0': {'energy_exported': 0.0, 'energy_imported': 0.746, - 'grid': True}}, + 'grid': True, + 'fault_state': 0}}, 'cp': {'all': {'energy_exported': 0.2, - 'energy_imported': 0.96}, + 'energy_imported': 0.96, + 'fault_state': 0}, 'cp3': {'energy_exported': 0.0, - 'energy_imported': 0.5762}, + 'energy_imported': 0.5762, + 'fault_state': 0}, 'cp4': {'energy_exported': 0.0, - 'energy_imported': 0.192}, + 'energy_imported': 0.192, + 'fault_state': 0}, 'cp5': {'energy_exported': 0.0, - 'energy_imported': 0.392}}, + 'energy_imported': 0.392, + 'fault_state': 0}}, 'date': '09:35', - 'ev': {'ev0': {'soc': 0}}, + 'ev': {'ev0': {'soc': 0, + 'fault_state': 0}}, 'hc': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.01}}, + 'energy_imported': 0.01, + 'fault_state': 0}}, 'pv': {'all': {'energy_exported': 0.15, - 'energy_imported': 0.0}, + 'energy_imported': 0.0, + 'fault_state': 0}, 'pv1': {'energy_exported': 0.15, - 'energy_imported': 0.0}}, + 'energy_imported': 0.0, + 'fault_state': 0}}, 'sh': {'sh1': {'energy_exported': 0.0, 'energy_imported': 0.0}}, 'timestamp': 1690529761, diff --git a/packages/helpermodules/measurement_logging/process_log.py b/packages/helpermodules/measurement_logging/process_log.py index b1c9bc9f4c..6d8bfd48f5 100644 --- a/packages/helpermodules/measurement_logging/process_log.py +++ b/packages/helpermodules/measurement_logging/process_log.py @@ -2,7 +2,7 @@ import json import logging from pathlib import Path -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union from helpermodules import timecheck from helpermodules.measurement_logging.write_log import (LegacySmartHomeLogData, LogType, create_entry, @@ -308,19 +308,27 @@ def add_daily_log(day: str) -> None: return {"entries": entries, "names": names} -def _analyse_energy_source(data) -> Dict: +def _analyse_energy_source(data, calc_cp: Optional[str] = None) -> Dict: if data and len(data["entries"]) > 0: try: for i in range(0, len(data["entries"])): - data["entries"][i] = analyse_percentage(data["entries"][i]) - data["entries"][i] = calc_energy_imported_by_source(data["entries"][i]) + data["entries"][i], message_analyse = analyse_percentage(data["entries"][i]) + if calc_cp is not None: + data["entries"][i], message_calc = calc_energy_imported_by_source_cp(data["entries"][i], calc_cp) + else: + data["entries"][i], message_calc = calc_energy_imported_by_source_all(data["entries"][i]) data["totals"] = analyse_percentage_totals(data["entries"], data["totals"]) except Exception: pub_system_message({}, "Fehler beim Berechnen des Strom-Mix", MessageType.ERROR) + data["message"] = message_analyse + message_calc return data -def analyse_percentage(entry): +def analyse_percentage(entry) -> Tuple[Dict, str]: + EOOR_STATE_MSG = ("Der Strom-Mix um " + entry["date"] + + " konnte nicht berechnet werden, da sich {} im Fehlerzustand befindet. Alle Verbräuche werden" + + " dem Netz zugerechnet.\n") + def format(value): return round(value, 4) @@ -330,72 +338,142 @@ def get_grid_from(entry) -> Tuple[float, float]: raise KeyError(f"Kein Zähler für das Netz gefunden in Eintrag '{entry['timestamp']}'.") return sum(grid["energy_imported"] for grid in grids), sum(grid["energy_exported"] for grid in grids) + def get_grid_error_state(entry) -> int: + grids = [counter for counter in entry["counter"].values() if counter.get("grid")] + if not grids: + raise KeyError(f"Kein Zähler für das Netz gefunden in Eintrag '{entry['timestamp']}'.") + max_grid = max(grids, key=lambda g: g.get("fault_state", 0)) + return max_grid.get("fault_state", 0) + try: - bat_imported = safe_get_nested(entry, "bat", "all", "energy_imported") - bat_exported = safe_get_nested(entry, "bat", "all", "energy_exported") - cp_exported = safe_get_nested(entry, "cp", "all", "energy_exported") - pv_exported = safe_get_nested(entry, "pv", "all", "energy_exported") - grid_imported, grid_exported = get_grid_from(entry) - consumption = grid_imported - grid_exported + pv_exported + bat_exported - bat_imported + cp_exported - if consumption < 0: - consumption = 0 + message = "" + if (safe_get_nested(entry, "bat", "all", "fault_state") == 2 or + safe_get_nested(entry, "cp", "all", "fault_state") == 2 or + safe_get_nested(entry, "pv", "all", "fault_state") == 2 or + get_grid_error_state(entry) == 2): + + entry["energy_source"] = {"grid": 1, "pv": 0, "bat": 0, "cp": 0} + if safe_get_nested(entry, "bat", "all", "fault_state") == 2: + message += EOOR_STATE_MSG.format("mind. einer der Speicher") + if safe_get_nested(entry, "cp", "all", "fault_state") == 2: + message += EOOR_STATE_MSG.format("mind. einer der Ladepunkte") + if safe_get_nested(entry, "pv", "all", "fault_state") == 2: + message += EOOR_STATE_MSG.format("mind. einer der Wechselrichter") + if get_grid_error_state(entry) == 2: + message += EOOR_STATE_MSG.format("der Zähler für das Netz") - try: - pv_direct = min(pv_exported, consumption) - remaining = consumption - pv_direct + else: + bat_imported = safe_get_nested(entry, "bat", "all", "energy_imported") + bat_exported = safe_get_nested(entry, "bat", "all", "energy_exported") + cp_exported = safe_get_nested(entry, "cp", "all", "energy_exported") + pv_exported = safe_get_nested(entry, "pv", "all", "energy_exported") + grid_imported, grid_exported = get_grid_from(entry) + consumption = grid_imported - grid_exported + pv_exported + bat_exported - bat_imported + cp_exported + if consumption < 0: + consumption = 0 + + try: + pv_direct = min(pv_exported, consumption) + remaining = consumption - pv_direct - bat_direct = min(bat_exported, remaining) - remaining -= bat_direct + bat_direct = min(bat_exported, remaining) + remaining -= bat_direct - cp_direct = min(cp_exported, remaining) - remaining -= cp_direct + cp_direct = min(cp_exported, remaining) + remaining -= cp_direct - grid_direct = min(grid_imported, remaining) + grid_direct = min(grid_imported, remaining) - entry["energy_source"] = { - "grid": format(grid_direct / consumption), - "pv": format(pv_direct / consumption), - "bat": format(bat_direct / consumption), - "cp": format(cp_direct / consumption)} - except ZeroDivisionError: - entry["energy_source"] = {"grid": 0, "pv": 0, "bat": 0, "cp": 0} + entry["energy_source"] = { + "grid": format(grid_direct / consumption), + "pv": format(pv_direct / consumption), + "bat": format(bat_direct / consumption), + "cp": format(cp_direct / consumption)} + except ZeroDivisionError: + entry["energy_source"] = {"grid": 0, "pv": 0, "bat": 0, "cp": 0} except Exception: log.exception(f"Fehler beim Berechnen des Strom-Mix von {entry['timestamp']}") finally: - return entry + return entry, message + + +ERROR_STATE_MESSAGE = ("Die Anteile der Energiequellen für {} konnten nicht berechnet werden, da er sich im " + + "Fehlerzustand befindet. Die Verbräuche werden mit 0 kWh angesetzt.\n") + + +def calc_energy_imported_by_source_all(entry) -> Tuple[Dict, str]: + try: + if "energy_source" not in entry.keys(): + return entry, "" + energy_source = entry["energy_source"] + message = "" + + hc_section = entry.get("hc") + if isinstance(hc_section, dict) and "all" in hc_section: + hc_all = hc_section["all"] + if isinstance(hc_all, dict): + if hc_all.get("fault_state", 0) == 0 and "energy_imported" in hc_all: + for source in ("grid", "pv", "bat", "cp"): + hc_all[f"energy_imported_{source}"] = decimal_multiply( + hc_all["energy_imported"], energy_source[source]) + else: + for source in ("grid", "pv", "bat", "cp"): + hc_all[f"energy_imported_{source}"] = 0 + message += ERROR_STATE_MESSAGE.format("den Hausverbrauch") + + cp_section = entry.get("cp") + if isinstance(cp_section, dict): + for cp_key, cp_data in cp_section.items(): + if isinstance(cp_data, dict): + if cp_data.get("fault_state", 0) == 0 and "energy_imported" in cp_data: + for source in ("grid", "pv", "bat", "cp"): + cp_data[f"energy_imported_{source}"] = decimal_multiply( + cp_data["energy_imported"], energy_source[source]) + else: + for source in ("grid", "pv", "bat", "cp"): + cp_data[f"energy_imported_{source}"] = 0 + message += ERROR_STATE_MESSAGE.format(f"Ladepunkt {entry['names'][cp_key]}") + + counter_section = entry.get("counter") + if isinstance(counter_section, dict): + for counter in counter_section.values(): + if isinstance(counter, dict) and counter.get("grid") is False: + if counter.get("fault_state", 0) == 0 and "energy_imported" in counter: + for source in ("grid", "pv", "bat", "cp"): + counter[f"energy_imported_{source}"] = decimal_multiply( + counter["energy_imported"], energy_source[source]) + else: + for source in ("grid", "pv", "bat", "cp"): + counter[f"energy_imported_{source}"] = 0 + message += ERROR_STATE_MESSAGE.format(f"Zähler {entry['names'][counter]}") + except Exception: + log.exception(f"Fehler beim Berechnen der Energie-Anteile aus dem Strom-Mix von {entry['timestamp']}") + finally: + return entry, message -def calc_energy_imported_by_source(entry): +def calc_energy_imported_by_source_cp(entry, cp: str) -> Tuple[Dict, str]: try: if "energy_source" not in entry.keys(): - return entry + return entry, "" energy_source = entry["energy_source"] - for source in ("grid", "pv", "bat", "cp"): - hc_section = entry.get("hc") - if isinstance(hc_section, dict) and "all" in hc_section: - hc_all = hc_section["all"] - if isinstance(hc_all, dict) and "energy_imported" in hc_all: - hc_all[f"energy_imported_{source}"] = decimal_multiply( - hc_all["energy_imported"], energy_source[source]) - - cp_section = entry.get("cp") - if isinstance(cp_section, dict): - for cp_data in cp_section.values(): - if isinstance(cp_data, dict) and "energy_imported" in cp_data: - cp_data[f"energy_imported_{source}"] = decimal_multiply( - cp_data["energy_imported"], energy_source[source]) - - counter_section = entry.get("counter") - if isinstance(counter_section, dict): - for counter in counter_section.values(): - if isinstance(counter, dict) and counter.get("grid") is False and "energy_imported" in counter: - counter[f"energy_imported_{source}"] = decimal_multiply( - counter["energy_imported"], energy_source[source]) + message = "" + + cp_data = entry["cp"][cp] + if cp_data.get("fault_state", 0) == 0 and "energy_imported" in cp_data: + for source in ("grid", "pv", "bat", "cp"): + cp_data[f"energy_imported_{source}"] = decimal_multiply( + cp_data["energy_imported"], energy_source[source]) + else: + for source in ("grid", "pv", "bat", "cp"): + cp_data[f"energy_imported_{source}"] = 0 + message += ERROR_STATE_MESSAGE.format(f"Ladepunkt {entry['names'][cp]}") + except Exception: log.exception(f"Fehler beim Berechnen der Energie-Anteile aus dem Strom-Mix von {entry['timestamp']}") finally: - return entry + return entry, message def analyse_percentage_totals(entries, totals): diff --git a/packages/helpermodules/measurement_logging/process_log_integration_test.py b/packages/helpermodules/measurement_logging/process_log_integration_test.py index 3bb1156e0a..540740ef2b 100644 --- a/packages/helpermodules/measurement_logging/process_log_integration_test.py +++ b/packages/helpermodules/measurement_logging/process_log_integration_test.py @@ -1,3 +1,4 @@ +from pprint import pprint from unittest.mock import Mock import pytest @@ -20,6 +21,6 @@ def test_get_daily_log(data, expected, monkeypatch): # execution daily_log_processed = process_log.get_daily_log("20250616") - + pprint(daily_log_processed) # evaluation assert daily_log_processed == expected diff --git a/packages/helpermodules/measurement_logging/process_log_testdata.py b/packages/helpermodules/measurement_logging/process_log_testdata.py index 37d05a3c7f..032b5f4712 100644 --- a/packages/helpermodules/measurement_logging/process_log_testdata.py +++ b/packages/helpermodules/measurement_logging/process_log_testdata.py @@ -1,47 +1,57 @@ counter_jumps_forward = {'entries': [{'bat': {'all': {'exported': 3195.13, 'imported': 629.37, - 'soc': 48}, + 'soc': 48, + 'fault_state': 0}, 'bat2': {'exported': 3195.13, 'imported': 629.37, - 'soc': 48}}, + 'soc': 48, + 'fault_state': 0}}, 'counter': {'counter0': {'exported': 26029.945, 'grid': True, - 'imported': 0}, + 'imported': 0, + 'fault_state': 0}, 'counter2': {'exported': 26029.945, 'grid': False, - 'imported': 0}}, - 'cp': {'all': {'exported': 0, 'imported': 12639.11}, - 'cp3': {'exported': 0, 'imported': 12639.11}, - 'cp4': {'exported': 0, 'imported': 0}, - 'cp5': {'exported': 0, 'imported': 0}}, + 'imported': 0, + 'fault_state': 0}}, + 'cp': {'all': {'exported': 0, 'imported': 12639.11, 'fault_state': 0}, + 'cp3': {'exported': 0, 'imported': 12639.11, 'fault_state': 0}, + 'cp4': {'exported': 0, 'imported': 0, 'fault_state': 0}, + 'cp5': {'exported': 0, 'imported': 0, 'fault_state': 0}}, 'date': '14:25', - 'ev': {'ev0': {'soc': None}}, - 'hc': {'all': {'imported': 2324.001611140539}}, + 'ev': {'ev0': {'soc': None, 'fault_state': 0}}, + 'hc': {'all': {'imported': 2324.001611140539, 'fault_state': 0}}, 'prices': {'bat': 0.0002, 'grid': 0.0003, 'pv': 0.00015}, - 'pv': {'all': {'exported': 35827}, 'pv1': {'exported': 35827}}, + 'pv': {'all': {'exported': 35827, 'fault_state': 0}, + 'pv1': {'exported': 35827, 'fault_state': 0}}, 'sh': {}, 'timestamp': 1750767902}, {'bat': {'all': {'exported': 3195.13, 'imported': 629.37, - 'soc': 48}, + 'soc': 48, + 'fault_state': 0}, 'bat2': {'exported': 3195.13, 'imported': 629.37, - 'soc': 48}}, + 'soc': 48, + 'fault_state': 0}}, 'counter': {'counter0': {'exported': 26802.355, 'grid': True, - 'imported': 0}, + 'imported': 0, + 'fault_state': 0}, 'counter2': {'exported': 26029.945, 'grid': False, - 'imported': 0}}, - 'cp': {'all': {'exported': 0, 'imported': 20639.11}, - 'cp3': {'exported': 0, 'imported': 20639.11}, - 'cp4': {'exported': 0, 'imported': 0}, - 'cp5': {'exported': 0, 'imported': 0}}, + 'imported': 0, + 'fault_state': 0}}, + 'cp': {'all': {'exported': 0, 'imported': 20639.11, 'fault_state': 0}, + 'cp3': {'exported': 0, 'imported': 20639.11, 'fault_state': 0}, + 'cp4': {'exported': 0, 'imported': 0, 'fault_state': 0}, + 'cp5': {'exported': 0, 'imported': 0, 'fault_state': 0}}, 'date': '14:30', - 'ev': {'ev0': {'soc': None}}, - 'hc': {'all': {'imported': 2361.178611303703}}, + 'ev': {'ev0': {'soc': None, 'fault_state': 0}}, + 'hc': {'all': {'imported': 2361.178611303703, 'fault_state': 0}}, 'prices': {'bat': 0.0002, 'grid': 0.0003, 'pv': 0.00015}, - 'pv': {'all': {'exported': 36636}, 'pv1': {'exported': 36636}}, + 'pv': {'all': {'exported': 36636, 'fault_state': 0}, + 'pv1': {'exported': 36636, 'fault_state': 0}}, 'sh': {}, 'timestamp': 1750768201}], 'names': {'bat2': 'MQTT-Speicher', @@ -53,114 +63,126 @@ 'ev0': 'Standard-Fahrzeug', 'pv1': 'MQTT-Wechselrichter'}} -counter_jumps_forward_processed = {'entries': [{'bat': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.0, +counter_jumps_forward_processed = {'entries': [{'bat': {'all': {'energy_exported': 0, + 'energy_imported': 0, 'exported': 3195.13, + 'fault_state': 0, 'imported': 629.37, - 'power_average': 0.0, + 'power_average': 0, 'power_exported': 0, - 'power_imported': 0.0, + 'power_imported': 0, 'soc': 48}, - 'bat2': {'energy_exported': 0.0, - 'energy_imported': 0.0, + 'bat2': {'energy_exported': 0, + 'energy_imported': 0, 'exported': 3195.13, + 'fault_state': 0, 'imported': 629.37, - 'power_average': 0.0, + 'power_average': 0, 'power_exported': 0, - 'power_imported': 0.0, + 'power_imported': 0, 'soc': 48}}, 'counter': {'counter0': {'energy_exported': 772.41, - 'energy_imported': 0.0, + 'energy_imported': 0, 'exported': 26029.945, + 'fault_state': 0, 'grid': True, 'imported': 0, 'power_average': -9299.92, 'power_exported': 9299.92, 'power_imported': 0}, - 'counter2': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0, + 'counter2': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0, 'exported': 26029.945, + 'fault_state': 0, 'grid': False, 'imported': 0, - 'power_average': 0.0, + 'power_average': 0, 'power_exported': 0, - 'power_imported': 0.0}}, - 'cp': {'all': {'energy_exported': 0.0, - 'energy_imported': 8000.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 8000.0, + 'power_imported': 0}}, + 'cp': {'all': {'energy_exported': 0, + 'energy_imported': 8000, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 8000, 'exported': 0, + 'fault_state': 0, 'imported': 12639.11, 'power_average': 96321.07, 'power_exported': 0, 'power_imported': 96321.07}, - 'cp3': {'energy_exported': 0.0, - 'energy_imported': 8000.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 8000.0, + 'cp3': {'energy_exported': 0, + 'energy_imported': 8000, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 8000, 'exported': 0, + 'fault_state': 0, 'imported': 12639.11, 'power_average': 96321.07, 'power_exported': 0, 'power_imported': 96321.07}, - 'cp4': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0, + 'cp4': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0, 'exported': 0, + 'fault_state': 0, 'imported': 0, - 'power_average': 0.0, + 'power_average': 0, 'power_exported': 0, - 'power_imported': 0.0}, - 'cp5': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0, + 'power_imported': 0}, + 'cp5': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0, 'exported': 0, + 'fault_state': 0, 'imported': 0, - 'power_average': 0.0, + 'power_average': 0, 'power_exported': 0, - 'power_imported': 0.0}}, + 'power_imported': 0}}, 'date': '14:25', 'energy_source': {'bat': 0.0, 'cp': 0.0, 'grid': 0.0, 'pv': 1.0}, - 'ev': {'ev0': {'soc': None}}, - 'hc': {'all': {'energy_exported': 0.0, + 'ev': {'ev0': {'fault_state': 0, 'soc': None}}, + 'hc': {'all': {'energy_exported': 0, 'energy_imported': 37.177, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, 'energy_imported_pv': 37.177, + 'fault_state': 0, 'imported': 2324.001611140539, 'power_average': 447.616, 'power_exported': 0, 'power_imported': 447.616}}, 'prices': {'bat': 0.0002, 'grid': 0.0003, 'pv': 0.00015}, - 'pv': {'all': {'energy_exported': 809.0, - 'energy_imported': 0.0, + 'pv': {'all': {'energy_exported': 809, + 'energy_imported': 0, 'exported': 35827, + 'fault_state': 0, 'power_average': -9740.468, 'power_exported': 9740.468, 'power_imported': 0}, - 'pv1': {'energy_exported': 809.0, - 'energy_imported': 0.0, + 'pv1': {'energy_exported': 809, + 'energy_imported': 0, 'exported': 35827, + 'fault_state': 0, 'power_average': -9740.468, 'power_exported': 9740.468, 'power_imported': 0}}, 'sh': {}, 'timestamp': 1750767902}], + 'message': '', 'names': {'bat2': 'MQTT-Speicher', 'counter0': 'MQTT-Zähler', 'counter2': 'Test-Zähler', @@ -169,89 +191,97 @@ 'cp5': 'MQTT-Ladepunkt', 'ev0': 'Standard-Fahrzeug', 'pv1': 'MQTT-Wechselrichter'}, - 'totals': {'bat': {'all': {'energy_exported': 0.0, 'energy_imported': 0.0}, - 'bat2': {'energy_exported': 0.0, 'energy_imported': 0.0}}, + 'totals': {'bat': {'all': {'energy_exported': 0, 'energy_imported': 0}, + 'bat2': {'energy_exported': 0, 'energy_imported': 0}}, 'counter': {'counter0': {'energy_exported': 772.41, - 'energy_imported': 0.0, + 'energy_imported': 0, 'grid': True}, - 'counter2': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0, + 'counter2': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0, 'grid': False}}, - 'cp': {'all': {'energy_exported': 0.0, - 'energy_imported': 8000.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 8000.0}, - 'cp3': {'energy_exported': 0.0, - 'energy_imported': 8000.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 8000.0}, - 'cp4': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0}, - 'cp5': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0}}, + 'cp': {'all': {'energy_exported': 0, + 'energy_imported': 8000, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 8000}, + 'cp3': {'energy_exported': 0, + 'energy_imported': 8000, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 8000}, + 'cp4': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0}, + 'cp5': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0}}, 'hc': {'all': {'energy_imported': 37.177, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, 'energy_imported_pv': 37.177}}, - 'pv': {'all': {'energy_exported': 809.0}, - 'pv1': {'energy_exported': 809.0}}, + 'pv': {'all': {'energy_exported': 809}, + 'pv1': {'energy_exported': 809}}, 'sh': {}}} regular_daily_log_entry = {'entries': [{'bat': {'all': {'exported': 3195.13, 'imported': 629.37, - 'soc': 48}, + 'soc': 48, + 'fault_state': 0}, 'bat2': {'exported': 3195.13, 'imported': 629.37, - 'soc': 48}}, + 'soc': 48, + 'fault_state': 0}}, 'counter': {'counter0': {'exported': 26029.945, 'grid': True, - 'imported': 2728.572}}, - 'cp': {'all': {'exported': 0, 'imported': 12639.11}, - 'cp3': {'exported': 0, 'imported': 12639.11}, - 'cp4': {'exported': 0, 'imported': 0}, - 'cp5': {'exported': 0, 'imported': 0}}, + 'imported': 2728.572, + 'fault_state': 0}, }, + 'cp': {'all': {'exported': 0, 'imported': 12639.11, 'fault_state': 0}, + 'cp3': {'exported': 0, 'imported': 12639.11, 'fault_state': 0}, + 'cp4': {'exported': 0, 'imported': 0, 'fault_state': 0}, + 'cp5': {'exported': 0, 'imported': 0, 'fault_state': 0}}, 'date': '14:25', 'ev': {'ev0': {'soc': None}}, - 'hc': {'all': {'imported': 2324.001611140539}}, + 'hc': {'all': {'imported': 2324.001611140539, 'fault_state': 0}}, 'prices': {'bat': 0.0002, 'grid': 0.0003, 'pv': 0.00015}, - 'pv': {'all': {'exported': 35827}, 'pv1': {'exported': 35827}}, + 'pv': {'all': {'exported': 35827, 'fault_state': 0}, + 'pv1': {'exported': 35827, 'fault_state': 0}}, 'sh': {}, 'timestamp': 1750767902}, {'bat': {'all': {'exported': 3195.13, 'imported': 629.37, - 'soc': 48}, + 'soc': 48, + 'fault_state': 0}, 'bat2': {'exported': 3195.13, 'imported': 629.37, - 'soc': 48}}, + 'soc': 48, + 'fault_state': 0}}, 'counter': {'counter0': {'exported': 26802.355, 'grid': True, - 'imported': 2728.572}}, - 'cp': {'all': {'exported': 0, 'imported': 12639.11}, - 'cp3': {'exported': 0, 'imported': 12639.11}, - 'cp4': {'exported': 0, 'imported': 0}, - 'cp5': {'exported': 0, 'imported': 0}}, + 'imported': 2728.572, + 'fault_state': 0}}, + 'cp': {'all': {'exported': 0, 'imported': 12639.11, 'fault_state': 0}, + 'cp3': {'exported': 0, 'imported': 12639.11, 'fault_state': 0}, + 'cp4': {'exported': 0, 'imported': 0, 'fault_state': 0}, + 'cp5': {'exported': 0, 'imported': 0, 'fault_state': 0}}, 'date': '14:30', - 'ev': {'ev0': {'soc': None}}, - 'hc': {'all': {'imported': 2361.178611303703}}, + 'ev': {'ev0': {'soc': None, 'fault_state': 0}}, + 'hc': {'all': {'imported': 2361.178611303703, 'fault_state': 0}}, 'prices': {'bat': 0.0002, 'grid': 0.0003, 'pv': 0.00015}, - 'pv': {'all': {'exported': 36636}, 'pv1': {'exported': 36636}}, + 'pv': {'all': {'exported': 36636, 'fault_state': 0}, + 'pv1': {'exported': 36636, 'fault_state': 0}}, 'sh': {}, 'timestamp': 1750768201}], 'names': {'bat2': 'MQTT-Speicher', @@ -263,102 +293,113 @@ 'ev0': 'Standard-Fahrzeug', 'pv1': 'MQTT-Wechselrichter'}} -regular_daily_log_entry_processed = {'entries': [{'bat': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.0, +regular_daily_log_entry_processed = {'entries': [{'bat': {'all': {'energy_exported': 0, + 'energy_imported': 0, 'exported': 3195.13, + 'fault_state': 0, 'imported': 629.37, - 'power_average': 0.0, + 'power_average': 0, 'power_exported': 0, - 'power_imported': 0.0, + 'power_imported': 0, 'soc': 48}, - 'bat2': {'energy_exported': 0.0, - 'energy_imported': 0.0, + 'bat2': {'energy_exported': 0, + 'energy_imported': 0, 'exported': 3195.13, + 'fault_state': 0, 'imported': 629.37, - 'power_average': 0.0, + 'power_average': 0, 'power_exported': 0, - 'power_imported': 0.0, + 'power_imported': 0, 'soc': 48}}, 'counter': {'counter0': {'energy_exported': 772.41, - 'energy_imported': 0.0, + 'energy_imported': 0, 'exported': 26029.945, + 'fault_state': 0, 'grid': True, 'imported': 2728.572, 'power_average': -9299.92, 'power_exported': 9299.92, 'power_imported': 0}}, - 'cp': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0, + 'cp': {'all': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0, 'exported': 0, + 'fault_state': 0, 'imported': 12639.11, - 'power_average': 0.0, + 'power_average': 0, 'power_exported': 0, - 'power_imported': 0.0}, - 'cp3': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0, + 'power_imported': 0}, + 'cp3': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0, 'exported': 0, + 'fault_state': 0, 'imported': 12639.11, - 'power_average': 0.0, + 'power_average': 0, 'power_exported': 0, - 'power_imported': 0.0}, - 'cp4': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0, + 'power_imported': 0}, + 'cp4': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0, 'exported': 0, + 'fault_state': 0, 'imported': 0, - 'power_average': 0.0, + 'power_average': 0, 'power_exported': 0, - 'power_imported': 0.0}, - 'cp5': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0, + 'power_imported': 0}, + 'cp5': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0, 'exported': 0, + 'fault_state': 0, 'imported': 0, - 'power_average': 0.0, + 'power_average': 0, 'power_exported': 0, - 'power_imported': 0.0}}, + 'power_imported': 0}}, 'date': '14:25', 'energy_source': {'bat': 0.0, 'cp': 0.0, 'grid': 0.0, 'pv': 1.0}, 'ev': {'ev0': {'soc': None}}, - 'hc': {'all': {'energy_exported': 0.0, + 'hc': {'all': {'energy_exported': 0, 'energy_imported': 37.177, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, 'energy_imported_pv': 37.177, + 'fault_state': 0, 'imported': 2324.001611140539, 'power_average': 447.616, 'power_exported': 0, 'power_imported': 447.616}}, 'prices': {'bat': 0.0002, 'grid': 0.0003, 'pv': 0.00015}, - 'pv': {'all': {'energy_exported': 809.0, - 'energy_imported': 0.0, + 'pv': {'all': {'energy_exported': 809, + 'energy_imported': 0, 'exported': 35827, + 'fault_state': 0, 'power_average': -9740.468, 'power_exported': 9740.468, 'power_imported': 0}, - 'pv1': {'energy_exported': 809.0, - 'energy_imported': 0.0, + 'pv1': {'energy_exported': 809, + 'energy_imported': 0, 'exported': 35827, + 'fault_state': 0, 'power_average': -9740.468, 'power_exported': 9740.468, 'power_imported': 0}}, 'sh': {}, 'timestamp': 1750767902}], + 'message': '', 'names': {'bat2': 'MQTT-Speicher', 'counter0': 'MQTT-Zähler', 'counter2': 'Test-Zähler', @@ -367,40 +408,40 @@ 'cp5': 'MQTT-Ladepunkt', 'ev0': 'Standard-Fahrzeug', 'pv1': 'MQTT-Wechselrichter'}, - 'totals': {'bat': {'all': {'energy_exported': 0.0, 'energy_imported': 0.0}, - 'bat2': {'energy_exported': 0.0, 'energy_imported': 0.0}}, + 'totals': {'bat': {'all': {'energy_exported': 0, 'energy_imported': 0}, + 'bat2': {'energy_exported': 0, 'energy_imported': 0}}, 'counter': {'counter0': {'energy_exported': 772.41, - 'energy_imported': 0.0, + 'energy_imported': 0, 'grid': True}}, - 'cp': {'all': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0}, - 'cp3': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0}, - 'cp4': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0}, - 'cp5': {'energy_exported': 0.0, - 'energy_imported': 0.0, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, - 'energy_imported_pv': 0.0}}, + 'cp': {'all': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0}, + 'cp3': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0}, + 'cp4': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0}, + 'cp5': {'energy_exported': 0, + 'energy_imported': 0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, + 'energy_imported_pv': 0}}, 'hc': {'all': {'energy_imported': 37.177, - 'energy_imported_bat': 0.0, - 'energy_imported_cp': 0.0, - 'energy_imported_grid': 0.0, + 'energy_imported_bat': 0, + 'energy_imported_cp': 0, + 'energy_imported_grid': 0, 'energy_imported_pv': 37.177}}, - 'pv': {'all': {'energy_exported': 809.0}, - 'pv1': {'energy_exported': 809.0}}, + 'pv': {'all': {'energy_exported': 809}, + 'pv1': {'energy_exported': 809}}, 'sh': {}}} diff --git a/packages/helpermodules/measurement_logging/process_log_unit_test.py b/packages/helpermodules/measurement_logging/process_log_unit_test.py index d8bdac6446..607282296b 100644 --- a/packages/helpermodules/measurement_logging/process_log_unit_test.py +++ b/packages/helpermodules/measurement_logging/process_log_unit_test.py @@ -11,7 +11,7 @@ process_entry, get_totals, _collect_daily_log_data, - calc_energy_imported_by_source, + calc_energy_imported_by_source_all, analyse_percentage_totals, CalculationType) @@ -48,10 +48,11 @@ def test_analyse_percentage(name: str, expected.update({"energy_source": {'bat': 0.2006, 'cp': 0.1459, 'grid': 0.5441, 'pv': 0.1094}}) # execution - entry = analyse_percentage(data) + entry, message = analyse_percentage(data) # evaluation assert entry == expected + assert message == "" @pytest.mark.parametrize("test_case, entry_data, expected_energy_source, should_be_unchanged", [ @@ -59,10 +60,11 @@ def test_analyse_percentage(name: str, "zero_consumption", { "timestamp": 1234567890, - "bat": {"all": {"energy_imported": 5.0, "energy_exported": 5.0}}, - "cp": {"all": {"energy_exported": 0.0}}, - "pv": {"all": {"energy_exported": 0.0}}, - "counter": {"counter0": {"grid": True, "energy_imported": 5.0, "energy_exported": 5.0}} + "date": "00:31", + "bat": {"all": {"energy_imported": 5.0, "energy_exported": 5.0, "fault_state": 0}}, + "cp": {"all": {"energy_exported": 0.0, "fault_state": 0}}, + "pv": {"all": {"energy_exported": 0.0, "fault_state": 0}}, + "counter": {"counter0": {"grid": True, "energy_imported": 5.0, "energy_exported": 5.0, "fault_state": 0}} }, {"grid": 0, "pv": 0, "bat": 0, "cp": 0}, False @@ -71,7 +73,8 @@ def test_analyse_percentage(name: str, "missing_sections", { "timestamp": 1234567890, - "counter": {"counter0": {"grid": True, "energy_imported": 10.0, "energy_exported": 2.0}} + "date": "00:31", + "counter": {"counter0": {"grid": True, "energy_imported": 10.0, "energy_exported": 2.0, "fault_state": 0}} }, {"grid": 1.0, "pv": 0.0, "bat": 0.0, "cp": 0.0}, False @@ -80,8 +83,9 @@ def test_analyse_percentage(name: str, "no_grid_counter", { "timestamp": 1234567890, - "bat": {"all": {"energy_imported": 0.0, "energy_exported": 5.0}}, - "counter": {"counter0": {"grid": False, "energy_imported": 10.0, "energy_exported": 2.0}} + "date": "00:31", + "bat": {"all": {"energy_imported": 0.0, "energy_exported": 5.0, "fault_state": 0}}, + "counter": {"counter0": {"grid": False, "energy_imported": 10.0, "energy_exported": 2.0, "fault_state": 0}} }, None, True @@ -89,7 +93,7 @@ def test_analyse_percentage(name: str, ]) def test_analyse_percentage_edge_cases(test_case, entry_data, expected_energy_source, should_be_unchanged): # execution - result = analyse_percentage(entry_data) + result, message = analyse_percentage(entry_data) # evaluation if should_be_unchanged: @@ -99,6 +103,7 @@ def test_analyse_percentage_edge_cases(test_case, entry_data, expected_energy_so else: # Energy source should be calculated correctly assert result["energy_source"] == expected_energy_source + assert message == "" def test_calculate_average_power(): @@ -109,24 +114,24 @@ def test_calculate_average_power(): assert power == 1800 -def test_calc_energy_imported_by_source(): +def test_calc_energy_imported_by_source_all(): # setup entry = { "timestamp": 1234567890, "energy_source": {"grid": 0.6523, "pv": 0.2487, "bat": 0.0789, "cp": 0.0201}, - "hc": {"all": {"energy_imported": 2345.6}}, + "hc": {"all": {"energy_imported": 2345.6, "fault_state": 0}}, "cp": { - "cp1": {"energy_imported": 15723.4}, - "cp2": {"energy_imported": 22108.7} + "cp1": {"energy_imported": 15723.4, "fault_state": 0}, + "cp2": {"energy_imported": 22108.7, "fault_state": 0} }, "counter": { - "counter0": {"grid": True, "energy_imported": 45892.3}, - "counter1": {"grid": False, "energy_imported": 8956.7} + "counter0": {"grid": True, "energy_imported": 45892.3, "fault_state": 0}, + "counter1": {"grid": False, "energy_imported": 8956.7, "fault_state": 0} } } # execution - result = calc_energy_imported_by_source(entry) + result, message = calc_energy_imported_by_source_all(entry) # evaluation - realistic Wh values with decimal precision assert result["hc"]["all"]["energy_imported_grid"] == 1530.035 @@ -153,6 +158,7 @@ def test_calc_energy_imported_by_source(): assert "energy_imported_pv" not in result["counter"]["counter0"] assert "energy_imported_bat" not in result["counter"]["counter0"] assert "energy_imported_cp" not in result["counter"]["counter0"] + assert message == "" def test_analyse_percentage_totals(): diff --git a/packages/helpermodules/measurement_logging/write_log.py b/packages/helpermodules/measurement_logging/write_log.py index 0fe9447147..68527985b4 100644 --- a/packages/helpermodules/measurement_logging/write_log.py +++ b/packages/helpermodules/measurement_logging/write_log.py @@ -200,19 +200,23 @@ def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previou prices = data.data.general_data.data.prices try: grid_price = data.data.optional_data.ep_get_current_price() + fault_state = max(data.data.optional_data.data.electricity_pricing.flexible_tariff.get.fault_state, + data.data.optional_data.data.electricity_pricing.grid_fee.get.fault_state) except Exception: grid_price = prices.grid prices_dict = {"grid": grid_price, "pv": prices.pv, "bat": prices.bat, - "cp": prices.cp} + "cp": prices.cp, + "fault_state": fault_state} except Exception: log.exception("Fehler im Werte-Logging-Modul für Preise") prices_dict = {} try: cp_dict = {"all": {"imported": data.data.cp_all_data.data.get.imported, - "exported": data.data.cp_all_data.data.get.exported}} + "exported": data.data.cp_all_data.data.get.exported, + "fault_state": data.data.cp_all_data.data.get.fault_state}} except Exception: log.exception("Fehler im Werte-Logging-Modul") cp_dict = {} @@ -220,7 +224,8 @@ def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previou try: if "cp" in cp: cp_dict.update({cp: {"imported": data.data.cp_data[cp].data.get.imported, - "exported": data.data.cp_data[cp].data.get.exported}}) + "exported": data.data.cp_data[cp].data.get.exported, + "fault_state": data.data.cp_data[cp].data.get.fault_state}}) except Exception: log.exception("Fehler im Werte-Logging-Modul für Ladepunkt "+str(cp)) @@ -229,7 +234,8 @@ def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previou try: if "ev" in ev: ev_dict.update( - {ev: {"soc": data.data.ev_data[ev].data.get.soc}}) + {ev: {"soc": data.data.ev_data[ev].data.get.soc, + "fault_state": data.data.ev_data[ev].data.get.fault_state}}) except Exception: log.exception("Fehler im Werte-Logging-Modul für EV "+str(ev)) @@ -242,7 +248,8 @@ def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previou {f"counter{counter.num}": { "imported": counter.data.get.imported, "exported": counter.data.get.exported, - "grid": True if data.data.counter_all_data.get_id_evu_counter() == counter.num else False}}) + "grid": True if data.data.counter_all_data.get_id_evu_counter() == counter.num else False, + "fault_state": counter.data.get.fault_state}}) except Exception: log.exception("Fehler im Werte-Logging-Modul für Zähler "+str(counter)) @@ -255,14 +262,16 @@ def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previou for pv in data.data.pv_data: try: pv_dict.update( - {pv: {"exported": data.data.pv_data[pv].data.get.exported}}) + {pv: {"exported": data.data.pv_data[pv].data.get.exported, + "fault_state": data.data.pv_data[pv].data.get.fault_state}}) except Exception: log.exception("Fehler im Werte-Logging-Modul für Wechselrichter "+str(pv)) try: bat_dict = {"all": {"imported": data.data.bat_all_data.data.get.imported, "exported": data.data.bat_all_data.data.get.exported, - "soc": data.data.bat_all_data.data.get.soc}} + "soc": data.data.bat_all_data.data.get.soc, + "fault_state": data.data.bat_all_data.data.get.fault_state}} except Exception: log.exception("Fehler im Werte-Logging-Modul für Batteriespeicher-Daten") bat_dict = {} @@ -271,12 +280,14 @@ def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previou try: bat_dict.update({bat: {"imported": data.data.bat_data[bat].data.get.imported, "exported": data.data.bat_data[bat].data.get.exported, - "soc": data.data.bat_data[bat].data.get.soc}}) + "soc": data.data.bat_data[bat].data.get.soc, + "fault_state": data.data.bat_data[bat].data.get.fault_state}}) except Exception: log.exception("Fehler im Werte-Logging-Modul für Speicher "+str(bat)) try: - hc_dict = {"all": {"imported": data.data.counter_all_data.data.set.imported_home_consumption}} + hc_dict = {"all": {"imported": data.data.counter_all_data.data.set.imported_home_consumption, + "fault_state": data.data.counter_all_data.data.set.invalid_home_consumption}} except Exception: log.exception("Fehler im Werte-Logging-Modul für Hausverbrauch") hc_dict = {} diff --git a/packages/helpermodules/update_config.py b/packages/helpermodules/update_config.py index 3384fdd9ef..c7cffd711e 100644 --- a/packages/helpermodules/update_config.py +++ b/packages/helpermodules/update_config.py @@ -3068,3 +3068,35 @@ def upgrade(topic: str, payload) -> Optional[dict]: return {topic: payload} self._loop_all_received_topics(upgrade) self._append_datastore_version(121) + + def upgrade_datastore_122(self) -> None: + def convert_file(file): + try: + with open(file, "r+") as jsonFile: + content = json.load(jsonFile) + for entry in content["entries"]: + if entry.get("prices") is not None: + entry["prices"]["fault_state"] = None + for cp in entry["cp"].values(): + cp["fault_state"] = None + for ev_data in entry["ev"].values(): + ev_data["fault_state"] = None + for counter in entry["counter"].values(): + counter["fault_state"] = None + for pv in entry["pv"].values(): + pv["fault_state"] = None + for bat in entry["bat"].values(): + bat["fault_state"] = None + if entry.get("hc") is not None and entry["hc"].get("all") is not None: + entry["hc"]["all"]["fault_state"] = None + + jsonFile.seek(0) + json.dump(content, jsonFile) + jsonFile.truncate() + log.debug(f"Format der Logdatei '{file}' aktualisiert.") + except FileNotFoundError: + pass + except Exception: + log.exception(f"Logdatei '{file}' konnte nicht konvertiert werden.") + convert_file(f"{str(self.base_path / 'data' / 'daily_log')}/{timecheck.create_timestamp_YYYYMMDD()}.json") + self._append_datastore_version(122) diff --git a/packages/helpermodules/update_config_test.py b/packages/helpermodules/update_config_test.py index 7101720081..d3f11425ac 100644 --- a/packages/helpermodules/update_config_test.py +++ b/packages/helpermodules/update_config_test.py @@ -1,3 +1,5 @@ +from unittest.mock import Mock, patch, mock_open +from helpermodules import update_config import json from pathlib import Path @@ -54,3 +56,65 @@ def test_upgrade_datastore_94(index_test_template, expected_index): "scheduled_charging"]["plans"]: plan_ids.append(plan["id"]) assert plan_ids == expected_index + + +@pytest.mark.parametrize("name", [ + "happy_path", + "missing_prices_dict", + "empty_entry"]) +def test_upgrade_datastore_122(name, monkeypatch): + if name == "happy_path": + log_content = { + "entries": [ + { + "prices": {}, "cp": {"cp1": {}}, "ev": {"ev1": {}}, "counter": {"counter1": {}}, + "pv": {"pv1": {}}, "bat": {"bat1": {}}, "hc": {"all": {}} + } + ] + } + expected_content = { + "entries": [ + { + "prices": {"fault_state": None}, "cp": {"cp1": {"fault_state": None}}, + "ev": {"ev1": {"fault_state": None}}, "counter": {"counter1": {"fault_state": None}}, + "pv": {"pv1": {"fault_state": None}}, "bat": {"bat1": {"fault_state": None}}, + "hc": {"all": {"fault_state": None}} + } + ] + } + elif name == "missing_prices_dict": + log_content = { + "entries": [ + { + "cp": {"cp1": {}}, "ev": {"ev1": {}}, "counter": {"counter1": {}}, + "pv": {"pv1": {}}, "bat": {"bat1": {}}, "hc": {"all": {}} + } + ] + } + expected_content = { + "entries": [ + { + "cp": {"cp1": {"fault_state": None}}, + "ev": {"ev1": {"fault_state": None}}, "counter": {"counter1": {"fault_state": None}}, + "pv": {"pv1": {"fault_state": None}}, "bat": {"bat1": {"fault_state": None}}, + "hc": {"all": {"fault_state": None}} + } + ] + } + elif name == "empty_entry": + log_content = {"entries": []} + expected_content = {"entries": []} + # Arrange + uc = UpdateConfig() + uc.all_received_topics = {"openWB/system/datastore_version": []} + + mock_dump = Mock() + monkeypatch.setattr(update_config.json, "dump", mock_dump) + + # Act + with patch("builtins.open", mock_open(read_data=json.dumps(log_content))): + uc.upgrade_datastore_122() + + # Assert + assert mock_dump.call_args_list[0].args[0] == expected_content + assert uc.all_received_topics["openWB/system/datastore_version"] == [122] From 2f58515972b2b33fbc42dda66b246a25e1112c0e Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 29 Apr 2026 16:35:31 +0200 Subject: [PATCH 06/15] review, fault state for chargepoint all --- .../control/chargepoint/chargepoint_all.py | 39 +++++++++++++------ .../measurement_logging/process_log.py | 26 +++++++------ .../process_log_integration_test.py | 3 +- .../measurement_logging/write_log.py | 5 ++- packages/helpermodules/setdata.py | 4 ++ packages/helpermodules/update_config.py | 10 ++--- 6 files changed, 54 insertions(+), 33 deletions(-) diff --git a/packages/control/chargepoint/chargepoint_all.py b/packages/control/chargepoint/chargepoint_all.py index 02de7cd17d..d8bfce5294 100644 --- a/packages/control/chargepoint/chargepoint_all.py +++ b/packages/control/chargepoint/chargepoint_all.py @@ -3,6 +3,7 @@ from control import data from control.chargepoint.chargepoint_state import ChargepointState +from helpermodules.constants import NO_ERROR log = logging.getLogger(__name__) @@ -15,6 +16,8 @@ class AllGet: power: float = field(default=0, metadata={"topic": "get/power"}) imported: float = field(default=0, metadata={"topic": "get/imported"}) exported: float = field(default=0, metadata={"topic": "get/exported"}) + fault_state: int = field(default=0, metadata={"topic": "get/fault_state"}) + fault_str: str = field(default=NO_ERROR, metadata={"topic": "get/fault_str"}) def all_get_factory() -> AllGet: @@ -69,17 +72,29 @@ def get_cp_sum(self): imported, exported, power = 0, 0, 0 try: for cp in data.data.cp_data.values(): - try: - imported = imported + cp.data.get.imported - exported = exported + cp.data.get.exported - except Exception: - log.exception("Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt "+cp) - try: - power = power + cp.data.get.power - except Exception: - log.exception("Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt "+cp) - self.data.get.power = power - self.data.get.imported = imported - self.data.get.exported = exported + if cp.data.get.fault_state < 2: + try: + imported = imported + cp.data.get.imported + exported = exported + cp.data.get.exported + except Exception: + log.exception("Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt "+cp) + try: + power = power + cp.data.get.power + except Exception: + log.exception("Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt "+cp) + else: + if fault_state < cp.data.get.fault_state: + fault_state = cp.data.get.fault_state + if fault_state == 0: + self.data.get.power = power + self.data.get.imported = imported + self.data.get.exported = exported + self.data.get.fault_state = 0 + self.data.get.fault_str = NO_ERROR + else: + self.data.get.fault_state = fault_state + self.data.get.fault_str = ("Bitte die Statusmeldungen der Ladepunkte prüfen. Es konnte kein " + "aktueller Zählerstand ermittelt werden, da nicht alle Ladepunkte Werte " + "liefern.") except Exception: log.exception("Fehler in der allgemeinen Ladepunkt-Klasse") diff --git a/packages/helpermodules/measurement_logging/process_log.py b/packages/helpermodules/measurement_logging/process_log.py index 6d8bfd48f5..6117cd117b 100644 --- a/packages/helpermodules/measurement_logging/process_log.py +++ b/packages/helpermodules/measurement_logging/process_log.py @@ -314,13 +314,15 @@ def _analyse_energy_source(data, calc_cp: Optional[str] = None) -> Dict: for i in range(0, len(data["entries"])): data["entries"][i], message_analyse = analyse_percentage(data["entries"][i]) if calc_cp is not None: - data["entries"][i], message_calc = calc_energy_imported_by_source_cp(data["entries"][i], calc_cp) + data["entries"][i], message_calc = calc_energy_imported_by_source_cp( + data["entries"][i], calc_cp, data["names"][calc_cp]) else: - data["entries"][i], message_calc = calc_energy_imported_by_source_all(data["entries"][i]) + data["entries"][i], message_calc = calc_energy_imported_by_source_all( + data["entries"][i], data["names"]) + data["message"] += message_analyse + message_calc data["totals"] = analyse_percentage_totals(data["entries"], data["totals"]) except Exception: pub_system_message({}, "Fehler beim Berechnen des Strom-Mix", MessageType.ERROR) - data["message"] = message_analyse + message_calc return data @@ -401,7 +403,7 @@ def get_grid_error_state(entry) -> int: "Fehlerzustand befindet. Die Verbräuche werden mit 0 kWh angesetzt.\n") -def calc_energy_imported_by_source_all(entry) -> Tuple[Dict, str]: +def calc_energy_imported_by_source_all(entry, names) -> Tuple[Dict, str]: try: if "energy_source" not in entry.keys(): return entry, "" @@ -412,7 +414,7 @@ def calc_energy_imported_by_source_all(entry) -> Tuple[Dict, str]: if isinstance(hc_section, dict) and "all" in hc_section: hc_all = hc_section["all"] if isinstance(hc_all, dict): - if hc_all.get("fault_state", 0) == 0 and "energy_imported" in hc_all: + if hc_all.get("fault_state", 0) != 2 and "energy_imported" in hc_all: for source in ("grid", "pv", "bat", "cp"): hc_all[f"energy_imported_{source}"] = decimal_multiply( hc_all["energy_imported"], energy_source[source]) @@ -425,34 +427,34 @@ def calc_energy_imported_by_source_all(entry) -> Tuple[Dict, str]: if isinstance(cp_section, dict): for cp_key, cp_data in cp_section.items(): if isinstance(cp_data, dict): - if cp_data.get("fault_state", 0) == 0 and "energy_imported" in cp_data: + if cp_data.get("fault_state", 0) != 2 and "energy_imported" in cp_data: for source in ("grid", "pv", "bat", "cp"): cp_data[f"energy_imported_{source}"] = decimal_multiply( cp_data["energy_imported"], energy_source[source]) else: for source in ("grid", "pv", "bat", "cp"): cp_data[f"energy_imported_{source}"] = 0 - message += ERROR_STATE_MESSAGE.format(f"Ladepunkt {entry['names'][cp_key]}") + message += ERROR_STATE_MESSAGE.format(f"Ladepunkt {names[cp_key]}") counter_section = entry.get("counter") if isinstance(counter_section, dict): for counter in counter_section.values(): if isinstance(counter, dict) and counter.get("grid") is False: - if counter.get("fault_state", 0) == 0 and "energy_imported" in counter: + if counter.get("fault_state", 0) != 2 and "energy_imported" in counter: for source in ("grid", "pv", "bat", "cp"): counter[f"energy_imported_{source}"] = decimal_multiply( counter["energy_imported"], energy_source[source]) else: for source in ("grid", "pv", "bat", "cp"): counter[f"energy_imported_{source}"] = 0 - message += ERROR_STATE_MESSAGE.format(f"Zähler {entry['names'][counter]}") + message += ERROR_STATE_MESSAGE.format(f"Zähler {names[counter]}") except Exception: log.exception(f"Fehler beim Berechnen der Energie-Anteile aus dem Strom-Mix von {entry['timestamp']}") finally: return entry, message -def calc_energy_imported_by_source_cp(entry, cp: str) -> Tuple[Dict, str]: +def calc_energy_imported_by_source_cp(entry, cp: str, name) -> Tuple[Dict, str]: try: if "energy_source" not in entry.keys(): return entry, "" @@ -461,14 +463,14 @@ def calc_energy_imported_by_source_cp(entry, cp: str) -> Tuple[Dict, str]: message = "" cp_data = entry["cp"][cp] - if cp_data.get("fault_state", 0) == 0 and "energy_imported" in cp_data: + if cp_data.get("fault_state", 0) != 2 and "energy_imported" in cp_data: for source in ("grid", "pv", "bat", "cp"): cp_data[f"energy_imported_{source}"] = decimal_multiply( cp_data["energy_imported"], energy_source[source]) else: for source in ("grid", "pv", "bat", "cp"): cp_data[f"energy_imported_{source}"] = 0 - message += ERROR_STATE_MESSAGE.format(f"Ladepunkt {entry['names'][cp]}") + message += ERROR_STATE_MESSAGE.format(f"Ladepunkt {name[cp]}") except Exception: log.exception(f"Fehler beim Berechnen der Energie-Anteile aus dem Strom-Mix von {entry['timestamp']}") diff --git a/packages/helpermodules/measurement_logging/process_log_integration_test.py b/packages/helpermodules/measurement_logging/process_log_integration_test.py index 540740ef2b..3bb1156e0a 100644 --- a/packages/helpermodules/measurement_logging/process_log_integration_test.py +++ b/packages/helpermodules/measurement_logging/process_log_integration_test.py @@ -1,4 +1,3 @@ -from pprint import pprint from unittest.mock import Mock import pytest @@ -21,6 +20,6 @@ def test_get_daily_log(data, expected, monkeypatch): # execution daily_log_processed = process_log.get_daily_log("20250616") - pprint(daily_log_processed) + # evaluation assert daily_log_processed == expected diff --git a/packages/helpermodules/measurement_logging/write_log.py b/packages/helpermodules/measurement_logging/write_log.py index 68527985b4..620e9bfdfa 100644 --- a/packages/helpermodules/measurement_logging/write_log.py +++ b/packages/helpermodules/measurement_logging/write_log.py @@ -211,7 +211,7 @@ def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previou "fault_state": fault_state} except Exception: log.exception("Fehler im Werte-Logging-Modul für Preise") - prices_dict = {} + prices_dict = {"fault_state": 0} try: cp_dict = {"all": {"imported": data.data.cp_all_data.data.get.imported, @@ -254,7 +254,8 @@ def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previou log.exception("Fehler im Werte-Logging-Modul für Zähler "+str(counter)) try: - pv_dict = {"all": {"exported": data.data.pv_all_data.data.get.exported}} + pv_dict = {"all": {"exported": data.data.pv_all_data.data.get.exported, + "fault_state": data.data.pv_all_data.data.get.fault_state}} except Exception: log.exception("Fehler im Werte-Logging-Modul für PV-Daten") pv_dict = {} diff --git a/packages/helpermodules/setdata.py b/packages/helpermodules/setdata.py index 6df640429a..e3de04db83 100644 --- a/packages/helpermodules/setdata.py +++ b/packages/helpermodules/setdata.py @@ -467,6 +467,10 @@ def process_chargepoint_topic(self, msg: mqtt.MQTTMessage): self._validate_value(msg, float) elif re.search("chargepoint/[0-9]+/config/template$", msg.topic) is not None: self._validate_value(msg, int, pub_json=True) + elif "openWB/set/chargepoint/get/fault_state" in msg.topic: + self._validate_value(msg, int, [(0, 2)]) + elif "openWB/set/chargepoint/get/fault_str" in msg.topic: + self._validate_value(msg, str) elif "template" in msg.topic: self._validate_value(msg, "json") elif re.search("chargepoint/[0-9]+/config$", msg.topic) is not None: diff --git a/packages/helpermodules/update_config.py b/packages/helpermodules/update_config.py index c7cffd711e..6fd95d1367 100644 --- a/packages/helpermodules/update_config.py +++ b/packages/helpermodules/update_config.py @@ -3077,15 +3077,15 @@ def convert_file(file): for entry in content["entries"]: if entry.get("prices") is not None: entry["prices"]["fault_state"] = None - for cp in entry["cp"].values(): + for cp in entry.get("cp", {}).values(): cp["fault_state"] = None - for ev_data in entry["ev"].values(): + for ev_data in entry.get("ev", {}).values(): ev_data["fault_state"] = None - for counter in entry["counter"].values(): + for counter in entry.get("counter", {}).values(): counter["fault_state"] = None - for pv in entry["pv"].values(): + for pv in entry.get("pv", {}).values(): pv["fault_state"] = None - for bat in entry["bat"].values(): + for bat in entry.get("bat", {}).values(): bat["fault_state"] = None if entry.get("hc") is not None and entry["hc"].get("all") is not None: entry["hc"]["all"]["fault_state"] = None From eef26710e2a74da9d7354f0e1c10336504743f8d Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 30 Apr 2026 07:43:42 +0200 Subject: [PATCH 07/15] update datastore version --- packages/helpermodules/update_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/helpermodules/update_config.py b/packages/helpermodules/update_config.py index 6fd95d1367..e2f50cf9e5 100644 --- a/packages/helpermodules/update_config.py +++ b/packages/helpermodules/update_config.py @@ -57,7 +57,7 @@ class UpdateConfig: - DATASTORE_VERSION = 121 + DATASTORE_VERSION = 122 valid_topic = [ "^openWB/bat/config/bat_control_permitted$", From 51847221044410fc62bd8e7a67b849dc109b372f Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 30 Apr 2026 08:17:23 +0200 Subject: [PATCH 08/15] review --- packages/control/chargelog/chargelog.py | 1 + .../control/chargepoint/chargepoint_all.py | 1 + .../measurement_logging/process_log.py | 20 ++++++++++--------- .../measurement_logging/write_log.py | 1 + 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/control/chargelog/chargelog.py b/packages/control/chargelog/chargelog.py index 1c9c8bd7aa..0b3ba19282 100644 --- a/packages/control/chargelog/chargelog.py +++ b/packages/control/chargelog/chargelog.py @@ -401,6 +401,7 @@ def _get_reference_entries(cp) -> Tuple[List[Dict], List]: reference_entries = [] try: entries = get_todays_daily_log()["entries"] + reference_entries["names"] = entries["names"] if len(entries) >= 2: reference_entries = [entries[-2], entries[-1]] else: diff --git a/packages/control/chargepoint/chargepoint_all.py b/packages/control/chargepoint/chargepoint_all.py index d8bfce5294..daededb8f7 100644 --- a/packages/control/chargepoint/chargepoint_all.py +++ b/packages/control/chargepoint/chargepoint_all.py @@ -71,6 +71,7 @@ def get_cp_sum(self): """ imported, exported, power = 0, 0, 0 try: + fault_state = 0 for cp in data.data.cp_data.values(): if cp.data.get.fault_state < 2: try: diff --git a/packages/helpermodules/measurement_logging/process_log.py b/packages/helpermodules/measurement_logging/process_log.py index 6117cd117b..677fc01fe4 100644 --- a/packages/helpermodules/measurement_logging/process_log.py +++ b/packages/helpermodules/measurement_logging/process_log.py @@ -311,6 +311,8 @@ def add_daily_log(day: str) -> None: def _analyse_energy_source(data, calc_cp: Optional[str] = None) -> Dict: if data and len(data["entries"]) > 0: try: + if data.get("message") is None: + data["message"] = "" for i in range(0, len(data["entries"])): data["entries"][i], message_analyse = analyse_percentage(data["entries"][i]) if calc_cp is not None: @@ -438,23 +440,23 @@ def calc_energy_imported_by_source_all(entry, names) -> Tuple[Dict, str]: counter_section = entry.get("counter") if isinstance(counter_section, dict): - for counter in counter_section.values(): - if isinstance(counter, dict) and counter.get("grid") is False: - if counter.get("fault_state", 0) != 2 and "energy_imported" in counter: + for counter_key, counter_data in counter_section.items(): + if isinstance(counter_data, dict) and counter_data.get("grid") is False: + if counter_data.get("fault_state", 0) != 2 and "energy_imported" in counter_data: for source in ("grid", "pv", "bat", "cp"): - counter[f"energy_imported_{source}"] = decimal_multiply( - counter["energy_imported"], energy_source[source]) + counter_data[f"energy_imported_{source}"] = decimal_multiply( + counter_data["energy_imported"], energy_source[source]) else: for source in ("grid", "pv", "bat", "cp"): - counter[f"energy_imported_{source}"] = 0 - message += ERROR_STATE_MESSAGE.format(f"Zähler {names[counter]}") + counter_data[f"energy_imported_{source}"] = 0 + message += ERROR_STATE_MESSAGE.format(f"Zähler {names[counter_key]}") except Exception: log.exception(f"Fehler beim Berechnen der Energie-Anteile aus dem Strom-Mix von {entry['timestamp']}") finally: return entry, message -def calc_energy_imported_by_source_cp(entry, cp: str, name) -> Tuple[Dict, str]: +def calc_energy_imported_by_source_cp(entry, cp: str, name: str) -> Tuple[Dict, str]: try: if "energy_source" not in entry.keys(): return entry, "" @@ -470,7 +472,7 @@ def calc_energy_imported_by_source_cp(entry, cp: str, name) -> Tuple[Dict, str]: else: for source in ("grid", "pv", "bat", "cp"): cp_data[f"energy_imported_{source}"] = 0 - message += ERROR_STATE_MESSAGE.format(f"Ladepunkt {name[cp]}") + message += ERROR_STATE_MESSAGE.format(f"Ladepunkt {name}") except Exception: log.exception(f"Fehler beim Berechnen der Energie-Anteile aus dem Strom-Mix von {entry['timestamp']}") diff --git a/packages/helpermodules/measurement_logging/write_log.py b/packages/helpermodules/measurement_logging/write_log.py index 620e9bfdfa..8f3cc95e70 100644 --- a/packages/helpermodules/measurement_logging/write_log.py +++ b/packages/helpermodules/measurement_logging/write_log.py @@ -204,6 +204,7 @@ def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previou data.data.optional_data.data.electricity_pricing.grid_fee.get.fault_state) except Exception: grid_price = prices.grid + fault_state = 0 prices_dict = {"grid": grid_price, "pv": prices.pv, "bat": prices.bat, From 891c02ec0806f5d65972c1649c091ab825e08623 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 30 Apr 2026 09:22:17 +0200 Subject: [PATCH 09/15] review --- packages/control/chargelog/chargelog.py | 6 ++++-- packages/control/chargelog/chargelog_test.py | 10 +++++++--- .../measurement_logging/process_log_unit_test.py | 2 +- .../helpermodules/measurement_logging/write_log.py | 5 +++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/control/chargelog/chargelog.py b/packages/control/chargelog/chargelog.py index 0b3ba19282..3accd3b9fe 100644 --- a/packages/control/chargelog/chargelog.py +++ b/packages/control/chargelog/chargelog.py @@ -400,8 +400,9 @@ def _get_reference_entries(cp) -> Tuple[List[Dict], List]: processed_entries = {} reference_entries = [] try: - entries = get_todays_daily_log()["entries"] - reference_entries["names"] = entries["names"] + log_data = get_todays_daily_log() + names = log_data["names"] + entries = log_data["entries"] if len(entries) >= 2: reference_entries = [entries[-2], entries[-1]] else: @@ -410,6 +411,7 @@ def _get_reference_entries(cp) -> Tuple[List[Dict], List]: reference_entries = [entries_day_before[-1], entries[0]] processed_entries["entries"] = copy.deepcopy(reference_entries) processed_entries["entries"] = _process_entries(processed_entries["entries"], CalculationType.ENERGY) + processed_entries["names"] = names processed_entries["totals"] = get_totals(processed_entries["entries"], False) processed_entries = _analyse_energy_source(processed_entries, f"cp{cp.num}") except Exception: diff --git a/packages/control/chargelog/chargelog_test.py b/packages/control/chargelog/chargelog_test.py index 1dbae9aa11..ec174b5c21 100644 --- a/packages/control/chargelog/chargelog_test.py +++ b/packages/control/chargelog/chargelog_test.py @@ -44,7 +44,8 @@ def mock_daily_log(monkeypatch): 'pv': {'all': {'exported': 2500}, 'pv1': {'exported': 2500}}, 'sh': {}, 'timestamp': 1652683200, - 'prices': {'grid': 0.0003, 'pv': 0.00015, 'bat': 0.0002, 'cp': 0}}]} + 'prices': {'grid': 0.0003, 'pv': 0.00015, 'bat': 0.0002, 'cp': 0}}], + "names": {"bat2": "Speicher2", "cp4": "LP 4", "pv1": "PV 1", "ev0": "EV0", "counter0": "Zähler0"}} mock_todays_daily_log = Mock(return_value=daily_log) monkeypatch.setattr(chargelog, "get_todays_daily_log", mock_todays_daily_log) return daily_log @@ -115,7 +116,9 @@ def test_calc_charge_cost_reference_middle_day_change(mock_data, monkeypatch): 'pv': {'all': {'exported': 2000}, 'pv1': {'exported': 2000}}, 'sh': {}, 'timestamp': 1652682900, - 'prices': {'grid': 0.0003, 'pv': 0.00015, 'bat': 0.0002, 'cp': 0}}]} + 'prices': {'grid': 0.0003, 'pv': 0.00015, 'bat': 0.0002, 'cp': 0}}], + "names": { + "bat2": "Speicher2", "cp4": "LP 4", "pv1": "PV 1", "ev0": "EV0", "counter0": "Zähler0"}} mock_yesterdays_daily_log = Mock(return_value=yesterday_daily_log) monkeypatch.setattr(chargelog, "get_daily_log", mock_yesterdays_daily_log) @@ -132,7 +135,8 @@ def test_calc_charge_cost_reference_middle_day_change(mock_data, monkeypatch): 'pv': {'all': {'exported': 2500}, 'pv1': {'exported': 2500}}, 'sh': {}, 'timestamp': 1652683200, - 'prices': {'grid': 0.0003, 'pv': 0.00015, 'bat': 0.0002, 'cp': 0}}]} + 'prices': {'grid': 0.0003, 'pv': 0.00015, 'bat': 0.0002, 'cp': 0}}], + "names": {"bat2": "Speicher2", "cp4": "LP 4", "pv1": "PV 1", "ev0": "EV0", "counter0": "Zähler0"}} mock_todays_daily_log = Mock(return_value=daily_log) monkeypatch.setattr(chargelog, "get_todays_daily_log", mock_todays_daily_log) diff --git a/packages/helpermodules/measurement_logging/process_log_unit_test.py b/packages/helpermodules/measurement_logging/process_log_unit_test.py index 607282296b..f4649f6b2b 100644 --- a/packages/helpermodules/measurement_logging/process_log_unit_test.py +++ b/packages/helpermodules/measurement_logging/process_log_unit_test.py @@ -131,7 +131,7 @@ def test_calc_energy_imported_by_source_all(): } # execution - result, message = calc_energy_imported_by_source_all(entry) + result, message = calc_energy_imported_by_source_all(entry, {}) # evaluation - realistic Wh values with decimal precision assert result["hc"]["all"]["energy_imported_grid"] == 1530.035 diff --git a/packages/helpermodules/measurement_logging/write_log.py b/packages/helpermodules/measurement_logging/write_log.py index 8f3cc95e70..48fbd707c8 100644 --- a/packages/helpermodules/measurement_logging/write_log.py +++ b/packages/helpermodules/measurement_logging/write_log.py @@ -288,8 +288,9 @@ def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previou log.exception("Fehler im Werte-Logging-Modul für Speicher "+str(bat)) try: - hc_dict = {"all": {"imported": data.data.counter_all_data.data.set.imported_home_consumption, - "fault_state": data.data.counter_all_data.data.set.invalid_home_consumption}} + hc_dict = {"all": { + "imported": data.data.counter_all_data.data.set.imported_home_consumption, + "fault_state": 3 if data.data.counter_all_data.data.set.invalid_home_consumption >= 3 else 0}} except Exception: log.exception("Fehler im Werte-Logging-Modul für Hausverbrauch") hc_dict = {} From 65b482c2233c70c6534357214a2e906237e9bf6d Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 30 Apr 2026 09:53:07 +0200 Subject: [PATCH 10/15] review --- .../control/chargepoint/chargepoint_all.py | 11 ++++---- .../measurement_logging/process_log.py | 25 ++++++++----------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/control/chargepoint/chargepoint_all.py b/packages/control/chargepoint/chargepoint_all.py index daededb8f7..0a70f2bd4d 100644 --- a/packages/control/chargepoint/chargepoint_all.py +++ b/packages/control/chargepoint/chargepoint_all.py @@ -78,18 +78,19 @@ def get_cp_sum(self): imported = imported + cp.data.get.imported exported = exported + cp.data.get.exported except Exception: - log.exception("Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt "+cp) + log.exception("Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt "+cp.num) try: power = power + cp.data.get.power except Exception: - log.exception("Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt "+cp) + log.exception("Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt "+cp.num) else: if fault_state < cp.data.get.fault_state: fault_state = cp.data.get.fault_state + # Ladepunkte setzen ihre Werte im Fehlerfall selbst zurück + self.data.get.power = power + self.data.get.imported = imported + self.data.get.exported = exported if fault_state == 0: - self.data.get.power = power - self.data.get.imported = imported - self.data.get.exported = exported self.data.get.fault_state = 0 self.data.get.fault_str = NO_ERROR else: diff --git a/packages/helpermodules/measurement_logging/process_log.py b/packages/helpermodules/measurement_logging/process_log.py index 677fc01fe4..4be9e2d2cd 100644 --- a/packages/helpermodules/measurement_logging/process_log.py +++ b/packages/helpermodules/measurement_logging/process_log.py @@ -336,25 +336,21 @@ def analyse_percentage(entry) -> Tuple[Dict, str]: def format(value): return round(value, 4) - def get_grid_from(entry) -> Tuple[float, float]: - grids = [counter for counter in entry["counter"].values() if counter.get("grid")] - if not grids: - raise KeyError(f"Kein Zähler für das Netz gefunden in Eintrag '{entry['timestamp']}'.") - return sum(grid["energy_imported"] for grid in grids), sum(grid["energy_exported"] for grid in grids) - - def get_grid_error_state(entry) -> int: - grids = [counter for counter in entry["counter"].values() if counter.get("grid")] - if not grids: + def get_grid_counter(entry) -> Dict: + # es gibt nur einen Zähler am EVU-Punkt + for counter in entry["counter"].values(): + if counter.get("grid") is True: + return counter + else: raise KeyError(f"Kein Zähler für das Netz gefunden in Eintrag '{entry['timestamp']}'.") - max_grid = max(grids, key=lambda g: g.get("fault_state", 0)) - return max_grid.get("fault_state", 0) try: message = "" + grid_counter = get_grid_counter(entry) if (safe_get_nested(entry, "bat", "all", "fault_state") == 2 or safe_get_nested(entry, "cp", "all", "fault_state") == 2 or safe_get_nested(entry, "pv", "all", "fault_state") == 2 or - get_grid_error_state(entry) == 2): + grid_counter.get("fault_state", None) == 2): entry["energy_source"] = {"grid": 1, "pv": 0, "bat": 0, "cp": 0} if safe_get_nested(entry, "bat", "all", "fault_state") == 2: @@ -363,7 +359,7 @@ def get_grid_error_state(entry) -> int: message += EOOR_STATE_MSG.format("mind. einer der Ladepunkte") if safe_get_nested(entry, "pv", "all", "fault_state") == 2: message += EOOR_STATE_MSG.format("mind. einer der Wechselrichter") - if get_grid_error_state(entry) == 2: + if grid_counter.get("fault_state", None) == 2: message += EOOR_STATE_MSG.format("der Zähler für das Netz") else: @@ -371,7 +367,8 @@ def get_grid_error_state(entry) -> int: bat_exported = safe_get_nested(entry, "bat", "all", "energy_exported") cp_exported = safe_get_nested(entry, "cp", "all", "energy_exported") pv_exported = safe_get_nested(entry, "pv", "all", "energy_exported") - grid_imported, grid_exported = get_grid_from(entry) + grid_imported = grid_counter.get("energy_imported", 0) + grid_exported = grid_counter.get("energy_exported", 0) consumption = grid_imported - grid_exported + pv_exported + bat_exported - bat_imported + cp_exported if consumption < 0: consumption = 0 From 30b7115aab55d2966e77a5dfa5c82bfac24ee869 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 30 Apr 2026 10:06:03 +0200 Subject: [PATCH 11/15] review --- packages/control/chargepoint/chargepoint_all.py | 4 ++-- packages/helpermodules/measurement_logging/process_log.py | 2 +- packages/helpermodules/measurement_logging/write_log.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/control/chargepoint/chargepoint_all.py b/packages/control/chargepoint/chargepoint_all.py index 0a70f2bd4d..381678fea1 100644 --- a/packages/control/chargepoint/chargepoint_all.py +++ b/packages/control/chargepoint/chargepoint_all.py @@ -78,11 +78,11 @@ def get_cp_sum(self): imported = imported + cp.data.get.imported exported = exported + cp.data.get.exported except Exception: - log.exception("Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt "+cp.num) + log.exception(f"Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt {cp.num}") try: power = power + cp.data.get.power except Exception: - log.exception("Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt "+cp.num) + log.exception(f"Fehler in der allgemeinen Ladepunkt-Klasse für Ladepunkt {cp.num}") else: if fault_state < cp.data.get.fault_state: fault_state = cp.data.get.fault_state diff --git a/packages/helpermodules/measurement_logging/process_log.py b/packages/helpermodules/measurement_logging/process_log.py index 4be9e2d2cd..5ba1bf322c 100644 --- a/packages/helpermodules/measurement_logging/process_log.py +++ b/packages/helpermodules/measurement_logging/process_log.py @@ -433,7 +433,7 @@ def calc_energy_imported_by_source_all(entry, names) -> Tuple[Dict, str]: else: for source in ("grid", "pv", "bat", "cp"): cp_data[f"energy_imported_{source}"] = 0 - message += ERROR_STATE_MESSAGE.format(f"Ladepunkt {names[cp_key]}") + message += ERROR_STATE_MESSAGE.format(f"Ladepunkt {names.get(cp_key, cp_key)}") counter_section = entry.get("counter") if isinstance(counter_section, dict): diff --git a/packages/helpermodules/measurement_logging/write_log.py b/packages/helpermodules/measurement_logging/write_log.py index 48fbd707c8..726ad8cdf0 100644 --- a/packages/helpermodules/measurement_logging/write_log.py +++ b/packages/helpermodules/measurement_logging/write_log.py @@ -290,7 +290,7 @@ def create_entry(log_type: LogType, sh_log_data: LegacySmartHomeLogData, previou try: hc_dict = {"all": { "imported": data.data.counter_all_data.data.set.imported_home_consumption, - "fault_state": 3 if data.data.counter_all_data.data.set.invalid_home_consumption >= 3 else 0}} + "fault_state": 2 if data.data.counter_all_data.data.set.invalid_home_consumption >= 3 else 0}} except Exception: log.exception("Fehler im Werte-Logging-Modul für Hausverbrauch") hc_dict = {} From e1e95ad14c2a7bfe7ad95706c1848c7d43952f3f Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 30 Apr 2026 10:44:22 +0200 Subject: [PATCH 12/15] error handling --- .../measurement_logging/process_log.py | 5 ++ packages/helpermodules/update_config.py | 59 ++++++++++--------- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/packages/helpermodules/measurement_logging/process_log.py b/packages/helpermodules/measurement_logging/process_log.py index 5ba1bf322c..3c2723188d 100644 --- a/packages/helpermodules/measurement_logging/process_log.py +++ b/packages/helpermodules/measurement_logging/process_log.py @@ -324,7 +324,9 @@ def _analyse_energy_source(data, calc_cp: Optional[str] = None) -> Dict: data["message"] += message_analyse + message_calc data["totals"] = analyse_percentage_totals(data["entries"], data["totals"]) except Exception: + log.exception("Fehler beim Analysieren der Energiequellen") pub_system_message({}, "Fehler beim Berechnen des Strom-Mix", MessageType.ERROR) + data["message"] = "Fehler beim Berechnen des Strom-Mix." return data @@ -394,6 +396,7 @@ def get_grid_counter(entry) -> Dict: entry["energy_source"] = {"grid": 0, "pv": 0, "bat": 0, "cp": 0} except Exception: log.exception(f"Fehler beim Berechnen des Strom-Mix von {entry['timestamp']}") + message += f"Fehler beim Berechnen des Strom-Mix von {entry['timestamp']}.\n" finally: return entry, message @@ -449,6 +452,7 @@ def calc_energy_imported_by_source_all(entry, names) -> Tuple[Dict, str]: message += ERROR_STATE_MESSAGE.format(f"Zähler {names[counter_key]}") except Exception: log.exception(f"Fehler beim Berechnen der Energie-Anteile aus dem Strom-Mix von {entry['timestamp']}") + message += f"Fehler beim Berechnen des Strom-Mix von {entry['timestamp']}.\n" finally: return entry, message @@ -473,6 +477,7 @@ def calc_energy_imported_by_source_cp(entry, cp: str, name: str) -> Tuple[Dict, except Exception: log.exception(f"Fehler beim Berechnen der Energie-Anteile aus dem Strom-Mix von {entry['timestamp']}") + message += f"Fehler beim Berechnen des Strom-Mix von {entry['timestamp']}.\n" finally: return entry, message diff --git a/packages/helpermodules/update_config.py b/packages/helpermodules/update_config.py index e2f50cf9e5..c6d5c2c2cc 100644 --- a/packages/helpermodules/update_config.py +++ b/packages/helpermodules/update_config.py @@ -3070,33 +3070,34 @@ def upgrade(topic: str, payload) -> Optional[dict]: self._append_datastore_version(121) def upgrade_datastore_122(self) -> None: - def convert_file(file): - try: - with open(file, "r+") as jsonFile: - content = json.load(jsonFile) - for entry in content["entries"]: - if entry.get("prices") is not None: - entry["prices"]["fault_state"] = None - for cp in entry.get("cp", {}).values(): - cp["fault_state"] = None - for ev_data in entry.get("ev", {}).values(): - ev_data["fault_state"] = None - for counter in entry.get("counter", {}).values(): - counter["fault_state"] = None - for pv in entry.get("pv", {}).values(): - pv["fault_state"] = None - for bat in entry.get("bat", {}).values(): - bat["fault_state"] = None - if entry.get("hc") is not None and entry["hc"].get("all") is not None: - entry["hc"]["all"]["fault_state"] = None + for folder in ("daily_log", "monthly_log"): + path_list = Path(Path(__file__).resolve().parents[2]/"data"/folder).glob('**/*.json') + for path in path_list: + with open(path, "r+") as jsonFile: + try: + content = json.load(jsonFile) + for entry in content["entries"]: + if entry.get("prices") is not None: + entry["prices"]["fault_state"] = None + for cp in entry.get("cp", {}).values(): + cp["fault_state"] = None + for ev_data in entry.get("ev", {}).values(): + ev_data["fault_state"] = None + for counter in entry.get("counter", {}).values(): + counter["fault_state"] = None + for pv in entry.get("pv", {}).values(): + pv["fault_state"] = None + for bat in entry.get("bat", {}).values(): + bat["fault_state"] = None + if entry.get("hc") is not None and entry["hc"].get("all") is not None: + entry["hc"]["all"]["fault_state"] = None - jsonFile.seek(0) - json.dump(content, jsonFile) - jsonFile.truncate() - log.debug(f"Format der Logdatei '{file}' aktualisiert.") - except FileNotFoundError: - pass - except Exception: - log.exception(f"Logdatei '{file}' konnte nicht konvertiert werden.") - convert_file(f"{str(self.base_path / 'data' / 'daily_log')}/{timecheck.create_timestamp_YYYYMMDD()}.json") - self._append_datastore_version(122) + jsonFile.seek(0) + json.dump(content, jsonFile) + jsonFile.truncate() + log.debug(f"Format der Logdatei '{path}' aktualisiert.") + except FileNotFoundError: + pass + except Exception: + log.exception(f"Logdatei '{path}' konnte nicht konvertiert werden.") + # self._append_datastore_version(122) From 3770306d6bb5438178dde563e22d2b98629749f0 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 30 Apr 2026 12:47:40 +0200 Subject: [PATCH 13/15] fixes --- packages/control/chargelog/chargelog.py | 2 +- .../control/chargepoint/chargepoint_data.py | 2 +- packages/helpermodules/update_config.py | 20 ++++++++++++------- .../helpermodules/utils/error_handling.py | 2 +- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/control/chargelog/chargelog.py b/packages/control/chargelog/chargelog.py index 3accd3b9fe..9afd560984 100644 --- a/packages/control/chargelog/chargelog.py +++ b/packages/control/chargelog/chargelog.py @@ -354,7 +354,7 @@ def calculate_charged_energy_by_source(cp, processed_entries, reference_entries, charged_energy = (reference_entries[-1]["cp"][f"cp{cp.num}"]["imported"] - reference_entries[0]["cp"][f"cp{cp.num}"]["imported"]) elif reference == ReferenceTime.END: - if ((timecheck.create_timestamp()-cp.data.set.log.timestamp_mode_switch) < MEASUREMENT_LOGGING_INTERVAL): + if sum(cp.data.set.log.charged_energy_by_source.values()) == 0: charged_energy = cp.data.set.log.imported_since_mode_switch else: log.debug(f"cp.data.get.imported {cp.data.get.imported}") diff --git a/packages/control/chargepoint/chargepoint_data.py b/packages/control/chargepoint/chargepoint_data.py index e112386f0d..f47e698735 100644 --- a/packages/control/chargepoint/chargepoint_data.py +++ b/packages/control/chargepoint/chargepoint_data.py @@ -115,7 +115,7 @@ class Get: currents: List[float] = field(default_factory=currents_list_factory) daily_imported: float = 0 daily_exported: float = 0 - error_timestamp: int = 0 + error_timestamp: Optional[int] = None evse_current: Optional[float] = None # kann auch zur Laufzeit geändert werden evse_signaling: Optional[EvseSignaling] = None diff --git a/packages/helpermodules/update_config.py b/packages/helpermodules/update_config.py index c6d5c2c2cc..f0be2892db 100644 --- a/packages/helpermodules/update_config.py +++ b/packages/helpermodules/update_config.py @@ -3077,20 +3077,26 @@ def upgrade_datastore_122(self) -> None: try: content = json.load(jsonFile) for entry in content["entries"]: - if entry.get("prices") is not None: + if entry.get("prices") is not None and entry["prices"].get("fault_state") is None: entry["prices"]["fault_state"] = None for cp in entry.get("cp", {}).values(): - cp["fault_state"] = None + if cp.get("fault_state") is None: + cp["fault_state"] = None for ev_data in entry.get("ev", {}).values(): - ev_data["fault_state"] = None + if ev_data.get("fault_state") is None: + ev_data["fault_state"] = None for counter in entry.get("counter", {}).values(): - counter["fault_state"] = None + if counter.get("fault_state") is None: + counter["fault_state"] = None for pv in entry.get("pv", {}).values(): - pv["fault_state"] = None + if pv.get("fault_state") is None: + pv["fault_state"] = None for bat in entry.get("bat", {}).values(): - bat["fault_state"] = None + if bat.get("fault_state") is None: + bat["fault_state"] = None if entry.get("hc") is not None and entry["hc"].get("all") is not None: - entry["hc"]["all"]["fault_state"] = None + if entry["hc"]["all"].get("fault_state") is None: + entry["hc"]["all"]["fault_state"] = None jsonFile.seek(0) json.dump(content, jsonFile) diff --git a/packages/helpermodules/utils/error_handling.py b/packages/helpermodules/utils/error_handling.py index ee2e2810f8..7695451ea8 100644 --- a/packages/helpermodules/utils/error_handling.py +++ b/packages/helpermodules/utils/error_handling.py @@ -47,7 +47,7 @@ def __exit__(self, exception_type, exception, exception_traceback) -> bool: return True def error_counter_exceeded(self) -> bool: - if self.error_timestamp and timecheck.check_timestamp(self.error_timestamp, self.timeout) is False: + if self.error_timestamp is not None and timecheck.check_timestamp(self.error_timestamp, self.timeout) is False: log.error(self.__exceeded_msg) return True else: From 0855a4c77bd3dfbc891ac2641185bedd6f2f16c1 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 30 Apr 2026 12:57:03 +0200 Subject: [PATCH 14/15] undo comment --- packages/helpermodules/update_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/helpermodules/update_config.py b/packages/helpermodules/update_config.py index f0be2892db..c5a6c1bbaa 100644 --- a/packages/helpermodules/update_config.py +++ b/packages/helpermodules/update_config.py @@ -3106,4 +3106,4 @@ def upgrade_datastore_122(self) -> None: pass except Exception: log.exception(f"Logdatei '{path}' konnte nicht konvertiert werden.") - # self._append_datastore_version(122) + self._append_datastore_version(122) From 14f6911c0c624ca97d6645b57a908e54814eedff Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Thu, 30 Apr 2026 15:43:29 +0200 Subject: [PATCH 15/15] mock path --- packages/helpermodules/update_config_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/helpermodules/update_config_test.py b/packages/helpermodules/update_config_test.py index d3f11425ac..6aa1001c75 100644 --- a/packages/helpermodules/update_config_test.py +++ b/packages/helpermodules/update_config_test.py @@ -110,6 +110,8 @@ def test_upgrade_datastore_122(name, monkeypatch): mock_dump = Mock() monkeypatch.setattr(update_config.json, "dump", mock_dump) + mock_glob = Mock(return_value=["dummy_path"]) + monkeypatch.setattr(update_config.Path, "glob", mock_glob) # Act with patch("builtins.open", mock_open(read_data=json.dumps(log_content))):