From 1eba8a03d317136a457846e99efa42bd8e087311 Mon Sep 17 00:00:00 2001 From: nathanmetzger Date: Sat, 25 Jul 2026 17:52:08 +0200 Subject: [PATCH] Move health monitor to standalone repository --- .../health-monitor/xbattlax/README.md | 127 ++-------- .../docs/recovery_safety_integration.md | 57 ----- .../xbattlax/docs/stack_watchdog_contract.md | 110 --------- .../launch/health_monitor.launch.py | 15 -- .../oomwoo_health_monitor/__init__.py | 1 - .../oomwoo_health_monitor/core.py | 175 -------------- .../health_monitor_node.py | 110 --------- .../oomwoo_health_monitor/package.xml | 20 -- .../resource/oomwoo_health_monitor | 1 - .../xbattlax/oomwoo_health_monitor/setup.cfg | 4 - .../xbattlax/oomwoo_health_monitor/setup.py | 27 --- .../tests/test_health_monitor_node_adapter.py | 227 ------------------ .../tests/test_oomwoo_health_monitor.py | 122 ---------- .../xbattlax/tools/sim_health_monitor.py | 71 ------ docs/SOFTWARE_INTERFACES.md | 15 ++ 15 files changed, 34 insertions(+), 1048 deletions(-) delete mode 100644 contributions/health-monitor/xbattlax/docs/recovery_safety_integration.md delete mode 100644 contributions/health-monitor/xbattlax/docs/stack_watchdog_contract.md delete mode 100644 contributions/health-monitor/xbattlax/oomwoo_health_monitor/launch/health_monitor.launch.py delete mode 100644 contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/__init__.py delete mode 100644 contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/core.py delete mode 100644 contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/health_monitor_node.py delete mode 100644 contributions/health-monitor/xbattlax/oomwoo_health_monitor/package.xml delete mode 100644 contributions/health-monitor/xbattlax/oomwoo_health_monitor/resource/oomwoo_health_monitor delete mode 100644 contributions/health-monitor/xbattlax/oomwoo_health_monitor/setup.cfg delete mode 100644 contributions/health-monitor/xbattlax/oomwoo_health_monitor/setup.py delete mode 100644 contributions/health-monitor/xbattlax/tests/test_health_monitor_node_adapter.py delete mode 100644 contributions/health-monitor/xbattlax/tests/test_oomwoo_health_monitor.py delete mode 100644 contributions/health-monitor/xbattlax/tools/sim_health_monitor.py diff --git a/contributions/health-monitor/xbattlax/README.md b/contributions/health-monitor/xbattlax/README.md index c4948c3..b920e05 100644 --- a/contributions/health-monitor/xbattlax/README.md +++ b/contributions/health-monitor/xbattlax/README.md @@ -1,114 +1,25 @@ -# Health Monitor Prototype by xbattlax +# Health Monitor by xbattlax -This contribution turns the health-monitor RFC into a concrete, testable -software-watchdog contract. +The maintained ROS 2 package, tests, simulator, and detailed contract docs now +live in +[xbattlax/health-monitor](https://github.com/xbattlax/health-monitor). -It answers the follow-up from `oomwoo#33`: `/cmd_vel` holding is only one local -freshness concern. The more general safety path is a stack health aggregator that -checks all expected critical components and emits one MCU-facing stack heartbeat -only while the active task is fully healthy. +Stable release: +[v0.1.0](https://github.com/xbattlax/health-monitor/releases/tag/v0.1.0). -## What This Adds +This code was first accepted in +[oomwoo#36](https://github.com/makerspet/oomwoo/pull/36) and moved to a +self-hosted package at the maintainer's request so it can be integrated, +tested, released, and vendored independently. -- a dependency-free health aggregator core -- a roster contract for critical vs. advisory components -- freshness and health-level checks for component heartbeats -- a fail-safe arming window before MCU heartbeat emission -- immediate heartbeat withholding on missing, stale, or unhealthy critical - components -- advisory-component reporting without stopping the robot -- a JSONL simulator for startup, stale-node, recovery, and advisory-fault traces -- unit tests covering the deadman behavior -- a proposed public ROS2 topic contract in `docs/SOFTWARE_INTERFACES.md` +The standalone repository includes: -## Files +- the `oomwoo_health_monitor` ROS 2 Jazzy package +- dependency-free core and adapter tests +- a deterministic health/fault simulator +- stack-watchdog and recovery-safety integration documents +- CI for the core and a real ROS 2 Jazzy package build -| File | Purpose | -|---|---| -| [`oomwoo_health_monitor/`](oomwoo_health_monitor/) | ROS2 package with the stack-health core and `health_monitor_node`. | -| [`tools/sim_health_monitor.py`](tools/sim_health_monitor.py) | Deterministic JSONL scenario runner. | -| [`tests/test_oomwoo_health_monitor.py`](tests/test_oomwoo_health_monitor.py) | Regression tests for arming, stale detection, advisory handling, and roster changes. | -| [`tests/test_health_monitor_node_adapter.py`](tests/test_health_monitor_node_adapter.py) | ROS2 adapter tests using lightweight stubs. | -| [`docs/stack_watchdog_contract.md`](docs/stack_watchdog_contract.md) | Detailed heartbeat, roster, aggregate, and MCU behavior contract. | -| [`docs/recovery_safety_integration.md`](docs/recovery_safety_integration.md) | How recovery-safety should feed the monitor without relying on `/cmd_vel` timeouts. | - -## Package - -`oomwoo_health_monitor` - -Location: - -```text -contributions/health-monitor/xbattlax/oomwoo_health_monitor -``` - -## Build - -From the OOMWOO ROS2 container: - -```bash -source /opt/ros/jazzy/setup.bash -cd /workspace -colcon build \ - --base-paths contributions/health-monitor/xbattlax/oomwoo_health_monitor \ - --packages-select oomwoo_health_monitor -``` - -## Test - -```bash -python3 -m unittest discover \ - -s contributions/health-monitor/xbattlax/tests \ - -p 'test_*.py' -``` - -## Run the Simulator - -```bash -python3 contributions/health-monitor/xbattlax/tools/sim_health_monitor.py -``` - -The output is JSON Lines with the current stack state, stale/missing/unhealthy -components, advisory faults, and whether the MCU heartbeat would be emitted. - -## Intended ROS2 Path - -Initial ROS2 integration can use JSON over `std_msgs/msg/String` while message -types are still unsettled: - -- `/oomwoo/health/roster` -- `/oomwoo/health/component` -- `/oomwoo/health/stack` -- `/oomwoo/health/mcu_heartbeat` - -## Run - -```bash -source /opt/ros/jazzy/setup.bash -source install/setup.bash -ros2 launch oomwoo_health_monitor health_monitor.launch.py -``` - -Publish a sample roster: - -```bash -ros2 topic pub --once /oomwoo/health/roster std_msgs/msg/String \ - "{data: '{\"task_id\":\"dock_cycle\",\"components\":[{\"component_id\":\"recovery_safety\",\"critical\":true,\"max_age_sec\":0.5}]}' }" -``` - -Publish a sample work-path heartbeat: - -```bash -ros2 topic pub --once /oomwoo/health/component std_msgs/msg/String \ - "{data: '{\"component_id\":\"recovery_safety\",\"health\":\"ok\",\"stamp_sec\":1.0}' }" -``` - -Once the contract stabilizes, these should become typed OOMWOO messages or a -small adapter around standard diagnostics. - -## Design Rule - -Component heartbeats must come from the component's real work path: control -cycle completed, scan processed, recovery decision evaluated, map update -consumed, and so on. A free-running heartbeat timer is explicitly not enough, -because it can keep ticking while the component's useful work loop is wedged. +Consumers should pin `v0.1.0` or an exact commit rather than tracking `main`. +The public OOMWOO topic contract remains in +[`docs/SOFTWARE_INTERFACES.md`](../../../docs/SOFTWARE_INTERFACES.md). diff --git a/contributions/health-monitor/xbattlax/docs/recovery_safety_integration.md b/contributions/health-monitor/xbattlax/docs/recovery_safety_integration.md deleted file mode 100644 index 19a50d9..0000000 --- a/contributions/health-monitor/xbattlax/docs/recovery_safety_integration.md +++ /dev/null @@ -1,57 +0,0 @@ -# Recovery Safety Integration - -The recovery-safety node already sends bounded motion and explicit stops. The -health monitor adds the more general failure path: if recovery-safety, Nav2, -localization, or dock-cycle stops doing real work, the MCU stack heartbeat stops. - -## Recovery-Safety Heartbeat - -`recovery_safety` should publish a component heartbeat after real work events: - -- safety input handled -- recovery event accepted or ignored -- active recovery step evaluated -- active recovery step succeeded/failed/timed out -- status publication completed - -It should not publish the health heartbeat from a timer that is independent of -the recovery controller. A timer could continue ticking while the useful recovery -state machine is wedged. - -## Relationship to `/cmd_vel` - -The `/cmd_vel` hold fix remains useful for motion quality: recovery maneuvers -longer than a base controller's command timeout should not be truncated. - -The health monitor is different. It is the robot-wide fail-safe: - -- if `recovery_safety` dies, MCU heartbeat stops -- if `dock_cycle` dies during final approach, MCU heartbeat stops -- if `localization` is stale during navigation, MCU heartbeat stops -- if an advisory logger dies, the stack reports it but the robot keeps moving - -## Proposed Critical Rosters - -Mapping: - -- `recovery_safety` -- `nav2_controller` -- `localization` -- `slam_toolbox` or map/localization equivalent - -Known-map navigation: - -- `recovery_safety` -- `nav2_controller` -- `localization` -- `map_server` - -Dock final approach: - -- `recovery_safety` -- `dock_cycle` -- `dock_ir_sensor` -- `health_monitor` - -The exact roster should be owned by the task launch files once those packages -exist. diff --git a/contributions/health-monitor/xbattlax/docs/stack_watchdog_contract.md b/contributions/health-monitor/xbattlax/docs/stack_watchdog_contract.md deleted file mode 100644 index 707b3d0..0000000 --- a/contributions/health-monitor/xbattlax/docs/stack_watchdog_contract.md +++ /dev/null @@ -1,110 +0,0 @@ -# Stack Watchdog Contract - -The health monitor is the software deadman between ROS2 and the MCU. It does not -replace the MCU's hard reflexes; it decides whether the ROS2 stack is healthy -enough for the MCU to keep accepting commanded motion. - -## Component Heartbeat - -Topic: - -```text -/oomwoo/health/component std_msgs/msg/String -``` - -Temporary JSON fields: - -- `component_id`: stable component id, for example `recovery_safety`. -- `health`: `ok`, `healthy`, `warn`, `degraded`, or `error`. -- `stamp_sec`: component work timestamp. -- `sequence`: optional monotonic sequence. -- `detail`: optional short diagnostic text. - -Rule: publish this from the work path, not from a free-running timer. Examples: - -- recovery-safety evaluated a safety/recovery cycle -- Nav2 controller produced a command decision -- localization consumed a scan and produced a pose update -- dock-cycle evaluated a fresh IR/dock state - -## Roster - -Topic: - -```text -/oomwoo/health/roster std_msgs/msg/String -``` - -QoS expectation: - -- `transient_local` -- reliable -- re-published on task transitions - -Temporary JSON fields: - -- `task_id`: active task or mode, for example `cleaning_job`. -- `components`: list of expected components. -- each component has `component_id`, `critical`, and `max_age_sec`. - -Critical components block MCU heartbeat if missing, stale, or unhealthy. -Advisory components appear in status but do not stop the robot. - -## Aggregated Stack State - -Topic: - -```text -/oomwoo/health/stack std_msgs/msg/String -``` - -Temporary JSON fields: - -- `state`: `no_roster`, `arming`, `healthy`, `healthy_with_advisory_faults`, or - `fault`. -- `task_id`: current roster task. -- `emit_mcu_heartbeat`: whether the MCU-facing heartbeat should be emitted. -- `missing_critical`, `stale_critical`, `unhealthy_critical`. -- `advisory_faults`. -- `healthy_for_sec`. -- `source`. - -## MCU Heartbeat - -Topic: - -```text -/oomwoo/health/mcu_heartbeat std_msgs/msg/String -``` - -The MCU-facing bridge should forward this as the stack watchdog heartbeat only -while `emit_mcu_heartbeat=true`. - -Fail-safe rules: - -- no roster: withhold heartbeat -- arming window not satisfied: withhold heartbeat -- missing critical component: withhold heartbeat -- stale critical component: withhold heartbeat -- unhealthy critical component: withhold heartbeat -- advisory component fault: report but keep heartbeat - -The MCU should soft-stop motors on a short heartbeat miss and assert CPU reset -only on a sustained miss, matching the architecture draft. - -The first ROS2 adapter in this contribution is -`oomwoo_health_monitor.health_monitor_node`. It consumes the JSON roster and -component heartbeat topics, publishes stack state every 50 ms, and publishes the -MCU heartbeat only while the aggregate state allows it. - -## Timing Budget - -Choose component freshness so: - -```text -component work period < component max_age < MCU soft-stop timeout < CPU reset timeout -``` - -At vacuum speeds, sub-second detection keeps the travel distance small. The -exact values should be validated under sim-clock jitter and Raspberry Pi load, -because too-tight watchdogs produce false trips. diff --git a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/launch/health_monitor.launch.py b/contributions/health-monitor/xbattlax/oomwoo_health_monitor/launch/health_monitor.launch.py deleted file mode 100644 index 6a0bc96..0000000 --- a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/launch/health_monitor.launch.py +++ /dev/null @@ -1,15 +0,0 @@ -from launch import LaunchDescription -from launch_ros.actions import Node - - -def generate_launch_description(): - return LaunchDescription( - [ - Node( - package="oomwoo_health_monitor", - executable="health_monitor_node", - name="health_monitor", - output="screen", - ) - ] - ) diff --git a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/__init__.py b/contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/__init__.py deleted file mode 100644 index bc8f847..0000000 --- a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""OOMWOO stack health monitor prototype package.""" diff --git a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/core.py b/contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/core.py deleted file mode 100644 index 531b345..0000000 --- a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/core.py +++ /dev/null @@ -1,175 +0,0 @@ -from __future__ import annotations - -from dataclasses import asdict, dataclass -import json -from typing import Iterable - - -HEALTHY_LEVELS = {"ok", "healthy"} - - -@dataclass(frozen=True) -class ComponentSpec: - component_id: str - critical: bool = True - max_age_sec: float = 0.5 - - def __post_init__(self): - if not self.component_id: - raise ValueError("component_id is required") - if self.max_age_sec <= 0.0: - raise ValueError("max_age_sec must be positive") - - -@dataclass(frozen=True) -class HealthRoster: - task_id: str - components: tuple[ComponentSpec, ...] - - def __post_init__(self): - if not self.task_id: - raise ValueError("task_id is required") - ids = [component.component_id for component in self.components] - if len(ids) != len(set(ids)): - raise ValueError("component ids must be unique") - - @classmethod - def from_components( - cls, - task_id: str, - components: Iterable[ComponentSpec], - ) -> "HealthRoster": - return cls(task_id, tuple(components)) - - @property - def by_id(self) -> dict[str, ComponentSpec]: - return {component.component_id: component for component in self.components} - - -@dataclass(frozen=True) -class ComponentHeartbeat: - component_id: str - health: str - stamp_sec: float - detail: str = "" - sequence: int | None = None - - @property - def is_well(self) -> bool: - return self.health.strip().lower() in HEALTHY_LEVELS - - -@dataclass(frozen=True) -class StackHealth: - state: str - task_id: str | None - emit_mcu_heartbeat: bool - missing_critical: tuple[str, ...] = () - stale_critical: tuple[str, ...] = () - unhealthy_critical: tuple[str, ...] = () - advisory_faults: tuple[str, ...] = () - healthy_for_sec: float = 0.0 - source: str = "oomwoo_health_monitor" - - def to_json(self) -> str: - return json.dumps(asdict(self), sort_keys=True) - - -class HealthAggregator: - def __init__(self, *, arm_window_sec: float = 0.25): - if arm_window_sec < 0.0: - raise ValueError("arm_window_sec must be non-negative") - self._arm_window_sec = arm_window_sec - self._roster: HealthRoster | None = None - self._heartbeats: dict[str, ComponentHeartbeat] = {} - self._healthy_since_sec: float | None = None - - @property - def roster(self) -> HealthRoster | None: - return self._roster - - def update_roster(self, roster: HealthRoster): - old_task = self._roster.task_id if self._roster else None - old_ids = set(self._roster.by_id) if self._roster else set() - new_ids = set(roster.by_id) - self._roster = roster - self._heartbeats = { - component_id: heartbeat - for component_id, heartbeat in self._heartbeats.items() - if component_id in new_ids - } - if old_task != roster.task_id or old_ids != new_ids: - self._healthy_since_sec = None - - def update_heartbeat(self, heartbeat: ComponentHeartbeat): - self._heartbeats[heartbeat.component_id] = heartbeat - - def evaluate(self, now_sec: float) -> StackHealth: - if self._roster is None: - self._healthy_since_sec = None - return StackHealth( - state="no_roster", - task_id=None, - emit_mcu_heartbeat=False, - ) - - missing: list[str] = [] - stale: list[str] = [] - unhealthy: list[str] = [] - advisory_faults: list[str] = [] - - for component_id, spec in self._roster.by_id.items(): - heartbeat = self._heartbeats.get(component_id) - if heartbeat is None: - if spec.critical: - missing.append(component_id) - else: - advisory_faults.append(component_id) - continue - - age_sec = now_sec - heartbeat.stamp_sec - if age_sec < 0.0 or age_sec > spec.max_age_sec: - if spec.critical: - stale.append(component_id) - else: - advisory_faults.append(component_id) - continue - - if not heartbeat.is_well: - if spec.critical: - unhealthy.append(component_id) - else: - advisory_faults.append(component_id) - - if missing or stale or unhealthy: - self._healthy_since_sec = None - return StackHealth( - state="fault", - task_id=self._roster.task_id, - emit_mcu_heartbeat=False, - missing_critical=tuple(sorted(missing)), - stale_critical=tuple(sorted(stale)), - unhealthy_critical=tuple(sorted(unhealthy)), - advisory_faults=tuple(sorted(advisory_faults)), - ) - - if self._healthy_since_sec is None: - self._healthy_since_sec = now_sec - - healthy_for_sec = now_sec - self._healthy_since_sec - if healthy_for_sec < self._arm_window_sec: - return StackHealth( - state="arming", - task_id=self._roster.task_id, - emit_mcu_heartbeat=False, - advisory_faults=tuple(sorted(advisory_faults)), - healthy_for_sec=healthy_for_sec, - ) - - return StackHealth( - state="healthy" if not advisory_faults else "healthy_with_advisory_faults", - task_id=self._roster.task_id, - emit_mcu_heartbeat=True, - advisory_faults=tuple(sorted(advisory_faults)), - healthy_for_sec=healthy_for_sec, - ) diff --git a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/health_monitor_node.py b/contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/health_monitor_node.py deleted file mode 100644 index ddcb3ad..0000000 --- a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/oomwoo_health_monitor/health_monitor_node.py +++ /dev/null @@ -1,110 +0,0 @@ -from __future__ import annotations - -from dataclasses import asdict -import json -from time import monotonic - -import rclpy -from rclpy.executors import ExternalShutdownException -from rclpy.exceptions import ROSInterruptException -from rclpy.node import Node -from std_msgs.msg import String - -from oomwoo_health_monitor.core import ( - ComponentHeartbeat, - ComponentSpec, - HealthAggregator, - HealthRoster, -) - - -class HealthMonitorNode(Node): - def __init__(self): - super().__init__("health_monitor") - self._aggregator = HealthAggregator(arm_window_sec=0.25) - self._heartbeat_sequence = 0 - - self._stack_pub = self.create_publisher(String, "oomwoo/health/stack", 10) - self._mcu_pub = self.create_publisher(String, "oomwoo/health/mcu_heartbeat", 10) - - self.create_subscription(String, "oomwoo/health/roster", self._roster_cb, 10) - self.create_subscription(String, "oomwoo/health/component", self._component_cb, 10) - self.create_timer(0.05, self._timer_cb) - - def _roster_cb(self, msg: String): - try: - self._aggregator.update_roster(_parse_roster(json.loads(msg.data))) - except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: - self.get_logger().warn(f"Ignoring invalid health roster: {exc}") - - def _component_cb(self, msg: String): - try: - self._aggregator.update_heartbeat(_parse_heartbeat(json.loads(msg.data))) - except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: - self.get_logger().warn(f"Ignoring invalid component heartbeat: {exc}") - - def _timer_cb(self): - now_sec = self._now_sec() - stack_health = self._aggregator.evaluate(now_sec) - self._stack_pub.publish(String(data=stack_health.to_json())) - if not stack_health.emit_mcu_heartbeat: - return - - self._heartbeat_sequence += 1 - payload = { - "sequence": self._heartbeat_sequence, - "source": "oomwoo_health_monitor", - "stamp_sec": now_sec, - "stack": asdict(stack_health), - "task_id": stack_health.task_id, - } - self._mcu_pub.publish(String(data=json.dumps(payload, sort_keys=True))) - - def _now_sec(self) -> float: - try: - return self.get_clock().now().nanoseconds / 1_000_000_000.0 - except Exception: - return monotonic() - - -def _parse_roster(payload: dict) -> HealthRoster: - components = [] - for item in payload["components"]: - components.append( - ComponentSpec( - component_id=str(item["component_id"]), - critical=bool(item.get("critical", True)), - max_age_sec=float(item.get("max_age_sec", 0.5)), - ) - ) - return HealthRoster.from_components(str(payload["task_id"]), components) - - -def _parse_heartbeat(payload: dict) -> ComponentHeartbeat: - return ComponentHeartbeat( - component_id=str(payload["component_id"]), - health=str(payload.get("health", "ok")), - stamp_sec=float(payload["stamp_sec"]), - detail=str(payload.get("detail", "")), - sequence=payload.get("sequence"), - ) - - -def main(args=None): - rclpy.init(args=args) - node = HealthMonitorNode() - try: - rclpy.spin(node) - except (KeyboardInterrupt, ExternalShutdownException, ROSInterruptException): - pass - except Exception as exc: - if "context is not valid" not in str(exc): - raise - finally: - node.destroy_node() - try: - if rclpy.ok(): - rclpy.shutdown() - except Exception as exc: - if "rcl_shutdown already called" not in str(exc): - raise diff --git a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/package.xml b/contributions/health-monitor/xbattlax/oomwoo_health_monitor/package.xml deleted file mode 100644 index 05d6f1d..0000000 --- a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/package.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - oomwoo_health_monitor - 0.1.0 - Stack health monitor and MCU software watchdog prototype for OOMWOO. - xbattlax - Apache-2.0 - - rclpy - std_msgs - - ament_copyright - ament_flake8 - ament_pep257 - python3-pytest - - - ament_python - - diff --git a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/resource/oomwoo_health_monitor b/contributions/health-monitor/xbattlax/oomwoo_health_monitor/resource/oomwoo_health_monitor deleted file mode 100644 index 5b7b25a..0000000 --- a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/resource/oomwoo_health_monitor +++ /dev/null @@ -1 +0,0 @@ -oomwoo_health_monitor diff --git a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/setup.cfg b/contributions/health-monitor/xbattlax/oomwoo_health_monitor/setup.cfg deleted file mode 100644 index cd3013d..0000000 --- a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[develop] -script_dir=$base/lib/oomwoo_health_monitor -[install] -install_scripts=$base/lib/oomwoo_health_monitor diff --git a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/setup.py b/contributions/health-monitor/xbattlax/oomwoo_health_monitor/setup.py deleted file mode 100644 index e3ab7a7..0000000 --- a/contributions/health-monitor/xbattlax/oomwoo_health_monitor/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -from setuptools import find_packages, setup - - -package_name = "oomwoo_health_monitor" - -setup( - name=package_name, - version="0.1.0", - packages=find_packages(exclude=["test"]), - data_files=[ - ("share/ament_index/resource_index/packages", ["resource/" + package_name]), - ("share/" + package_name, ["package.xml"]), - ("share/" + package_name + "/launch", ["launch/health_monitor.launch.py"]), - ], - install_requires=["setuptools"], - zip_safe=True, - maintainer="xbattlax", - maintainer_email="xbattlax@gmail.com", - description="Stack health monitor and MCU software watchdog prototype for OOMWOO.", - license="Apache-2.0", - tests_require=["pytest"], - entry_points={ - "console_scripts": [ - "health_monitor_node = oomwoo_health_monitor.health_monitor_node:main", - ], - }, -) diff --git a/contributions/health-monitor/xbattlax/tests/test_health_monitor_node_adapter.py b/contributions/health-monitor/xbattlax/tests/test_health_monitor_node_adapter.py deleted file mode 100644 index bb13c8f..0000000 --- a/contributions/health-monitor/xbattlax/tests/test_health_monitor_node_adapter.py +++ /dev/null @@ -1,227 +0,0 @@ -import json -from pathlib import Path -import sys -import types -import unittest -from unittest import mock - - -PACKAGE_ROOT = Path(__file__).resolve().parents[1] / "oomwoo_health_monitor" -sys.path.insert(0, str(PACKAGE_ROOT)) - - -class _String: - def __init__(self, data=""): - self.data = data - - -class _Publisher: - def __init__(self): - self.messages = [] - - def publish(self, msg): - self.messages.append(msg) - - -class _Logger: - def __init__(self): - self.warnings = [] - - def warn(self, message): - self.warnings.append(message) - - -class _Time: - def __init__(self, seconds): - self.nanoseconds = int(seconds * 1_000_000_000) - - -class _Clock: - def __init__(self, node): - self._node = node - - def now(self): - return _Time(self._node.clock_sec) - - -class _Node: - def __init__(self, name): - self.name = name - self.clock_sec = 0.0 - self.publishers = {} - self.subscriptions = [] - self.timers = [] - self.logger = _Logger() - - def create_publisher(self, _msg_type, topic, _qos): - publisher = _Publisher() - self.publishers[topic] = publisher - return publisher - - def create_subscription(self, msg_type, topic, callback, qos): - self.subscriptions.append((msg_type, topic, callback, qos)) - - def create_timer(self, period_sec, callback): - self.timers.append((period_sec, callback)) - - def get_clock(self): - return _Clock(self) - - def get_logger(self): - return self.logger - - def destroy_node(self): - pass - - -class _ExternalShutdownException(Exception): - pass - - -class _ROSInterruptException(Exception): - pass - - -def _ros_stubs(): - rclpy = types.ModuleType("rclpy") - rclpy.init = lambda args=None: None - rclpy.spin = lambda node: None - rclpy.ok = lambda: False - rclpy.shutdown = lambda: None - - rclpy_executors = types.ModuleType("rclpy.executors") - rclpy_executors.ExternalShutdownException = _ExternalShutdownException - - rclpy_exceptions = types.ModuleType("rclpy.exceptions") - rclpy_exceptions.ROSInterruptException = _ROSInterruptException - - rclpy_node = types.ModuleType("rclpy.node") - rclpy_node.Node = _Node - - std_msgs = types.ModuleType("std_msgs") - std_msgs_msg = types.ModuleType("std_msgs.msg") - std_msgs_msg.String = _String - - return { - "rclpy": rclpy, - "rclpy.executors": rclpy_executors, - "rclpy.exceptions": rclpy_exceptions, - "rclpy.node": rclpy_node, - "std_msgs": std_msgs, - "std_msgs.msg": std_msgs_msg, - } - - -class HealthMonitorNodeAdapterTest(unittest.TestCase): - def setUp(self): - self._module_patch = mock.patch.dict(sys.modules, _ros_stubs()) - self._module_patch.start() - sys.modules.pop("oomwoo_health_monitor.health_monitor_node", None) - - def tearDown(self): - self._module_patch.stop() - sys.modules.pop("oomwoo_health_monitor.health_monitor_node", None) - - def test_node_publishes_stack_state_and_mcu_heartbeat_after_arming(self): - from oomwoo_health_monitor.health_monitor_node import HealthMonitorNode - - node = HealthMonitorNode() - node.clock_sec = 10.0 - node._roster_cb( - _String( - data=json.dumps( - { - "task_id": "dock_cycle", - "components": [ - { - "component_id": "recovery_safety", - "critical": True, - "max_age_sec": 1.0, - } - ], - } - ) - ) - ) - node._component_cb( - _String( - data=json.dumps( - { - "component_id": "recovery_safety", - "health": "ok", - "stamp_sec": 10.0, - } - ) - ) - ) - - node._timer_cb() - node.clock_sec = 10.3 - node._timer_cb() - - stack_messages = node.publishers["oomwoo/health/stack"].messages - mcu_messages = node.publishers["oomwoo/health/mcu_heartbeat"].messages - self.assertEqual(json.loads(stack_messages[0].data)["state"], "arming") - self.assertEqual(json.loads(stack_messages[-1].data)["state"], "healthy") - self.assertEqual(len(mcu_messages), 1) - self.assertEqual(json.loads(mcu_messages[0].data)["task_id"], "dock_cycle") - - def test_stale_critical_component_stops_mcu_heartbeat(self): - from oomwoo_health_monitor.health_monitor_node import HealthMonitorNode - - node = HealthMonitorNode() - node.clock_sec = 20.0 - node._roster_cb( - _String( - data=json.dumps( - { - "task_id": "cleaning_job", - "components": [ - { - "component_id": "nav2_controller", - "critical": True, - "max_age_sec": 1.0, - } - ], - } - ) - ) - ) - node._component_cb( - _String( - data=json.dumps( - { - "component_id": "nav2_controller", - "health": "ok", - "stamp_sec": 20.0, - } - ) - ) - ) - - node.clock_sec = 20.3 - node._timer_cb() - node.clock_sec = 20.6 - node._timer_cb() - node.clock_sec = 21.2 - node._timer_cb() - - stack_messages = node.publishers["oomwoo/health/stack"].messages - self.assertEqual(len(node.publishers["oomwoo/health/mcu_heartbeat"].messages), 1) - self.assertEqual(json.loads(stack_messages[-1].data)["state"], "fault") - self.assertEqual(json.loads(stack_messages[-1].data)["stale_critical"], ["nav2_controller"]) - - def test_invalid_roster_is_warned_and_ignored(self): - from oomwoo_health_monitor.health_monitor_node import HealthMonitorNode - - node = HealthMonitorNode() - node._roster_cb(_String(data="{not json")) - - self.assertEqual(len(node.logger.warnings), 1) - node._timer_cb() - stack = json.loads(node.publishers["oomwoo/health/stack"].messages[-1].data) - self.assertEqual(stack["state"], "no_roster") - - -if __name__ == "__main__": - unittest.main() diff --git a/contributions/health-monitor/xbattlax/tests/test_oomwoo_health_monitor.py b/contributions/health-monitor/xbattlax/tests/test_oomwoo_health_monitor.py deleted file mode 100644 index 6e7d3b7..0000000 --- a/contributions/health-monitor/xbattlax/tests/test_oomwoo_health_monitor.py +++ /dev/null @@ -1,122 +0,0 @@ -import unittest - -from pathlib import Path -import sys - - -PACKAGE_ROOT = Path(__file__).resolve().parents[1] / "oomwoo_health_monitor" -sys.path.insert(0, str(PACKAGE_ROOT)) - -from oomwoo_health_monitor.core import ( # noqa: E402 - ComponentHeartbeat, - ComponentSpec, - HealthAggregator, - HealthRoster, -) - - -class HealthAggregatorTest(unittest.TestCase): - def _roster(self): - return HealthRoster.from_components( - "dock_cycle", - ( - ComponentSpec("recovery_safety", critical=True, max_age_sec=0.5), - ComponentSpec("dock_cycle", critical=True, max_age_sec=0.5), - ComponentSpec("logger", critical=False, max_age_sec=0.5), - ), - ) - - def _make_armed(self): - aggregator = HealthAggregator(arm_window_sec=0.2) - aggregator.update_roster(self._roster()) - aggregator.update_heartbeat(ComponentHeartbeat("recovery_safety", "ok", 1.0)) - aggregator.update_heartbeat(ComponentHeartbeat("dock_cycle", "ok", 1.0)) - aggregator.update_heartbeat(ComponentHeartbeat("logger", "ok", 1.0)) - aggregator.evaluate(1.0) - return aggregator - - def test_no_roster_withholds_mcu_heartbeat(self): - result = HealthAggregator().evaluate(0.0) - - self.assertEqual(result.state, "no_roster") - self.assertFalse(result.emit_mcu_heartbeat) - - def test_missing_critical_component_faults(self): - aggregator = HealthAggregator() - aggregator.update_roster(self._roster()) - aggregator.update_heartbeat(ComponentHeartbeat("recovery_safety", "ok", 1.0)) - - result = aggregator.evaluate(1.0) - - self.assertEqual(result.state, "fault") - self.assertEqual(result.missing_critical, ("dock_cycle",)) - self.assertFalse(result.emit_mcu_heartbeat) - - def test_full_roster_arms_before_emitting_mcu_heartbeat(self): - aggregator = self._make_armed() - - arming = aggregator.evaluate(1.1) - healthy = aggregator.evaluate(1.25) - - self.assertEqual(arming.state, "arming") - self.assertFalse(arming.emit_mcu_heartbeat) - self.assertEqual(healthy.state, "healthy") - self.assertTrue(healthy.emit_mcu_heartbeat) - - def test_stale_critical_component_stops_mcu_heartbeat(self): - aggregator = self._make_armed() - aggregator.evaluate(1.25) - - result = aggregator.evaluate(1.7) - - self.assertEqual(result.state, "fault") - self.assertEqual(result.stale_critical, ("dock_cycle", "recovery_safety")) - self.assertFalse(result.emit_mcu_heartbeat) - - def test_unhealthy_critical_component_faults(self): - aggregator = HealthAggregator(arm_window_sec=0.0) - aggregator.update_roster(self._roster()) - aggregator.update_heartbeat(ComponentHeartbeat("recovery_safety", "error", 2.0)) - aggregator.update_heartbeat(ComponentHeartbeat("dock_cycle", "ok", 2.0)) - - result = aggregator.evaluate(2.0) - - self.assertEqual(result.state, "fault") - self.assertEqual(result.unhealthy_critical, ("recovery_safety",)) - self.assertFalse(result.emit_mcu_heartbeat) - - def test_advisory_fault_does_not_block_mcu_heartbeat(self): - aggregator = HealthAggregator(arm_window_sec=0.0) - aggregator.update_roster(self._roster()) - aggregator.update_heartbeat(ComponentHeartbeat("recovery_safety", "ok", 3.0)) - aggregator.update_heartbeat(ComponentHeartbeat("dock_cycle", "ok", 3.0)) - - result = aggregator.evaluate(3.0) - - self.assertEqual(result.state, "healthy_with_advisory_faults") - self.assertEqual(result.advisory_faults, ("logger",)) - self.assertTrue(result.emit_mcu_heartbeat) - - def test_roster_change_disarms_until_new_critical_is_healthy(self): - aggregator = self._make_armed() - self.assertTrue(aggregator.evaluate(1.25).emit_mcu_heartbeat) - aggregator.update_roster( - HealthRoster.from_components( - "dock_cycle", - ( - ComponentSpec("recovery_safety", critical=True, max_age_sec=0.5), - ComponentSpec("dock_cycle", critical=True, max_age_sec=0.5), - ComponentSpec("localization", critical=True, max_age_sec=0.5), - ), - ) - ) - - result = aggregator.evaluate(1.3) - - self.assertEqual(result.state, "fault") - self.assertEqual(result.missing_critical, ("localization",)) - self.assertFalse(result.emit_mcu_heartbeat) - - -if __name__ == "__main__": - unittest.main() diff --git a/contributions/health-monitor/xbattlax/tools/sim_health_monitor.py b/contributions/health-monitor/xbattlax/tools/sim_health_monitor.py deleted file mode 100644 index 782f8dc..0000000 --- a/contributions/health-monitor/xbattlax/tools/sim_health_monitor.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -from pathlib import Path -import sys - - -if __package__ in (None, ""): - sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "oomwoo_health_monitor")) - -from oomwoo_health_monitor.core import ( # noqa: E402 - ComponentHeartbeat, - ComponentSpec, - HealthAggregator, - HealthRoster, -) - - -def main() -> int: - aggregator = HealthAggregator(arm_window_sec=0.25) - roster = HealthRoster.from_components( - "cleaning_job", - ( - ComponentSpec("recovery_safety", critical=True, max_age_sec=0.30), - ComponentSpec("nav2_controller", critical=True, max_age_sec=0.30), - ComponentSpec("localization", critical=True, max_age_sec=0.50), - ComponentSpec("telemetry_logger", critical=False, max_age_sec=1.00), - ), - ) - - for now_sec, event in ( - (0.00, "boot_no_roster"), - (0.05, "roster_published"), - (0.10, "recovery_healthy"), - (0.15, "nav2_healthy"), - (0.20, "localization_healthy"), - (0.30, "advisory_healthy"), - (0.50, "armed"), - (0.90, "nav2_stale"), - (1.00, "nav2_recovers"), - (1.36, "advisory_stale_only"), - ): - if event == "roster_published": - aggregator.update_roster(roster) - elif event == "recovery_healthy": - aggregator.update_heartbeat(ComponentHeartbeat("recovery_safety", "ok", now_sec)) - elif event == "nav2_healthy": - aggregator.update_heartbeat(ComponentHeartbeat("nav2_controller", "ok", now_sec)) - elif event == "localization_healthy": - aggregator.update_heartbeat(ComponentHeartbeat("localization", "ok", now_sec)) - elif event == "advisory_healthy": - aggregator.update_heartbeat(ComponentHeartbeat("telemetry_logger", "ok", now_sec)) - elif event == "armed": - for component_id in ("recovery_safety", "nav2_controller", "localization"): - aggregator.update_heartbeat(ComponentHeartbeat(component_id, "ok", now_sec)) - elif event == "nav2_recovers": - aggregator.update_heartbeat(ComponentHeartbeat("nav2_controller", "ok", now_sec)) - aggregator.update_heartbeat(ComponentHeartbeat("recovery_safety", "ok", now_sec)) - aggregator.update_heartbeat(ComponentHeartbeat("localization", "ok", now_sec)) - elif event == "advisory_stale_only": - for component_id in ("recovery_safety", "nav2_controller", "localization"): - aggregator.update_heartbeat(ComponentHeartbeat(component_id, "ok", now_sec)) - - result = aggregator.evaluate(now_sec) - print(result.to_json()) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/docs/SOFTWARE_INTERFACES.md b/docs/SOFTWARE_INTERFACES.md index 8e6c1f3..1107472 100644 --- a/docs/SOFTWARE_INTERFACES.md +++ b/docs/SOFTWARE_INTERFACES.md @@ -175,6 +175,18 @@ OOMWOO message package exists: | `/oomwoo/health/stack` | Health monitor | Diagnostics / app layer | Aggregated state: no roster, arming, healthy, healthy with advisory faults, or fault. | | `/oomwoo/health/mcu_heartbeat` | Health monitor / bridge | MCU bridge | Single stack-health heartbeat forwarded to the MCU only while all expected critical components are fresh and well. | +QoS and clock rules: + +- `/oomwoo/health/roster` is reliable, transient-local, depth 1 so a monitor + started after the active roster publisher still receives the current task. +- `/oomwoo/health/stack` is reliable, transient-local, depth 1 for late-joining + diagnostics consumers. +- component and MCU heartbeats are reliable and volatile. In particular, an old + healthy MCU heartbeat must not be replayed to a late subscriber. +- component `stamp_sec` values use the producer node's ROS clock, which must be + the same clock as the monitor. With `use_sim_time=true`, both follow `/clock`; + wall-clock and simulation timestamps must not be mixed. + Fail-safe behavior: - no roster means no MCU heartbeat @@ -185,6 +197,9 @@ Fail-safe behavior: - component heartbeats must be emitted from useful work paths, not from independent timers that can keep running while the component is wedged +Reference implementation: +[xbattlax/health-monitor](https://github.com/xbattlax/health-monitor). + ## Validation Checklist A module submission that depends on the MVP simulation should document how to