diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index f12b077d..9f72748d 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -477,6 +477,19 @@ storage: Packets are stored in a local SQLite database. Old packets are pruned automatically based on `max_packets_retained`. +### Stray frames (Unknown RF) + +When a LoRa frame fails **both** Meshtastic and MeshCore decode, Meshpoint can log RF metadata only (no relay, no re-encode). View results in the dashboard **Unknown RF** tab or export via `GET /api/stray_frames?format=csv`. + +```yaml +stray_frames: + enabled: true + max_retained: 10000 # hard cap on SQLite rows + retention_hours: 168 # drop rows older than this (7 days) +``` + +Pruning runs in the background during the normal storage cleanup loop and does not block the receive path. + --- ## Dashboard @@ -490,6 +503,10 @@ dashboard: Access at `http://:8080`. Bind to `127.0.0.1` to restrict to local access only. +### Unknown RF tab + +Open **Unknown RF** in the sidebar to browse undecodable frames logged by `stray_frames` (see Storage). Filter by time window and minimum RSSI; use **Export CSV** to download the current filter via `/api/stray_frames?format=csv`. Disable logging with `stray_frames.enabled: false` in `local.yaml`. + --- ## Device Identity @@ -744,6 +761,11 @@ storage: # local SQLite packet store max_packets_retained: 100000 cleanup_interval_seconds: 3600 +stray_frames: # undecodable RF metadata (Unknown RF tab) + enabled: true + max_retained: 10000 + retention_hours: 168 + dashboard: # local web UI host: "0.0.0.0" port: 8080 diff --git a/frontend/css/status_strip.css b/frontend/css/status_strip.css new file mode 100644 index 00000000..aa3dc0c4 --- /dev/null +++ b/frontend/css/status_strip.css @@ -0,0 +1,105 @@ +/* Operator status footer — console-style telemetry in panel chrome. */ + +.status-strip { + display: flex; + align-items: baseline; + flex-wrap: wrap; + gap: 0.35rem; + margin-top: 1.25rem; + padding: 0.5rem 0.75rem; + border-radius: var(--radius-sm); + border: 1px solid rgba(255, 255, 255, 0.04); + background: rgba(6, 182, 212, 0.03); + font-family: var(--font-mono); + font-size: 0.68rem; + letter-spacing: 0.04em; + color: var(--text-muted); +} + +.status-strip__label { + color: var(--text-secondary); + font-weight: 600; + letter-spacing: 0.1em; + font-size: 0.62rem; +} + +.status-strip__sep { + opacity: 0.35; +} + +.status-strip__items { + color: var(--accent-cyan); + text-shadow: 0 0 6px rgba(6, 182, 212, 0.12); +} + +.status-strip__updated { + color: var(--text-muted); + font-style: italic; +} + +.status-strip--quiet .status-strip__items { + color: var(--text-muted); + text-shadow: none; + font-style: italic; +} + +.map-topology-hint { + position: absolute; + left: 50%; + bottom: 2.75rem; + transform: translateX(-50%); + z-index: 800; + max-width: min(92%, 28rem); + padding: 0.45rem 0.75rem; + border-radius: var(--radius-sm); + border: 1px solid rgba(245, 158, 11, 0.35); + background: rgba(15, 23, 42, 0.92); + color: var(--accent-amber); + font-family: var(--font-mono); + font-size: 0.68rem; + letter-spacing: 0.03em; + text-align: center; + pointer-events: none; + box-shadow: var(--glow-amber); +} + +.mqtt-runtime { + margin: 0 0 1rem; + padding: 0.55rem 0.75rem; + border-radius: var(--radius-sm); + border: 1px solid rgba(255, 255, 255, 0.05); + background: rgba(0, 229, 160, 0.04); + font-family: var(--font-mono); + font-size: 0.68rem; + letter-spacing: 0.03em; + color: var(--text-muted); +} + +.mqtt-runtime--offline { + background: rgba(239, 68, 68, 0.06); + border-color: rgba(239, 68, 68, 0.25); +} + +.mqtt-runtime--disabled { + background: rgba(100, 116, 139, 0.08); +} + +.mqtt-runtime__label { + color: var(--text-secondary); + font-weight: 600; + letter-spacing: 0.1em; + font-size: 0.62rem; + margin-right: 0.35rem; +} + +.mqtt-runtime__state { + color: var(--accent-green); +} + +.mqtt-runtime--offline .mqtt-runtime__state { + color: var(--accent-red); +} + +.mqtt-runtime--disabled .mqtt-runtime__state { + color: var(--text-muted); +} diff --git a/frontend/css/unknown_rf.css b/frontend/css/unknown_rf.css new file mode 100644 index 00000000..6b24f08f --- /dev/null +++ b/frontend/css/unknown_rf.css @@ -0,0 +1,113 @@ +.section[data-section="unknown-rf"] { + overflow-y: auto; +} + +.unknown-rf-panel { + padding: 1.25rem; + max-width: 1100px; + margin: 0 auto; +} + +.unknown-rf-panel__header { + display: flex; + flex-wrap: wrap; + align-items: flex-start; + justify-content: space-between; + gap: 1rem; + margin-bottom: 1rem; +} + +.unknown-rf-panel__title { + margin: 0; + font-size: 1.1rem; + color: var(--text-primary, #f1f5f9); +} + +.unknown-rf-panel__desc { + margin: 0.35rem 0 0; + font-size: 0.78rem; + color: var(--text-secondary, #94a3b8); + max-width: 40rem; +} + +.unknown-rf-controls { + display: flex; + flex-wrap: wrap; + align-items: flex-end; + gap: 0.75rem; + margin-bottom: 1rem; +} + +.unknown-rf-field label { + display: block; + font-size: 0.68rem; + color: var(--text-muted, #64748b); + margin-bottom: 0.2rem; +} + +.unknown-rf-field input, +.unknown-rf-field select { + background: rgba(15, 23, 42, 0.8); + border: 1px solid rgba(51, 65, 85, 0.8); + border-radius: 6px; + color: var(--text-primary, #e2e8f0); + padding: 0.35rem 0.5rem; + font-size: 0.78rem; +} + +.unknown-rf-btn { + background: rgba(6, 182, 212, 0.15); + border: 1px solid rgba(6, 182, 212, 0.45); + color: var(--accent-cyan); + border-radius: 6px; + padding: 0.4rem 0.75rem; + font-size: 0.75rem; + cursor: pointer; +} + +.unknown-rf-btn:hover { + background: rgba(6, 182, 212, 0.25); +} + +.unknown-rf-meta { + font-size: 0.72rem; + color: var(--text-secondary, #94a3b8); + margin-bottom: 0.75rem; +} + +.unknown-rf-table-wrap { + overflow-x: auto; + border: 1px solid rgba(51, 65, 85, 0.6); + border-radius: 8px; +} + +.unknown-rf-table { + width: 100%; + border-collapse: collapse; + font-size: 0.75rem; +} + +.unknown-rf-table th, +.unknown-rf-table td { + padding: 0.45rem 0.6rem; + text-align: left; + border-bottom: 1px solid rgba(51, 65, 85, 0.35); +} + +.unknown-rf-table th { + color: var(--text-muted, #64748b); + font-weight: 600; + background: rgba(15, 23, 42, 0.6); +} + +.unknown-rf-table td { + color: var(--text-primary, #e2e8f0); + font-variant-numeric: tabular-nums; +} + +.unknown-rf-empty { + padding: 2rem; + text-align: center; + color: var(--text-secondary, #94a3b8); + font-size: 0.8rem; +} diff --git a/frontend/index.html b/frontend/index.html index 239398cc..de1cd63b 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -21,6 +21,7 @@ + @@ -33,6 +34,7 @@ + @@ -102,6 +104,24 @@ Stats +
  • + + + + + Unknown RF + +
  • @@ -391,6 +411,12 @@ + + @@ -693,6 +719,7 @@

    Service actions

    + @@ -747,6 +774,7 @@

    Service actions

    + diff --git a/frontend/js/app.js b/frontend/js/app.js index b6b237ed..6b626365 100644 --- a/frontend/js/app.js +++ b/frontend/js/app.js @@ -41,7 +41,7 @@ document.addEventListener('DOMContentLoaded', async () => { const router = new Router({ defaultRoute: 'dashboard', allowedRoutes: [ - 'dashboard', 'stats', 'messages', 'radio', 'terminal', + 'dashboard', 'stats', 'unknown-rf', 'messages', 'radio', 'terminal', 'configuration/identity', 'configuration/radio', 'configuration/channels', 'configuration/transmit', 'configuration/mqtt', diff --git a/frontend/js/status_strip.js b/frontend/js/status_strip.js new file mode 100644 index 00000000..afba91ea --- /dev/null +++ b/frontend/js/status_strip.js @@ -0,0 +1,65 @@ +/** + * Console-style operator status footer for analytics tabs. + * + * Renders a slim monospace strip at the bottom of a panel: + * TRAFFIC · concentrator · 8,247 pkts · updated 14s ago + */ +class StatusStrip { + constructor(hostEl, label) { + this._host = hostEl; + this._label = (label || 'STATUS').toUpperCase(); + this._root = null; + this._itemsEl = null; + this._updatedEl = null; + } + + mount() { + if (!this._host || this._root) return; + const root = document.createElement('footer'); + root.className = 'status-strip'; + root.setAttribute('role', 'status'); + root.setAttribute('aria-live', 'polite'); + root.innerHTML = ` + ${this._label} + + + + `; + this._host.appendChild(root); + this._root = root; + this._itemsEl = root.querySelector('.status-strip__items'); + this._updatedEl = root.querySelector('.status-strip__updated'); + } + + /** + * @param {string[]} items - dot-separated status fragments + * @param {Date|number|null} updatedAt - when underlying data was fetched + */ + update(items, updatedAt) { + if (!this._root) return; + const parts = (items || []).filter((item) => item != null && String(item).trim()); + this._itemsEl.textContent = parts.length ? parts.join(' · ') : 'waiting for data'; + this._root.classList.toggle('status-strip--quiet', parts.length === 0); + + if (this._updatedEl) { + if (updatedAt) { + this._updatedEl.textContent = ` · updated ${_formatRelative(updatedAt)}`; + } else { + this._updatedEl.textContent = ''; + } + } + } +} + +function _formatRelative(timestamp) { + const ms = timestamp instanceof Date ? timestamp.getTime() : Number(timestamp); + if (!ms) return 'just now'; + const seconds = Math.max(0, Math.floor((Date.now() - ms) / 1000)); + if (seconds < 30) return 'just now'; + if (seconds < 60) return `${seconds}s ago`; + if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`; + if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`; + return `${Math.floor(seconds / 86400)}d ago`; +} + +window.StatusStrip = StatusStrip; diff --git a/frontend/js/unknown_rf_tab.js b/frontend/js/unknown_rf_tab.js new file mode 100644 index 00000000..ad641736 --- /dev/null +++ b/frontend/js/unknown_rf_tab.js @@ -0,0 +1,195 @@ +/** + * Unknown RF tab — undecodable frames from GET /api/stray_frames. + */ +class UnknownRfTab { + constructor(containerId) { + this._container = document.getElementById(containerId); + this._rendered = false; + this._hours = 24; + this._minRssi = ''; + this._refreshInterval = null; + this._statusStrip = null; + this._fetchedAt = null; + } + + async refresh() { + if (!this._container) return; + + try { + if (!this._rendered) { + this._buildLayout(); + this._rendered = true; + } + const params = new URLSearchParams({ + limit: '500', + hours: String(this._hours), + }); + if (this._minRssi !== '' && !Number.isNaN(Number(this._minRssi))) { + params.set('min_rssi', this._minRssi); + } + const res = await fetch(`/api/stray_frames?${params}`); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const data = await res.json(); + this._fetchedAt = Date.now(); + this._renderTable(data); + } catch (e) { + console.error('Unknown RF refresh failed:', e); + } + + if (!this._refreshInterval) { + this._refreshInterval = setInterval(() => { + const section = document.querySelector('[data-section="unknown-rf"]'); + if (section && section.classList.contains('section--active')) { + this.refresh(); + } else { + clearInterval(this._refreshInterval); + this._refreshInterval = null; + } + }, 15_000); + } + } + + _buildLayout() { + this._container.innerHTML = ` +
    +
    +
    +

    Unknown RF

    +

    + Frames that failed both Meshtastic and MeshCore decode. RF metadata only: + no payload stored, no relay, no re-encode. +

    +
    +
    +
    +
    + + +
    +
    + + +
    + +     Export CSV +
    +

    +
    + + + + + + + + + + + + + + + +
    TimeSizeChFreqSFBWRSSISNRSource
    + +
    +
    +
    + `; + + const stripHost = document.getElementById('urf-status-strip-host'); + if (stripHost && window.StatusStrip) { + this._statusStrip = new window.StatusStrip(stripHost, 'UNKNOWN RF'); + this._statusStrip.mount(); + } + + document.getElementById('urf-hours').addEventListener('change', (e) => { + this._hours = Number(e.target.value); + }); + document.getElementById('urf-apply').addEventListener('click', () => { + this._minRssi = document.getElementById('urf-min-rssi').value.trim(); + this.refresh(); + }); + document.getElementById('urf-csv').addEventListener('click', (e) => { + e.preventDefault(); + const params = new URLSearchParams({ + format: 'csv', + limit: '2000', + hours: String(this._hours), + }); + if (this._minRssi !== '' && !Number.isNaN(Number(this._minRssi))) { + params.set('min_rssi', this._minRssi); + } + window.location.assign(`/api/stray_frames?${params}`); + }); + } + + _renderTable(data) { + const tbody = document.getElementById('urf-tbody'); + const meta = document.getElementById('urf-meta'); + const empty = document.getElementById('urf-empty'); + if (!tbody) return; + + const frames = data.frames || []; + const totalStored = data.total_stored ?? frames.length; + const windowH = data.window_hours ?? this._hours; + meta.textContent = + `Showing ${frames.length} of ${totalStored} stored (last ${windowH}h)`; + + const sfSet = new Set(frames.map((f) => f.spreading_factor).filter((v) => v != null)); + this._statusStrip?.update( + [ + `${frames.length} shown`, + `${totalStored} stored`, + `${windowH}h window`, + `${sfSet.size} SF buckets`, + ], + this._fetchedAt, + ); + + if (!frames.length) { + tbody.innerHTML = ''; + if (empty) empty.hidden = false; + return; + } + if (empty) empty.hidden = true; + + tbody.innerHTML = frames.map((f) => { + const ts = this._formatTime(f.timestamp); + return ` + ${ts} + ${f.frame_size ?? '--'} B + ${f.channel_hash != null ? f.channel_hash : '--'} + ${f.frequency_mhz != null ? `${Number(f.frequency_mhz).toFixed(3)} MHz` : '--'} + ${f.spreading_factor ?? '--'} + ${f.bandwidth_khz != null ? `${f.bandwidth_khz} kHz` : '--'} + ${f.rssi != null ? `${Number(f.rssi).toFixed(0)}` : '--'} + ${f.snr != null ? `${Number(f.snr).toFixed(1)}` : '--'} + ${this._esc(f.capture_source || '--')} + `; + }).join(''); + } + + _formatTime(ts) { + if (!ts) return '--'; + try { + const d = new Date(ts); + return d.toLocaleString(); + } catch (_e) { + return ts; + } + } + + _esc(str) { + const el = document.createElement('span'); + el.textContent = str; + return el.innerHTML; + } +} + +window.unknownRfTab = new UnknownRfTab('unknown-rf-panel'); diff --git a/frontend/sidebar/sidebar_controller.js b/frontend/sidebar/sidebar_controller.js index 800cc4ff..ad8ffa01 100644 --- a/frontend/sidebar/sidebar_controller.js +++ b/frontend/sidebar/sidebar_controller.js @@ -183,6 +183,9 @@ class SidebarController { if (root === 'stats' && window.statsTab) { window.statsTab.refresh(); } + if (root === 'unknown-rf' && window.unknownRfTab) { + window.unknownRfTab.refresh(); + } if (route === 'terminal' && window.terminalController) { window.terminalController.onActivated(); } diff --git a/src/api/routes/stray_frames_routes.py b/src/api/routes/stray_frames_routes.py new file mode 100644 index 00000000..0331cdb6 --- /dev/null +++ b/src/api/routes/stray_frames_routes.py @@ -0,0 +1,67 @@ +"""Read-only API for undecodable RF frames logged by the pipeline.""" +from __future__ import annotations + +import csv +import io +from datetime import datetime, timezone + +from fastapi import APIRouter, Query +from fastapi.responses import PlainTextResponse + +from src.storage.stray_frame_repository import StrayFrameRepository + +router = APIRouter(prefix="/api/stray_frames", tags=["stray_frames"]) + +_repo: StrayFrameRepository | None = None + + +def init_routes(repo: StrayFrameRepository) -> None: + global _repo + _repo = repo + + +@router.get("") +async def list_stray_frames( + limit: int = Query(200, ge=1, le=2000), + hours: float | None = Query(24, ge=1, le=720), + min_rssi: float | None = Query(None, ge=-150, le=0), + format: str | None = Query(None, alias="format"), +): + frames = await _repo.list_recent( + limit=limit, hours=hours, min_rssi=min_rssi, + ) + if format == "csv": + return _as_csv(frames) + total = await _repo.get_count() + return { + "frames": [f.to_dict() for f in frames], + "count": len(frames), + "total_stored": total, + "window_hours": hours, + } + + +def _as_csv(frames) -> PlainTextResponse: + buffer = io.StringIO() + writer = csv.writer(buffer) + writer.writerow([ + "id", "timestamp", "frame_size", "channel_hash", + "frequency_mhz", "spreading_factor", "bandwidth_khz", + "rssi", "snr", "capture_source", + ]) + for frame in frames: + d = frame.to_dict() + writer.writerow([ + d["id"], d["timestamp"], d["frame_size"], d["channel_hash"], + d["frequency_mhz"], d["spreading_factor"], d["bandwidth_khz"], + d["rssi"], d["snr"], d["capture_source"], + ]) + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + headers = { + "Content-Disposition": f'attachment; filename="stray_frames_{stamp}.csv"', + } + return PlainTextResponse( + buffer.getvalue(), + media_type="text/csv", + headers=headers, + ) diff --git a/src/api/server.py b/src/api/server.py index 3bd4fb06..07833d3a 100644 --- a/src/api/server.py +++ b/src/api/server.py @@ -51,6 +51,7 @@ packets, public_radar_routes, stats_routes, + stray_frames_routes, system_config_routes, system_metrics, telemetry, @@ -265,6 +266,7 @@ async def lifespan(app: FastAPI): protected = [Depends(require_auth)] app.include_router(nodes.router, dependencies=protected) app.include_router(packets.router, dependencies=protected) + app.include_router(stray_frames_routes.router, dependencies=protected) app.include_router(analytics.router, dependencies=protected) app.include_router(device.router, dependencies=protected) app.include_router(system_metrics.router, dependencies=protected) @@ -1224,6 +1226,7 @@ def _init_routes( telemetry_repo=coord.telemetry_repo, ) packets.init_routes(coord.packet_repo) + stray_frames_routes.init_routes(coord.stray_repo) analytics.init_routes(signal_analyzer, traffic_monitor, coord.packet_repo) device.init_routes(identity, ws_manager, coord.relay_manager) telemetry.init_routes(coord.telemetry_repo) diff --git a/src/config.py b/src/config.py index 4622e2b4..f9216d41 100644 --- a/src/config.py +++ b/src/config.py @@ -114,6 +114,15 @@ class StorageConfig: cleanup_interval_seconds: int = 3600 +@dataclass +class StrayFramesConfig: + """Undecodable RF frame logger (PR 08).""" + + enabled: bool = True + max_retained: int = 10_000 + retention_hours: float = 168.0 + + @dataclass class DashboardConfig: host: str = "0.0.0.0" # nosec B104 -- intentional for local device dashboard @@ -319,6 +328,7 @@ class AppConfig: transmit: TransmitConfig = field(default_factory=TransmitConfig) web_auth: WebAuthConfig = field(default_factory=WebAuthConfig) location: LocationConfig = field(default_factory=LocationConfig) + stray_frames: StrayFramesConfig = field(default_factory=StrayFramesConfig) def _resolve_radio_frequency(radio: "RadioConfig") -> None: @@ -401,6 +411,7 @@ def _apply_yaml(cfg: AppConfig, path: Path) -> None: "transmit": cfg.transmit, "web_auth": cfg.web_auth, "location": cfg.location, + "stray_frames": cfg.stray_frames, } unknown_keys: list[str] = [] diff --git a/src/coordinator.py b/src/coordinator.py index 665e9ca7..bd1a67e7 100644 --- a/src/coordinator.py +++ b/src/coordinator.py @@ -20,6 +20,7 @@ from src.storage.database import DatabaseManager from src.storage.node_repository import NodeRepository from src.storage.packet_repository import PacketRepository +from src.storage.stray_frame_repository import StrayFrameRepository from src.storage.telemetry_repository import TelemetryRepository logger = logging.getLogger(__name__) @@ -60,6 +61,7 @@ def __init__(self, config: AppConfig): self._node_repo: Optional[NodeRepository] = None self._packet_repo: Optional[PacketRepository] = None self._telemetry_repo: Optional[TelemetryRepository] = None + self._stray_repo: Optional[StrayFrameRepository] = None self._last_node_update: dict[str, Any] = {} self._on_packet_callbacks: list[Callable[[Packet], None]] = [] @@ -90,6 +92,12 @@ def packet_repo(self) -> PacketRepository: raise RuntimeError("Pipeline not started") return self._packet_repo + @property + def stray_repo(self) -> StrayFrameRepository: + if self._stray_repo is None: + raise RuntimeError("Pipeline not started") + return self._stray_repo + @property def telemetry_repo(self) -> TelemetryRepository: if self._telemetry_repo is None: @@ -133,6 +141,7 @@ async def start(self) -> None: self._node_repo = NodeRepository(self._db) self._packet_repo = PacketRepository(self._db) self._telemetry_repo = TelemetryRepository(self._db) + self._stray_repo = StrayFrameRepository(self._db) self._setup_channel_keys() self._setup_relay_transmitter() @@ -194,6 +203,18 @@ async def _cleanup_loop(self) -> None: f"pruned {removed} old packets " f"{DIM}(max {max_retained}){RESET}" ) + if self._stray_repo and self._config.stray_frames.enabled: + stray_cfg = self._config.stray_frames + stray_removed = await self._stray_repo.cleanup( + stray_cfg.max_retained, + stray_cfg.retention_hours, + ) + if stray_removed: + logger.info( + f" {CYAN}--{RESET} {DIM}CLEANUP{RESET} " + f"pruned {stray_removed} stray frames " + f"{DIM}(max {stray_cfg.max_retained}){RESET}" + ) except asyncio.CancelledError: pass except Exception: @@ -266,6 +287,8 @@ async def _process_capture(self, raw: RawCapture) -> None: raw.payload, signal=raw.signal, protocol_hint=raw.protocol_hint ) if packet is None: + if raw.capture_source != "meshcore_usb": + await self._log_stray_frame(raw) return packet.capture_source = raw.capture_source @@ -280,6 +303,14 @@ def _adapt_meshcore_usb(raw: RawCapture) -> Optional[Packet]: from src.decode.meshcore_event_adapter import adapt_event return adapt_event(raw.payload, signal=raw.signal) + async def _log_stray_frame(self, raw: RawCapture) -> None: + if not self._config.stray_frames.enabled or self._stray_repo is None: + return + try: + await self._stray_repo.insert_from_capture(raw) + except Exception: + logger.exception("Failed to log stray RF frame") + async def _store_packet(self, packet: Packet) -> None: try: await self._packet_repo.insert(packet) diff --git a/src/storage/database.py b/src/storage/database.py index eb70368f..97fdb394 100644 --- a/src/storage/database.py +++ b/src/storage/database.py @@ -89,6 +89,21 @@ CREATE INDEX IF NOT EXISTS idx_messages_node ON messages(node_id); CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp); CREATE INDEX IF NOT EXISTS idx_messages_direction ON messages(direction); + +CREATE TABLE IF NOT EXISTS stray_frames ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + frame_size INTEGER NOT NULL, + channel_hash INTEGER, + frequency_mhz REAL, + spreading_factor INTEGER, + bandwidth_khz REAL, + rssi REAL, + snr REAL, + capture_source TEXT, + timestamp TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_stray_frames_timestamp ON stray_frames(timestamp); """ diff --git a/src/storage/stray_frame_repository.py b/src/storage/stray_frame_repository.py new file mode 100644 index 00000000..1ce98509 --- /dev/null +++ b/src/storage/stray_frame_repository.py @@ -0,0 +1,166 @@ +"""Persistence for undecodable RF frames (stray / unknown protocol).""" +from __future__ import annotations + +import logging +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone + +from src.models.packet import RawCapture +from src.storage.database import DatabaseManager + +logger = logging.getLogger(__name__) + +MESHTASTIC_HEADER_SIZE = 16 + + +@dataclass +class StrayFrame: + id: int + frame_size: int + channel_hash: int | None + frequency_mhz: float | None + spreading_factor: int | None + bandwidth_khz: float | None + rssi: float | None + snr: float | None + capture_source: str | None + timestamp: datetime + + def to_dict(self) -> dict: + return { + "id": self.id, + "frame_size": self.frame_size, + "channel_hash": self.channel_hash, + "frequency_mhz": self.frequency_mhz, + "spreading_factor": self.spreading_factor, + "bandwidth_khz": self.bandwidth_khz, + "rssi": self.rssi, + "snr": self.snr, + "capture_source": self.capture_source, + "timestamp": self.timestamp.isoformat(), + } + + +class StrayFrameRepository: + """CRUD for frames that fail both Meshtastic and MeshCore decode.""" + + def __init__(self, db: DatabaseManager): + self._db = db + + async def insert_from_capture(self, raw: RawCapture) -> None: + signal = raw.signal + channel_hash = _peek_channel_hash(raw.payload) + ts = raw.timestamp or datetime.now(timezone.utc) + await self._db.execute( + """ + INSERT INTO stray_frames ( + frame_size, channel_hash, frequency_mhz, spreading_factor, + bandwidth_khz, rssi, snr, capture_source, timestamp + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + len(raw.payload), + channel_hash, + signal.frequency_mhz if signal else None, + signal.spreading_factor if signal else None, + signal.bandwidth_khz if signal else None, + signal.rssi if signal else None, + signal.snr if signal else None, + raw.capture_source, + ts.isoformat(), + ), + ) + await self._db.commit() + + async def list_recent( + self, + *, + limit: int = 200, + hours: float | None = 24, + min_rssi: float | None = None, + ) -> list[StrayFrame]: + clauses = [] + params: list = [] + if hours is not None and hours > 0: + cutoff = ( + datetime.now(timezone.utc) - timedelta(hours=hours) + ).isoformat() + clauses.append("timestamp >= ?") + params.append(cutoff) + if min_rssi is not None: + clauses.append("rssi >= ?") + params.append(min_rssi) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" + params.append(limit) + rows = await self._db.fetch_all( + f""" + SELECT * FROM stray_frames + {where} + ORDER BY timestamp DESC + LIMIT ? + """, + tuple(params), + ) + return [self._row_to_frame(row) for row in rows] + + async def get_count(self) -> int: + row = await self._db.fetch_one("SELECT COUNT(*) AS cnt FROM stray_frames") + return int(row["cnt"]) if row else 0 + + async def cleanup(self, max_retained: int, retention_hours: float) -> int: + """Drop rows older than retention window, then trim to max_retained.""" + removed = 0 + if retention_hours > 0: + cutoff = ( + datetime.now(timezone.utc) - timedelta(hours=retention_hours) + ).isoformat() + result = await self._db.execute( + "DELETE FROM stray_frames WHERE timestamp < ?", + (cutoff,), + ) + removed += int(getattr(result, "rowcount", 0) or 0) + + total = await self.get_count() + if total > max_retained: + excess = total - max_retained + await self._db.execute( + """ + DELETE FROM stray_frames WHERE id IN ( + SELECT id FROM stray_frames + ORDER BY timestamp ASC + LIMIT ? + ) + """, + (excess,), + ) + removed += excess + + if removed: + await self._db.commit() + logger.info("Pruned %d stray frame rows", removed) + return removed + + @staticmethod + def _row_to_frame(row: dict) -> StrayFrame: + return StrayFrame( + id=row["id"], + frame_size=row["frame_size"], + channel_hash=row.get("channel_hash"), + frequency_mhz=row.get("frequency_mhz"), + spreading_factor=row.get("spreading_factor"), + bandwidth_khz=row.get("bandwidth_khz"), + rssi=row.get("rssi"), + snr=row.get("snr"), + capture_source=row.get("capture_source"), + timestamp=datetime.fromisoformat(row["timestamp"]), + ) + + +def _peek_channel_hash(payload: bytes) -> int | None: + """Best-effort channel hash byte when a Meshtastic-shaped header exists.""" + if len(payload) < MESHTASTIC_HEADER_SIZE: + return None + try: + return int(payload[13]) + except (IndexError, TypeError, ValueError): + return None diff --git a/tests/test_stray_frame_repository.py b/tests/test_stray_frame_repository.py new file mode 100644 index 00000000..04f59c8e --- /dev/null +++ b/tests/test_stray_frame_repository.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import asyncio +import unittest +from datetime import datetime, timedelta, timezone + +from src.models.packet import RawCapture +from src.models.signal import SignalMetrics +from src.storage.database import DatabaseManager +from src.storage.stray_frame_repository import StrayFrameRepository, _peek_channel_hash + + +def _run(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +def _capture( + *, + payload: bytes = b"\x00" * 20, + rssi: float = -85.0, + timestamp: datetime | None = None, +) -> RawCapture: + ts = timestamp or datetime.now(timezone.utc) + return RawCapture( + payload=payload, + signal=SignalMetrics( + rssi=rssi, + snr=8.5, + frequency_mhz=906.875, + spreading_factor=11, + bandwidth_khz=250.0, + timestamp=ts, + ), + capture_source="sx1302", + timestamp=ts, + ) + + +class TestPeekChannelHash(unittest.TestCase): + def test_short_payload_returns_none(self) -> None: + self.assertIsNone(_peek_channel_hash(b"\x00" * 10)) + + def test_reads_byte_thirteen_when_header_present(self) -> None: + payload = bytearray(b"\x00" * 16) + payload[13] = 0x2A + self.assertEqual(_peek_channel_hash(bytes(payload)), 0x2A) + + +class TestStrayFrameRepository(unittest.TestCase): + def setUp(self) -> None: + self.db = DatabaseManager(":memory:") + _run(self.db.connect()) + self.repo = StrayFrameRepository(self.db) + + def tearDown(self) -> None: + _run(self.db.disconnect()) + + def test_insert_and_list_recent(self) -> None: + payload = bytearray(b"\x00" * 16) + payload[13] = 7 + _run(self.repo.insert_from_capture(_capture(payload=bytes(payload)))) + + frames = _run(self.repo.list_recent(limit=10, hours=24)) + self.assertEqual(len(frames), 1) + frame = frames[0] + self.assertEqual(frame.frame_size, 16) + self.assertEqual(frame.channel_hash, 7) + self.assertAlmostEqual(frame.frequency_mhz, 906.875) + self.assertEqual(frame.spreading_factor, 11) + self.assertAlmostEqual(frame.bandwidth_khz, 250.0) + self.assertAlmostEqual(frame.rssi, -85.0) + self.assertAlmostEqual(frame.snr, 8.5) + self.assertEqual(frame.capture_source, "sx1302") + + def test_list_recent_filters_by_min_rssi(self) -> None: + _run(self.repo.insert_from_capture(_capture(rssi=-95.0))) + _run(self.repo.insert_from_capture(_capture(rssi=-70.0))) + + frames = _run(self.repo.list_recent(limit=10, hours=24, min_rssi=-80.0)) + self.assertEqual(len(frames), 1) + self.assertAlmostEqual(frames[0].rssi, -70.0) + + def test_cleanup_drops_rows_older_than_retention(self) -> None: + now = datetime.now(timezone.utc) + old = now - timedelta(hours=200) + _run(self.repo.insert_from_capture(_capture(timestamp=old))) + _run(self.repo.insert_from_capture(_capture(timestamp=now))) + + removed = _run(self.repo.cleanup(max_retained=10_000, retention_hours=168)) + self.assertEqual(removed, 1) + self.assertEqual(_run(self.repo.get_count()), 1) + + def test_cleanup_trims_to_max_retained(self) -> None: + base = datetime.now(timezone.utc) + for offset in range(5): + ts = base + timedelta(seconds=offset) + _run(self.repo.insert_from_capture(_capture(timestamp=ts))) + + removed = _run(self.repo.cleanup(max_retained=2, retention_hours=0)) + self.assertEqual(removed, 3) + frames = _run(self.repo.list_recent(limit=10, hours=None)) + self.assertEqual(len(frames), 2) + self.assertGreater(frames[0].timestamp, frames[1].timestamp) + + +if __name__ == "__main__": + unittest.main()