diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 76697686..551db4e0 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -222,12 +222,31 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, } } + // Status topic: meshcore///status + // Checked BEFORE JSON parse — non-JSON or malformed payloads must still update last_seen (#463) + if len(parts) >= 4 && parts[3] == "status" { + observerID := parts[2] + iata := parts[1] + var name string + var meta *ObserverMeta + var statusMsg map[string]interface{} + if json.Unmarshal(m.Payload(), &statusMsg) == nil { + name, _ = statusMsg["origin"].(string) + meta = extractObserverMeta(statusMsg) + } + if err := store.UpsertObserver(observerID, name, iata, meta); err != nil { + log.Printf("MQTT [%s] observer status error: %v", tag, err) + } + log.Printf("MQTT [%s] status: %s (%s)", tag, firstNonEmpty(name, observerID), iata) + return + } + var msg map[string]interface{} if err := json.Unmarshal(m.Payload(), &msg); err != nil { return } - // Skip status/connection topics + // Skip global status/connection topics if topic == "meshcore/status" || topic == "meshcore/events/connection" { return } @@ -261,6 +280,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, return } + // Format 1: Raw packet (meshcoretomqtt / Cisien format) rawHex, _ := msg["raw"].(string) if rawHex != "" { diff --git a/cmd/ingestor/main_test.go b/cmd/ingestor/main_test.go index 11935826..ec0f4f2d 100644 --- a/cmd/ingestor/main_test.go +++ b/cmd/ingestor/main_test.go @@ -201,6 +201,28 @@ func TestHandleMessageStatusTopic(t *testing.T) { } } +// #463: status messages with non-JSON payloads must still update last_seen. +// Some firmware versions may send plain-text or binary status payloads. +func TestHandleMessageStatusNonJSONPayload(t *testing.T) { + store := newTestStore(t) + source := MQTTSource{Name: "test"} + msg := &mockMessage{ + topic: "meshcore/SJC/obs1/status", + payload: []byte(`not json`), + } + + handleMessage(store, "test", source, msg, nil, nil) + + var lastSeen string + err := store.db.QueryRow("SELECT last_seen FROM observers WHERE id = 'obs1'").Scan(&lastSeen) + if err != nil { + t.Fatalf("observer not found after non-JSON status: %v", err) + } + if lastSeen == "" { + t.Error("last_seen should be set even for non-JSON status payloads") + } +} + func TestHandleMessageSkipStatusTopics(t *testing.T) { store := newTestStore(t) source := MQTTSource{Name: "test"} diff --git a/cmd/server/neighbor_persist.go b/cmd/server/neighbor_persist.go index 2bce39f7..c942d99d 100644 --- a/cmd/server/neighbor_persist.go +++ b/cmd/server/neighbor_persist.go @@ -496,6 +496,12 @@ func backfillResolvedPathsAsync(store *PacketStore, dbPath string, chunkSize int affectedSet[r.txHash] = true if tx, ok := store.byHash[r.txHash]; ok { pickBestObservation(tx) + // tx.ResolvedPath is now updated; add newly resolved pubkeys to + // the relay time index. The first call during buildPathHopIndex + // was a no-op (ResolvedPath was nil then), so no duplicates. + if ms, err := time.Parse(time.RFC3339, tx.FirstSeen); err == nil { + addTxToRelayTimeIndex(store.relayTimes, tx, ms.UnixMilli()) + } } } } diff --git a/cmd/server/relay_liveness_test.go b/cmd/server/relay_liveness_test.go new file mode 100644 index 00000000..0190bbf1 --- /dev/null +++ b/cmd/server/relay_liveness_test.go @@ -0,0 +1,340 @@ +package main + +import ( + "sort" + "strings" + "testing" + "time" +) + +func makeRp(s string) *string { return &s } + +func TestAddTxToRelayTimeIndex_SingleNode(t *testing.T) { + idx := make(map[string][]int64) + pk := "aabbccdd11223344" + ts := time.Now().Add(-30 * time.Minute).UTC() + tx := &StoreTx{ + FirstSeen: ts.Format(time.RFC3339), + ResolvedPath: []*string{makeRp(pk)}, + } + addTxToRelayTimeIndex(idx, tx, ts.UnixMilli()) + if len(idx[pk]) != 1 { + t.Fatalf("expected 1 entry, got %d", len(idx[pk])) + } + wantMs := ts.UnixMilli() + // RFC3339 has second precision, so allow ±1000ms + if diff := idx[pk][0] - wantMs; diff < -1000 || diff > 1000 { + t.Errorf("timestamp mismatch: got %d, want ~%d", idx[pk][0], wantMs) + } +} + +func TestAddTxToRelayTimeIndex_SortedOrder(t *testing.T) { + idx := make(map[string][]int64) + pk := "aabbccdd11223344" + t1 := time.Now().Add(-2 * time.Hour).UTC() + t2 := time.Now().Add(-30 * time.Minute).UTC() + + // Insert newer first, expect sorted ascending + tx2 := &StoreTx{FirstSeen: t2.Format(time.RFC3339), ResolvedPath: []*string{makeRp(pk)}} + tx1 := &StoreTx{FirstSeen: t1.Format(time.RFC3339), ResolvedPath: []*string{makeRp(pk)}} + addTxToRelayTimeIndex(idx, tx2, t2.UnixMilli()) + addTxToRelayTimeIndex(idx, tx1, t1.UnixMilli()) + + if len(idx[pk]) != 2 { + t.Fatalf("expected 2 entries, got %d", len(idx[pk])) + } + if !sort.SliceIsSorted(idx[pk], func(i, j int) bool { return idx[pk][i] < idx[pk][j] }) { + t.Error("relayTimes slice not sorted ascending") + } +} + +func TestAddTxToRelayTimeIndex_MultipleNodes(t *testing.T) { + idx := make(map[string][]int64) + pk1 := "aabbccdd11223344" + pk2 := "eeff001122334455" + ts := time.Now().Add(-10 * time.Minute).UTC() + tx := &StoreTx{ + FirstSeen: ts.Format(time.RFC3339), + ResolvedPath: []*string{makeRp(pk1), makeRp(pk2)}, + } + addTxToRelayTimeIndex(idx, tx, ts.UnixMilli()) + if len(idx[pk1]) != 1 { + t.Errorf("pk1: expected 1 entry, got %d", len(idx[pk1])) + } + if len(idx[pk2]) != 1 { + t.Errorf("pk2: expected 1 entry, got %d", len(idx[pk2])) + } +} + +func TestAddTxToRelayTimeIndex_NilResolvedPath(t *testing.T) { + idx := make(map[string][]int64) + tx := &StoreTx{FirstSeen: time.Now().UTC().Format(time.RFC3339), ResolvedPath: nil} + addTxToRelayTimeIndex(idx, tx, time.Now().UnixMilli()) // must not panic + if len(idx) != 0 { + t.Error("expected empty index for nil ResolvedPath") + } +} + +func TestAddTxToRelayTimeIndex_DuplicatePubkeyInPath(t *testing.T) { + idx := make(map[string][]int64) + pk := "aabbccdd11223344" + ts := time.Now().UTC() + tx := &StoreTx{ + FirstSeen: ts.Format(time.RFC3339), + ResolvedPath: []*string{makeRp(pk), makeRp(pk)}, // same pubkey twice + } + addTxToRelayTimeIndex(idx, tx, ts.UnixMilli()) + if len(idx[pk]) != 1 { + t.Errorf("duplicate pubkey should produce only 1 entry, got %d", len(idx[pk])) + } +} + +func TestRemoveFromRelayTimeIndex_RemovesEntry(t *testing.T) { + idx := make(map[string][]int64) + pk := "aabbccdd11223344" + ts := time.Now().Add(-1 * time.Hour).Truncate(time.Second).UTC() + tx := &StoreTx{FirstSeen: ts.Format(time.RFC3339), ResolvedPath: []*string{makeRp(pk)}} + + addTxToRelayTimeIndex(idx, tx, ts.UnixMilli()) + if len(idx[pk]) != 1 { + t.Fatal("setup: expected 1 entry") + } + removeFromRelayTimeIndex(idx, tx) + if _, ok := idx[pk]; ok { + t.Error("expected key deleted after last entry removed") + } +} + +func TestRemoveFromRelayTimeIndex_PartialRemove(t *testing.T) { + idx := make(map[string][]int64) + pk := "aabbccdd11223344" + t1 := time.Now().Add(-2 * time.Hour).Truncate(time.Second).UTC() + t2 := time.Now().Add(-30 * time.Minute).Truncate(time.Second).UTC() + tx1 := &StoreTx{FirstSeen: t1.Format(time.RFC3339), ResolvedPath: []*string{makeRp(pk)}} + tx2 := &StoreTx{FirstSeen: t2.Format(time.RFC3339), ResolvedPath: []*string{makeRp(pk)}} + + addTxToRelayTimeIndex(idx, tx1, t1.UnixMilli()) + addTxToRelayTimeIndex(idx, tx2, t2.UnixMilli()) + removeFromRelayTimeIndex(idx, tx1) + + if len(idx[pk]) != 1 { + t.Errorf("expected 1 entry after removing one, got %d", len(idx[pk])) + } +} + +func TestRelayMetrics_Counts(t *testing.T) { + now := time.Now().UnixMilli() + times := []int64{ + now - 90*60*1000, // 90 min ago — inside 24h, outside 1h + now - 30*60*1000, // 30 min ago — inside both + now - 10*60*1000, // 10 min ago — inside both + } + c1h, c24h, lastRelayed := relayMetrics(times, now) + if c1h != 2 { + t.Errorf("relay_count_1h: expected 2, got %d", c1h) + } + if c24h != 3 { + t.Errorf("relay_count_24h: expected 3, got %d", c24h) + } + wantLast := time.UnixMilli(times[2]).UTC().Format(time.RFC3339) + if lastRelayed != wantLast { + t.Errorf("last_relayed: got %q, want %q", lastRelayed, wantLast) + } +} + +func TestRelayMetrics_EmptySlice(t *testing.T) { + c1h, c24h, lastRelayed := relayMetrics(nil, time.Now().UnixMilli()) + if c1h != 0 || c24h != 0 || lastRelayed != "" { + t.Errorf("empty slice: expected zeros and empty string, got %d %d %q", c1h, c24h, lastRelayed) + } +} + +func TestRelayMetrics_AllOutsideWindow(t *testing.T) { + now := time.Now().UnixMilli() + times := []int64{now - 30*24*60*60*1000} // 30 days ago + c1h, c24h, _ := relayMetrics(times, now) + if c1h != 0 || c24h != 0 { + t.Errorf("expected 0/0 for old entry, got %d/%d", c1h, c24h) + } +} + +func TestRelayTimesWiredIntoIngest(t *testing.T) { + srv, _ := setupTestServer(t) + + srv.store.mu.RLock() + hopKeys := len(srv.store.byPathHop) + relayKeys := len(srv.store.relayTimes) + srv.store.mu.RUnlock() + + if hopKeys == 0 { + t.Skip("no path-hop data in test store — skipping relay wiring test") + } + // relayTimes will only be populated if test packets have ResolvedPath entries. + // At minimum it must not panic and must be initialised. + if srv.store.relayTimes == nil { + t.Fatal("relayTimes map is nil after load") + } + if relayKeys == 0 { + t.Fatalf("relayTimes not populated: byPathHop has %d keys but relayTimes has 0", hopKeys) + } + if relayKeys > hopKeys { + t.Errorf("relayTimes has more keys (%d) than byPathHop (%d) — relay index should be a subset", relayKeys, hopKeys) + } + t.Logf("byPathHop keys: %d, relayTimes keys: %d", hopKeys, relayKeys) +} + +func TestGetBulkHealthRepeaterRelayFields(t *testing.T) { + srv, _ := setupTestServer(t) + + // Insert a synthetic repeater node into the DB if none exists + _, err := srv.db.conn.Exec(`INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('relay662test0001', 'TestRepeater662', 'repeater', datetime('now'), datetime('now'), 1)`) + if err != nil { + t.Fatalf("insert test node: %v", err) + } + + // Inject a relay timestamp within the last hour + pk := "relay662test0001" + now := time.Now().UnixMilli() + recentMs := now - 10*60*1000 // 10 min ago + srv.store.mu.Lock() + srv.store.relayTimes[pk] = []int64{recentMs} + srv.store.mu.Unlock() + + results := srv.store.GetBulkHealth(200, "") + + var found map[string]interface{} + for _, r := range results { + if r["public_key"] == pk { + found = r + break + } + } + if found == nil { + t.Fatal("test repeater not found in GetBulkHealth results") + } + + stats, ok := found["stats"].(map[string]interface{}) + if !ok { + t.Fatal("missing stats map in result") + } + + if v, ok := stats["relay_count_1h"].(int); !ok || v != 1 { + t.Errorf("relay_count_1h: expected 1, got %v", stats["relay_count_1h"]) + } + if v, ok := stats["relay_count_24h"].(int); !ok || v != 1 { + t.Errorf("relay_count_24h: expected 1, got %v", stats["relay_count_24h"]) + } + if _, ok := stats["last_relayed"].(string); !ok { + t.Errorf("last_relayed: expected string, got %T", stats["last_relayed"]) + } +} + +func TestGetBulkHealthCompanionNoRelayFields(t *testing.T) { + srv, _ := setupTestServer(t) + + _, err := srv.db.conn.Exec(`INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('comp662test0001', 'TestCompanion662', 'companion', datetime('now'), datetime('now'), 1)`) + if err != nil { + t.Fatalf("insert test node: %v", err) + } + + // Give the companion a relay entry (should be ignored by role gate) + pk := "comp662test0001" + srv.store.mu.Lock() + srv.store.relayTimes[pk] = []int64{time.Now().UnixMilli() - 5*60*1000} + srv.store.mu.Unlock() + + results := srv.store.GetBulkHealth(200, "") + for _, r := range results { + if r["public_key"] == pk { + stats, _ := r["stats"].(map[string]interface{}) + if _, present := stats["relay_count_24h"]; present { + t.Error("relay_count_24h should be absent for companion nodes") + } + return + } + } + t.Fatal("test companion not found in GetBulkHealth results") +} + +func TestGetBulkHealthRepeaterNoRelayActivity(t *testing.T) { + srv, _ := setupTestServer(t) + + _, err := srv.db.conn.Exec(`INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('relay662idle001', 'IdleRepeater662', 'repeater', datetime('now'), datetime('now'), 1)`) + if err != nil { + t.Fatalf("insert test node: %v", err) + } + + // No entry in relayTimes for this node + results := srv.store.GetBulkHealth(200, "") + for _, r := range results { + if r["public_key"] == "relay662idle001" { + stats, _ := r["stats"].(map[string]interface{}) + if v, ok := stats["relay_count_24h"].(int); !ok || v != 0 { + t.Errorf("relay_count_24h: expected 0, got %v", stats["relay_count_24h"]) + } + if _, present := stats["last_relayed"]; present { + t.Error("last_relayed should be absent when no relay activity") + } + return + } + } + t.Fatal("idle repeater not found in results") +} + +func TestAddTxToRelayTimeIndex_LowercasesKey(t *testing.T) { + idx := make(map[string][]int64) + pkUpper := "AABBCCDD11223344" + pkLower := strings.ToLower(pkUpper) + ts := time.Now().UTC() + tx := &StoreTx{FirstSeen: ts.Format(time.RFC3339), ResolvedPath: []*string{makeRp(pkUpper)}} + addTxToRelayTimeIndex(idx, tx, ts.UnixMilli()) + if len(idx[pkLower]) != 1 { + t.Errorf("expected index keyed by lowercase, found %d entries at lowercase key", len(idx[pkLower])) + } + if len(idx[pkUpper]) != 0 { + t.Errorf("expected no entry at uppercase key") + } +} + +func TestGetNodeHealthRepeaterRelayFields(t *testing.T) { + srv, _ := setupTestServer(t) + + pk := "relay662node0001" + _, err := srv.db.conn.Exec(`INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('relay662node0001', 'TestRepeaterNode662', 'repeater', datetime('now'), datetime('now'), 1)`) + if err != nil { + t.Fatalf("insert test node: %v", err) + } + + now := time.Now().UnixMilli() + recentMs := now - 15*60*1000 // 15 min ago + srv.store.mu.Lock() + srv.store.relayTimes[pk] = []int64{recentMs} + srv.store.mu.Unlock() + + result, err := srv.store.GetNodeHealth(pk) + if err != nil { + t.Fatalf("GetNodeHealth error: %v", err) + } + if result == nil { + t.Fatal("GetNodeHealth returned nil") + } + + stats, ok := result["stats"].(map[string]interface{}) + if !ok { + t.Fatal("missing stats map in GetNodeHealth result") + } + + if v, ok := stats["relay_count_1h"].(int); !ok || v != 1 { + t.Errorf("relay_count_1h: expected 1, got %v", stats["relay_count_1h"]) + } + if v, ok := stats["relay_count_24h"].(int); !ok || v != 1 { + t.Errorf("relay_count_24h: expected 1, got %v", stats["relay_count_24h"]) + } + if _, ok := stats["last_relayed"].(string); !ok { + t.Errorf("last_relayed: expected string, got %T", stats["last_relayed"]) + } +} diff --git a/cmd/server/routes.go b/cmd/server/routes.go index b79cf239..9f841584 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -1038,9 +1038,46 @@ func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) { } if s.store != nil { hashInfo := s.store.GetNodeHashSizeInfo() + now := time.Now().UnixMilli() + + // Snapshot relay times for repeater nodes under the read lock, then + // release before enrichment so the lock isn't held for the full loop. + relaySnap := make(map[string][]int64) + s.store.mu.RLock() + for _, node := range nodes { + pk, ok := node["public_key"].(string) + if !ok { + continue + } + if role, _ := node["role"].(string); strings.ToLower(role) == "repeater" { + lk := strings.ToLower(pk) + if times := s.store.relayTimes[lk]; len(times) > 0 { + cp := make([]int64, len(times)) + copy(cp, times) + relaySnap[lk] = cp + } + } + } + s.store.mu.RUnlock() + for _, node := range nodes { - if pk, ok := node["public_key"].(string); ok { - EnrichNodeWithHashSize(node, hashInfo[pk]) + pk, ok := node["public_key"].(string) + if !ok { + continue + } + EnrichNodeWithHashSize(node, hashInfo[pk]) + role, _ := node["role"].(string) + if strings.ToLower(role) == "repeater" { + lk := strings.ToLower(pk) + c1h, c24h, lastRel := relayMetrics(relaySnap[lk], now) + stats := map[string]interface{}{ + "relay_count_1h": c1h, + "relay_count_24h": c24h, + } + if lastRel != "" && (c1h > 0 || c24h > 0) { + stats["last_relayed"] = lastRel + } + node["stats"] = stats } } } diff --git a/cmd/server/store.go b/cmd/server/store.go index fc905f1d..4b8cda57 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -132,6 +132,7 @@ type PacketStore struct { byNode map[string][]*StoreTx // pubkey → transmissions nodeHashes map[string]map[string]bool // pubkey → Set byPathHop map[string][]*StoreTx // lowercase hop/pubkey → transmissions with that hop in path + relayTimes map[string][]int64 // lowercase pubkey → sorted unix-millis of relay events (full pubkeys only) byPayloadType map[int][]*StoreTx // payload_type → transmissions loaded bool totalObs int @@ -297,6 +298,7 @@ func NewPacketStore(db *DB, cfg *PacketStoreConfig, cacheTTLs ...map[string]inte byObserver: make(map[string][]*StoreObs), byNode: make(map[string][]*StoreTx), byPathHop: make(map[string][]*StoreTx), + relayTimes: make(map[string][]int64), nodeHashes: make(map[string]map[string]bool), byPayloadType: make(map[int][]*StoreTx), rfCache: make(map[string]*cachedResult), @@ -371,10 +373,15 @@ func (s *PacketStore) Load() error { if s.db.hasResolvedPath { rpCol = ",\n\t\t\t\to.resolved_path" } - - limitClause := "" - if maxPackets > 0 { - limitClause = fmt.Sprintf( + var whereClause string + if s.retentionHours > 0 && maxPackets > 0 { + whereClause = fmt.Sprintf( + "\n\t\t\tWHERE t.first_seen >= datetime('now', '-%.0f hours') AND t.id IN (SELECT id FROM transmissions ORDER BY first_seen DESC LIMIT %d)", + s.retentionHours, maxPackets) + } else if s.retentionHours > 0 { + whereClause = fmt.Sprintf("\n\t\t\tWHERE t.first_seen >= datetime('now', '-%.0f hours')", s.retentionHours) + } else if maxPackets > 0 { + whereClause = fmt.Sprintf( "\n\t\t\tWHERE t.id IN (SELECT id FROM transmissions ORDER BY first_seen DESC LIMIT %d)", maxPackets) } @@ -385,7 +392,7 @@ func (s *PacketStore) Load() error { o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + rpCol + ` FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + limitClause + ` + LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + whereClause + ` ORDER BY t.first_seen ASC, o.timestamp DESC` } else { loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, @@ -393,7 +400,7 @@ func (s *PacketStore) Load() error { o.id, o.observer_id, o.observer_name, o.direction, o.snr, o.rssi, o.score, o.path_json, o.timestamp` + rpCol + ` FROM transmissions t - LEFT JOIN observations o ON o.transmission_id = t.id` + limitClause + ` + LEFT JOIN observations o ON o.transmission_id = t.id` + whereClause + ` ORDER BY t.first_seen ASC, o.timestamp DESC` } @@ -1541,6 +1548,9 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac s.spTotalPaths++ } addTxToPathHopIndex(s.byPathHop, tx) + if ms, err := time.Parse(time.RFC3339, tx.FirstSeen); err == nil { + addTxToRelayTimeIndex(s.relayTimes, tx, ms.UnixMilli()) + } } // Incrementally update precomputed distance index with new transmissions @@ -1918,6 +1928,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] tx.parsedPath, tx.pathParsed = oldHops, true tx.ResolvedPath = oldResolvedPaths[txID] removeTxFromPathHopIndex(s.byPathHop, tx) + removeFromRelayTimeIndex(s.relayTimes, tx) tx.parsedPath, tx.pathParsed = saved, savedFlag tx.ResolvedPath = savedRP } @@ -1927,6 +1938,15 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] s.spTotalPaths++ } addTxToPathHopIndex(s.byPathHop, tx) + if ms, err := time.Parse(time.RFC3339, tx.FirstSeen); err == nil { + addTxToRelayTimeIndex(s.relayTimes, tx, ms.UnixMilli()) + } + } else { + // Path unchanged: new observation may have relay hops in its resolved + // path that aren't indexed yet (idempotent — safe to call repeatedly). + if ms, err := time.Parse(time.RFC3339, tx.FirstSeen); err == nil { + addTxToRelayTimeIndex(s.relayTimes, tx, ms.UnixMilli()) + } } } @@ -2479,10 +2499,15 @@ func (s *PacketStore) buildSubpathIndex() { // Must be called with s.mu held. func (s *PacketStore) buildPathHopIndex() { s.byPathHop = make(map[string][]*StoreTx, 4096) + s.relayTimes = make(map[string][]int64, 4096) + cutoff := time.Now().Add(-24 * time.Hour) for _, tx := range s.packets { addTxToPathHopIndex(s.byPathHop, tx) + if t, err := time.Parse(time.RFC3339, tx.FirstSeen); err == nil && t.After(cutoff) { + addTxToRelayTimeIndex(s.relayTimes, tx, t.UnixMilli()) + } } - log.Printf("[store] Built path-hop index: %d unique keys", len(s.byPathHop)) + log.Printf("[store] Built path-hop index: %d unique keys, %d relay-time keys", len(s.byPathHop), len(s.relayTimes)) } // addTxToPathHopIndex indexes a transmission under each unique hop key @@ -2501,7 +2526,7 @@ func addTxToPathHopIndex(idx map[string][]*StoreTx, tx *StoreTx) { } // Also index by resolved pubkey if available if tx.ResolvedPath != nil && i < len(tx.ResolvedPath) && tx.ResolvedPath[i] != nil { - pk := *tx.ResolvedPath[i] + pk := strings.ToLower(*tx.ResolvedPath[i]) if !seen[pk] { seen[pk] = true idx[pk] = append(idx[pk], tx) @@ -2510,6 +2535,105 @@ func addTxToPathHopIndex(idx map[string][]*StoreTx, tx *StoreTx) { } } +// addTxToRelayTimeIndex records the relay timestamp for each full pubkey that +// appears in ANY observation's resolved path. Scanning all observations (not +// just the best one) ensures relay activity is captured even when the best +// observer received the packet directly without a relay hop. +// Insert is idempotent: if the same timestamp already exists for a pubkey it is +// not duplicated, so the function may be called multiple times safely. +// millis is tx.FirstSeen already parsed to unix-milliseconds by the caller. +// Must be called with s.mu held (or during build before store is live). +func addTxToRelayTimeIndex(idx map[string][]int64, tx *StoreTx, millis int64) { + seen := make(map[string]bool) + insert := func(rp *string) { + if rp == nil { + return + } + pk := strings.ToLower(*rp) + if seen[pk] { + return + } + seen[pk] = true + slice := idx[pk] + i := sort.Search(len(slice), func(j int) bool { return slice[j] >= millis }) + if i < len(slice) && slice[i] == millis { + return // idempotent: already present + } + slice = append(slice, 0) + copy(slice[i+1:], slice[i:]) + slice[i] = millis + idx[pk] = slice + } + // Scan all observations — relay hops appear in any observation's path. + for _, obs := range tx.Observations { + for _, rp := range obs.ResolvedPath { + insert(rp) + } + } + // Fallback for DB-loaded packets where Observations slice may be empty + // but tx.ResolvedPath (best observation) is populated. + for _, rp := range tx.ResolvedPath { + insert(rp) + } +} + +// removeFromRelayTimeIndex removes the relay timestamp for every full pubkey +// that appears in any observation's resolved path. Symmetric with +// addTxToRelayTimeIndex so eviction does not leave orphaned entries. +func removeFromRelayTimeIndex(idx map[string][]int64, tx *StoreTx) { + ms, err := time.Parse(time.RFC3339, tx.FirstSeen) + if err != nil { + return + } + millis := ms.UnixMilli() + seen := make(map[string]bool) + remove := func(rp *string) { + if rp == nil { + return + } + pk := strings.ToLower(*rp) + if seen[pk] { + return + } + seen[pk] = true + slice := idx[pk] + i := sort.Search(len(slice), func(j int) bool { return slice[j] >= millis }) + if i < len(slice) && slice[i] == millis { + copy(slice[i:], slice[i+1:]) + slice = slice[:len(slice)-1] + if len(slice) == 0 { + delete(idx, pk) + } else { + idx[pk] = slice + } + } + } + for _, obs := range tx.Observations { + for _, rp := range obs.ResolvedPath { + remove(rp) + } + } + for _, rp := range tx.ResolvedPath { + remove(rp) + } +} + +// relayMetrics computes relay_count_1h, relay_count_24h, and last_relayed from a +// sorted unix-millis slice. now is time.Now().UnixMilli(). O(log n). +// last_relayed is always the most recent entry regardless of window age — callers +// receive it even when both counts are zero (e.g. "last relayed 3 days ago"). +func relayMetrics(times []int64, now int64) (count1h, count24h int, lastRelayed string) { + if len(times) == 0 { + return 0, 0, "" + } + i1h := sort.Search(len(times), func(i int) bool { return times[i] >= now-3600000 }) + i24h := sort.Search(len(times), func(i int) bool { return times[i] >= now-86400000 }) + count1h = len(times) - i1h + count24h = len(times) - i24h + lastRelayed = time.UnixMilli(times[len(times)-1]).UTC().Format(time.RFC3339) + return +} + // removeTxFromPathHopIndex removes a transmission from all its path-hop index entries. func removeTxFromPathHopIndex(idx map[string][]*StoreTx, tx *StoreTx) { hops := txGetParsedPath(tx) @@ -2524,7 +2648,7 @@ func removeTxFromPathHopIndex(idx map[string][]*StoreTx, tx *StoreTx) { removeTxFromSlice(idx, key, tx) } if tx.ResolvedPath != nil && i < len(tx.ResolvedPath) && tx.ResolvedPath[i] != nil { - pk := *tx.ResolvedPath[i] + pk := strings.ToLower(*tx.ResolvedPath[i]) if !seen[pk] { seen[pk] = true removeTxFromSlice(idx, pk, tx) @@ -2909,6 +3033,7 @@ func (s *PacketStore) EvictStale() int { removeTxFromSubpathIndexFull(s.spIndex, s.spTxIndex, tx) // Remove from path-hop index removeTxFromPathHopIndex(s.byPathHop, tx) + removeFromRelayTimeIndex(s.relayTimes, tx) } // Batch-remove from byObserver: single pass per affected observer slice @@ -6238,21 +6363,30 @@ func (s *PacketStore) GetBulkHealth(limit int, region string) []map[string]inter lhVal = lastHeard } + statsMap := map[string]interface{}{ + "totalTransmissions": len(packets), + "totalObservations": totalObservations, + "totalPackets": len(packets), + "packetsToday": packetsToday, + "avgSnr": avgSnr, + "lastHeard": lhVal, + } + if strings.ToLower(n.role) == "repeater" { + c1h, c24h, lastRel := relayMetrics(s.relayTimes[strings.ToLower(n.pk)], time.Now().UnixMilli()) + statsMap["relay_count_1h"] = c1h + statsMap["relay_count_24h"] = c24h + if lastRel != "" && (c1h > 0 || c24h > 0) { + statsMap["last_relayed"] = lastRel + } + } results = append(results, map[string]interface{}{ "public_key": n.pk, "name": nilIfEmpty(n.name), "role": nilIfEmpty(n.role), "lat": n.lat, "lon": n.lon, - "stats": map[string]interface{}{ - "totalTransmissions": len(packets), - "totalObservations": totalObservations, - "totalPackets": len(packets), - "packetsToday": packetsToday, - "avgSnr": avgSnr, - "lastHeard": lhVal, - }, - "observers": observerRows, + "stats": statsMap, + "observers": observerRows, }) } @@ -6389,18 +6523,32 @@ func (s *PacketStore) GetNodeHealth(pubkey string) (map[string]interface{}, erro recentPackets = append(recentPackets, p) } + nodeStats := map[string]interface{}{ + "totalTransmissions": len(packets), + "totalObservations": totalObservations, + "totalPackets": len(packets), + "packetsToday": packetsToday, + "avgSnr": avgSnr, + "avgHops": avgHops, + "lastHeard": lhVal, + } + role := "" + if r, ok := node["role"].(string); ok { + role = strings.ToLower(r) + } + if role == "repeater" { + lowerPK := strings.ToLower(pubkey) + c1h, c24h, lastRel := relayMetrics(s.relayTimes[lowerPK], time.Now().UnixMilli()) + nodeStats["relay_count_1h"] = c1h + nodeStats["relay_count_24h"] = c24h + if lastRel != "" && (c1h > 0 || c24h > 0) { + nodeStats["last_relayed"] = lastRel + } + } return map[string]interface{}{ - "node": node, - "observers": observerRows, - "stats": map[string]interface{}{ - "totalTransmissions": len(packets), - "totalObservations": totalObservations, - "totalPackets": len(packets), - "packetsToday": packetsToday, - "avgSnr": avgSnr, - "avgHops": avgHops, - "lastHeard": lhVal, - }, + "node": node, + "observers": observerRows, + "stats": nodeStats, "recentPackets": recentPackets, }, nil } diff --git a/deploy-live.sh b/deploy-live.sh index bc100d83..71fae575 100644 --- a/deploy-live.sh +++ b/deploy-live.sh @@ -2,7 +2,7 @@ set -e DEPLOY_DIR="$(cd "$(dirname "$0")" && pwd)" -MATOMO_COMMIT="38c30f9" +MATOMO_COMMIT="bb8fb43" cd "$DEPLOY_DIR" diff --git a/public/customize.js b/public/customize.js index f5cf61ec..7af2305c 100644 --- a/public/customize.js +++ b/public/customize.js @@ -439,6 +439,7 @@ // Current state let state = {}; + let _serverState = null; function deepClone(o) { return JSON.parse(JSON.stringify(o)); } @@ -475,6 +476,15 @@ mergedUi.timestampCustomFormat = (localTsCustomFormat != null) ? localTsCustomFormat : (typeof mergedUi.timestampCustomFormat === 'string' ? mergedUi.timestampCustomFormat : serverTsCustomFormat); + _serverState = { + branding: Object.assign({}, DEFAULTS.branding, cfg.branding || {}), + theme: Object.assign({}, DEFAULTS.theme, cfg.theme || {}), + themeDark: Object.assign({}, DEFAULTS.themeDark, cfg.themeDark || {}), + nodeColors: Object.assign({}, DEFAULTS.nodeColors, cfg.nodeColors || {}), + typeColors: Object.assign({}, DEFAULTS.typeColors, cfg.typeColors || {}), + home: Object.assign({}, DEFAULTS.home, cfg.home || {}), + ui: { timestampMode: serverTsMode, timestampTimezone: serverTsTimezone, timestampFormat: serverTsFormat, timestampCustomFormat: serverTsCustomFormat }, + }; state = { branding: mergeSection('branding'), theme: mergeSection('theme'), @@ -546,9 +556,12 @@ try { var data = buildExport(); localStorage.setItem('meshcore-user-theme', JSON.stringify(data)); - // Sync to SITE_CONFIG so live pages (home, etc.) pick up changes + showSaved(); + refreshOverrideIndicators(_custInner); + // Sync to SITE_CONFIG so live pages (home, branding) pick up changes if (window.SITE_CONFIG) { if (state.branding) window.SITE_CONFIG.branding = Object.assign(window.SITE_CONFIG.branding || {}, state.branding); + if (state.home) window.SITE_CONFIG.home = Object.assign({}, window._SITE_CONFIG_ORIGINAL_HOME || {}, state.home); } // Re-render current page to reflect home/branding changes window.dispatchEvent(new HashChangeEvent('hashchange')); @@ -562,6 +575,112 @@ } } + function isFieldOverridden(section, key) { + if (!_serverState || !_serverState[section]) return false; + var sv = _serverState[section][key]; + var cv = state[section] ? state[section][key] : undefined; + return (typeof sv === 'object' || typeof cv === 'object') + ? JSON.stringify(sv) !== JSON.stringify(cv) + : sv !== cv; + } + + function countSectionOverrides() { + var n = 0; + for (var s = 0; s < arguments.length; s++) { + var sec = arguments[s]; + if (!_serverState || !_serverState[sec]) continue; + for (var k in _serverState[sec]) { if (isFieldOverridden(sec, k)) n++; } + } + return n; + } + + function overrideDot(section, key) { + return isFieldOverridden(section, key) + ? ' ⬤' + : ''; + } + + var _savedToastTimer = null; + function showSaved() { + if (!panelEl) return; + var toast = panelEl.querySelector('.cust-saved-toast'); + if (!toast) { + toast = document.createElement('div'); + toast.className = 'cust-saved-toast'; + toast.textContent = '✓ Saved'; + panelEl.appendChild(toast); + } + toast.classList.add('visible'); + clearTimeout(_savedToastTimer); + _savedToastTimer = setTimeout(function() { toast.classList.remove('visible'); }, 1500); + } + + function refreshOverrideIndicators(container) { + if (!container || !_initialized || !_serverState) return; + + // Update tab badges + var tabCounts = { + branding: countSectionOverrides('branding'), + theme: countSectionOverrides('theme', 'themeDark'), + nodes: countSectionOverrides('nodeColors', 'typeColors'), + home: countSectionOverrides('home'), + display: countSectionOverrides('ui'), + export: 0 + }; + container.querySelectorAll('.cust-tab').forEach(function(btn) { + var count = tabCounts[btn.dataset.tab] || 0; + var badge = btn.querySelector('.cust-tab-badge'); + if (count > 0) { + if (!badge) { + badge = document.createElement('span'); + badge.className = 'cust-tab-badge'; + var tabText = btn.querySelector('.cust-tab-text'); + btn.insertBefore(badge, tabText); + } + badge.textContent = count; + } else if (badge) { + badge.remove(); + } + }); + + // Helper: add/remove override dot on a label element + function setDot(label, section, key) { + if (!label) return; + var dot = label.querySelector('.cust-override-dot'); + if (isFieldOverridden(section, key)) { + if (!dot) { + dot = document.createElement('span'); + dot.className = 'cust-override-dot'; + dot.title = 'Changed from server default'; + dot.textContent = '⬤'; + label.appendChild(dot); + } + } else if (dot) { + dot.remove(); + } + } + + // Theme color labels + var themeSection = isDarkMode() ? 'themeDark' : 'theme'; + container.querySelectorAll('input[data-theme]').forEach(function(inp) { + setDot(container.querySelector('label[for="cust-theme-' + inp.dataset.theme + '"]'), themeSection, inp.dataset.theme); + }); + // Node color labels + container.querySelectorAll('input[data-node]').forEach(function(inp) { + setDot(container.querySelector('label[for="cust-node-' + inp.dataset.node + '"]'), 'nodeColors', inp.dataset.node); + }); + // Type color labels + container.querySelectorAll('input[data-type-color]').forEach(function(inp) { + setDot(container.querySelector('label[for="cust-type-' + inp.dataset.typeColor + '"]'), 'typeColors', inp.dataset.typeColor); + }); + // Branding text fields + container.querySelectorAll('input[data-key^="branding."]').forEach(function(inp) { + var key = inp.dataset.key.replace('branding.', ''); + var label = inp.closest('.cust-field') && inp.closest('.cust-field').querySelector('label'); + setDot(label, 'branding', key); + }); + } + function esc(s) { var d = document.createElement('div'); d.textContent = s || ''; return d.innerHTML; } function escAttr(s) { return (s || '').replace(/&/g, '&').replace(/"/g, '"').replace(/' + tabs.map(function (t) { - return ''; + var badge = t.count > 0 ? '' + t.count + '' : ''; + return ''; }).join('') + ''; } + function brandingField(id, key, label, extra) { + var val = state.branding[key] || ''; + var sval = (_serverState && _serverState.branding) ? (_serverState.branding[key] || '') : val; + var changed = val !== sval; + var resetBtn = changed ? '' : ''; + var dot = changed ? overrideDot('branding', key) : ''; + return '
' + + '
' + resetBtn + '
'; + } + function renderBranding() { var b = state.branding; var logoPreview = b.logoUrl ? 'Logo preview' : ''; return '
' + - '
' + - '
' + - '
' + logoPreview + '
' + - '
' + + brandingField('cust-siteName', 'siteName', 'Site Name') + + brandingField('cust-tagline', 'tagline', 'Tagline') + + brandingField('cust-logoUrl', 'logoUrl', 'Logo URL', ' placeholder="https://..."') + + (b.logoUrl ? '
' + logoPreview + '
' : '') + + brandingField('cust-faviconUrl', 'faviconUrl', 'Favicon URL', ' placeholder="https://..."') + '
'; } @@ -726,17 +865,20 @@ ''; } - function renderColorRow(key, val, def, dataAttr) { + function renderColorRow(key, val, def, dataAttr, srvDef) { + var baseline = (srvDef !== undefined) ? srvDef : def; + var changed = val !== baseline; var isFont = key === 'font' || key === 'mono'; + var dot = changed ? ' ⬤' : ''; var inputHtml = isFont ? '' : '' + '' + val + ''; return '
' + - '
' + + '
' + '
' + (THEME_HINTS[key] || '') + '
' + inputHtml + - (val !== def ? '' : '') + + (changed ? '' : '') + '
'; } @@ -745,23 +887,25 @@ var modeLabel = dark ? '🌙 Dark Mode' : '☀️ Light Mode'; var defs = activeDefaults(); var current = activeTheme(); + var themeSection = dark ? 'themeDark' : 'theme'; + var srv = (_serverState && _serverState[themeSection]) ? _serverState[themeSection] : {}; var basicRows = ''; for (var i = 0; i < BASIC_KEYS.length; i++) { var key = BASIC_KEYS[i]; - basicRows += renderColorRow(key, current[key] || defs[key] || '#000000', defs[key] || '#000000', 'theme'); + basicRows += renderColorRow(key, current[key] || defs[key] || '#000000', defs[key] || '#000000', 'theme', srv[key] || defs[key] || '#000000'); } var advancedRows = ''; for (var j = 0; j < ADVANCED_KEYS.length; j++) { var akey = ADVANCED_KEYS[j]; - advancedRows += renderColorRow(akey, current[akey] || defs[akey] || '#000000', defs[akey] || '#000000', 'theme'); + advancedRows += renderColorRow(akey, current[akey] || defs[akey] || '#000000', defs[akey] || '#000000', 'theme', srv[akey] || defs[akey] || '#000000'); } var fontRows = ''; for (var f = 0; f < FONT_KEYS.length; f++) { var fkey = FONT_KEYS[f]; - fontRows += renderColorRow(fkey, current[fkey] || defs[fkey] || '', defs[fkey] || '', 'theme'); + fontRows += renderColorRow(fkey, current[fkey] || defs[fkey] || '', defs[fkey] || '', 'theme', srv[fkey] || defs[fkey] || ''); } return '
' + @@ -780,30 +924,36 @@ } function renderNodes() { + var srvNode = (_serverState && _serverState.nodeColors) ? _serverState.nodeColors : {}; + var srvType = (_serverState && _serverState.typeColors) ? _serverState.typeColors : {}; var rows = ''; for (var key in NODE_LABELS) { var val = state.nodeColors[key]; - var def = DEFAULTS.nodeColors[key]; + var baseline = srvNode[key] || DEFAULTS.nodeColors[key]; + var changed = val !== baseline; + var dot = changed ? ' ⬤' : ''; rows += '
' + - '
' + + '
' + '
' + (NODE_HINTS[key] || '') + '
' + '' + '' + '' + val + '' + - (val !== def ? '' : '') + + (changed ? '' : '') + '
'; } var typeRows = ''; for (var tkey in TYPE_LABELS) { var tval = state.typeColors[tkey]; - var tdef = DEFAULTS.typeColors[tkey]; + var tbaseline = srvType[tkey] || DEFAULTS.typeColors[tkey]; + var tchanged = tval !== tbaseline; + var tdot = tchanged ? ' ⬤' : ''; typeRows += '
' + - '
' + + '
' + '
' + (TYPE_HINTS[tkey] || '') + '
' + '' + '' + '' + tval + '' + - (tval !== tdef ? '' : '') + + (tchanged ? '' : '') + '
'; } var heatOpacity = parseFloat(localStorage.getItem('meshcore-heatmap-opacity')); @@ -919,13 +1069,15 @@ } if (Object.keys(tc).length) out.typeColors = tc; - // Home + // Home — diff against server config (not DEFAULTS) so unmodified server fields + // are not captured in localStorage and don't block future server updates. + var serverHome = window._SITE_CONFIG_ORIGINAL_HOME || {}; var hm = {}; - if (state.home.heroTitle !== DEFAULTS.home.heroTitle) hm.heroTitle = state.home.heroTitle; - if (state.home.heroSubtitle !== DEFAULTS.home.heroSubtitle) hm.heroSubtitle = state.home.heroSubtitle; - if (JSON.stringify(state.home.steps) !== JSON.stringify(DEFAULTS.home.steps)) hm.steps = state.home.steps; - if (JSON.stringify(state.home.checklist) !== JSON.stringify(DEFAULTS.home.checklist)) hm.checklist = state.home.checklist; - if (JSON.stringify(state.home.footerLinks) !== JSON.stringify(DEFAULTS.home.footerLinks)) hm.footerLinks = state.home.footerLinks; + if (state.home.heroTitle !== (serverHome.heroTitle !== undefined ? serverHome.heroTitle : DEFAULTS.home.heroTitle)) hm.heroTitle = state.home.heroTitle; + if (state.home.heroSubtitle !== (serverHome.heroSubtitle !== undefined ? serverHome.heroSubtitle : DEFAULTS.home.heroSubtitle)) hm.heroSubtitle = state.home.heroSubtitle; + if (JSON.stringify(state.home.steps) !== JSON.stringify(serverHome.steps !== undefined ? serverHome.steps : DEFAULTS.home.steps)) hm.steps = state.home.steps; + if (JSON.stringify(state.home.checklist) !== JSON.stringify(serverHome.checklist !== undefined ? serverHome.checklist : DEFAULTS.home.checklist)) hm.checklist = state.home.checklist; + if (JSON.stringify(state.home.footerLinks) !== JSON.stringify(serverHome.footerLinks !== undefined ? serverHome.footerLinks : DEFAULTS.home.footerLinks)) hm.footerLinks = state.home.footerLinks; if (Object.keys(hm).length) out.home = hm; // UI @@ -939,14 +1091,34 @@ return out; } + function buildFullExport() { + return { + branding: Object.assign({}, state.branding), + theme: Object.assign({}, state.theme), + themeDark: Object.assign({}, state.themeDark), + nodeColors: Object.assign({}, state.nodeColors), + typeColors: Object.assign({}, state.typeColors), + home: { + heroTitle: state.home.heroTitle, + heroSubtitle: state.home.heroSubtitle, + steps: deepClone(state.home.steps), + checklist: deepClone(state.home.checklist), + footerLinks: deepClone(state.home.footerLinks), + }, + ui: Object.assign({}, state.ui), + }; + } + function renderExport() { - var json = JSON.stringify(buildExport(), null, 2); + var overrideCount = countSectionOverrides('branding', 'theme', 'themeDark', 'nodeColors', 'typeColors', 'home', 'ui'); var hasUserTheme = !!localStorage.getItem('meshcore-user-theme'); + var json = JSON.stringify(buildFullExport(), null, 2); return '
' + '

My Preferences

' + - '

Save these colors just for you — stored in your browser, works on any instance.

' + + '

' + + (overrideCount > 0 ? '' + overrideCount + ' setting' + (overrideCount !== 1 ? 's' : '') + ' changed from server defaults. ' : 'No changes from server defaults. ') + + 'Auto-saved to your browser.

' + '
' + - '' + (hasUserTheme ? '' : '') + '
' + '
' + @@ -965,8 +1137,10 @@ } let panelEl = null; + let _custInner = null; function render(container) { + _custInner = container; container.innerHTML = renderTabs() + '
' + @@ -1084,12 +1258,23 @@ btn.addEventListener('click', function () { var key = btn.dataset.resetTheme; var themeKey = isDarkMode() ? 'themeDark' : 'theme'; - state[themeKey][key] = activeDefaults()[key]; + var srvVal = (_serverState && _serverState[themeKey]) ? _serverState[themeKey][key] : undefined; + state[themeKey][key] = srvVal !== undefined ? srvVal : activeDefaults()[key]; applyThemePreview(); autoSave(); render(container); }); }); + container.querySelectorAll('[data-reset-branding]').forEach(function (btn) { + btn.addEventListener('click', function () { + var key = btn.dataset.resetBranding; + var srvVal = (_serverState && _serverState.branding) ? _serverState.branding[key] : undefined; + state.branding[key] = srvVal !== undefined ? srvVal : (DEFAULTS.branding[key] || ''); + autoSave(); + render(container); + }); + }); + // Reset preview button var resetBtn = document.getElementById('custResetPreview'); if (resetBtn) { @@ -1121,9 +1306,11 @@ container.querySelectorAll('[data-reset-node]').forEach(function (btn) { btn.addEventListener('click', function () { var key = btn.dataset.resetNode; - state.nodeColors[key] = DEFAULTS.nodeColors[key]; - if (window.ROLE_COLORS) window.ROLE_COLORS[key] = DEFAULTS.nodeColors[key]; - if (window.ROLE_STYLE && window.ROLE_STYLE[key]) window.ROLE_STYLE[key].color = DEFAULTS.nodeColors[key]; + var srvVal = (_serverState && _serverState.nodeColors) ? _serverState.nodeColors[key] : undefined; + var resetTo = srvVal !== undefined ? srvVal : DEFAULTS.nodeColors[key]; + state.nodeColors[key] = resetTo; + if (window.ROLE_COLORS) window.ROLE_COLORS[key] = resetTo; + if (window.ROLE_STYLE && window.ROLE_STYLE[key]) window.ROLE_STYLE[key].color = resetTo; render(container); }); }); @@ -1145,8 +1332,10 @@ container.querySelectorAll('[data-reset-type]').forEach(function (btn) { btn.addEventListener('click', function () { var key = btn.dataset.resetType; - state.typeColors[key] = DEFAULTS.typeColors[key]; - if (window.TYPE_COLORS) window.TYPE_COLORS[key] = DEFAULTS.typeColors[key]; + var srvVal = (_serverState && _serverState.typeColors) ? _serverState.typeColors[key] : undefined; + var resetTo = srvVal !== undefined ? srvVal : DEFAULTS.typeColors[key]; + state.typeColors[key] = resetTo; + if (window.TYPE_COLORS) window.TYPE_COLORS[key] = resetTo; render(container); }); }); @@ -1268,10 +1457,10 @@ } }); - // Export download + // Export download — full export including all current settings var dlBtn = document.getElementById('custDownload'); if (dlBtn) dlBtn.addEventListener('click', function () { - var json = JSON.stringify(buildExport(), null, 2); + var json = JSON.stringify(buildFullExport(), null, 2); var blob = new Blob([json], { type: 'application/json' }); var a = document.createElement('a'); a.href = URL.createObjectURL(blob); @@ -1280,15 +1469,6 @@ URL.revokeObjectURL(a.href); }); - // Save user theme to localStorage - var saveUserBtn = document.getElementById('custSaveUser'); - if (saveUserBtn) saveUserBtn.addEventListener('click', function () { - var exportData = buildExport(); - localStorage.setItem('meshcore-user-theme', JSON.stringify(exportData)); - saveUserBtn.textContent = '✓ Saved!'; - setTimeout(function () { saveUserBtn.textContent = '💾 Save as my theme'; }, 2000); - }); - // Reset user theme var resetUserBtn = document.getElementById('custResetUser'); if (resetUserBtn) resetUserBtn.addEventListener('click', function () { @@ -1429,6 +1609,10 @@ } } catch {} + // Test hooks + window._customizeBuildExport = buildExport; + window._customizeInitState = initState; + // Wire up toggle button (needs DOM) document.addEventListener('DOMContentLoaded', () => { const btn = document.getElementById('customizeToggle'); diff --git a/public/index.html b/public/index.html index 1187e0ce..51ab2ace 100644 --- a/public/index.html +++ b/public/index.html @@ -117,5 +117,19 @@ + + + diff --git a/public/nodes.js b/public/nodes.js index 2547541c..169e7b6f 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -114,7 +114,13 @@ const isInfra = role === 'repeater' || role === 'room'; const threshMs = isInfra ? HEALTH_THRESHOLDS.infraSilentMs : HEALTH_THRESHOLDS.nodeSilentMs; const threshold = threshMs >= 3600000 ? Math.round(threshMs / 3600000) + 'h' : Math.round(threshMs / 60000) + 'm'; + if (status === 'relaying') { + return 'Relaying \u2014 actively forwarding traffic within the last 24h.'; + } if (status === 'active') { + if (role === 'repeater') { + return 'Idle \u2014 alive (heard within ' + threshold + ') but no relay traffic observed in the last 24h. May be in a quiet area or have RF issues.'; + } return 'Active \u2014 heard within the last ' + threshold + '.' + (isInfra ? ' Repeaters typically advertise every 12-24h.' : ''); } if (role === 'companion') { @@ -133,13 +139,28 @@ // Prefer last_heard (from in-memory packets) > _lastHeard (health API) > last_seen (DB) const lastHeardTime = n._lastHeard || n.last_heard || n.last_seen; const lastHeardMs = lastHeardTime ? new Date(lastHeardTime).getTime() : 0; - const status = getNodeStatus(role, lastHeardMs); + const relayCount24h = (n.stats && typeof n.stats.relay_count_24h === 'number') ? n.stats.relay_count_24h : undefined; + const relayCount1h = (n.stats && typeof n.stats.relay_count_1h === 'number') ? n.stats.relay_count_1h : undefined; + const lastRelayed = n.stats && n.stats.last_relayed; + const status = getNodeStatus(role, lastHeardMs, relayCount24h); const statusTooltip = getStatusTooltip(role, status); - const statusLabel = status === 'active' ? '🟢 Active' : '⚪ Stale'; const statusAge = lastHeardMs ? (Date.now() - lastHeardMs) : Infinity; + let statusLabel; + if (role === 'repeater') { + statusLabel = status === 'relaying' ? '🟢 Relaying' : status === 'active' ? '🟡 Idle' : '⚪ Stale'; + } else { + statusLabel = status === 'active' ? '🟢 Active' : '⚪ Stale'; + } + let explanation = ''; - if (status === 'active') { + if (status === 'relaying') { + explanation = 'Relayed ' + relayCount24h + ' packet' + (relayCount24h === 1 ? '' : 's') + ' in last 24h' + + (lastRelayed ? ', last ' + renderNodeTimestampText(lastRelayed) : ''); + } else if (status === 'active' && role === 'repeater') { + explanation = 'Alive but no relay traffic observed in last 24h' + + (lastRelayed ? ', last relayed ' + renderNodeTimestampText(lastRelayed) : ''); + } else if (status === 'active') { explanation = 'Last heard ' + (lastHeardTime ? renderNodeTimestampText(lastHeardTime) : 'unknown'); } else { const ageDays = Math.floor(statusAge / 86400000); @@ -152,7 +173,7 @@ explanation = 'Not heard for ' + ageStr + ' — ' + reason; } - return { status, statusLabel, statusTooltip, statusAge, explanation, roleColor, lastHeardMs, role }; + return { status, statusLabel, statusTooltip, statusAge, explanation, roleColor, lastHeardMs, role, relayCount1h, relayCount24h, lastRelayed }; } function renderNodeBadges(n, roleColor) { @@ -450,8 +471,9 @@ const recent = h.recentPackets || []; const lastHeard = stats.lastHeard; - // Attach health lastHeard for shared helpers + // Attach health lastHeard and relay stats for shared helpers n._lastHeard = lastHeard || n.last_seen; + n.stats = stats; const si = getStatusInfo(n); const roleColor = si.roleColor; const statusLabel = si.statusLabel; @@ -491,6 +513,9 @@ Packets Today${stats.packetsToday || 0} ${stats.avgSnr != null ? `Avg SNR${Number(stats.avgSnr).toFixed(1)} dB` : ''} ${stats.avgHops ? `Avg Hops${stats.avgHops}` : ''} + ${si.role === 'repeater' ? `Relay (1h)${typeof si.relayCount1h === 'number' ? si.relayCount1h + ' packet' + (si.relayCount1h === 1 ? '' : 's') : '—'}` : ''} + ${si.role === 'repeater' ? `Relay (24h)${typeof si.relayCount24h === 'number' ? si.relayCount24h + ' packet' + (si.relayCount24h === 1 ? '' : 's') : '—'}` : ''} + ${si.role === 'repeater' && si.lastRelayed ? `Last Relayed${renderNodeTimestampHtml(si.lastRelayed)}` : ''} ${hasLoc ? `Location${Number(n.lat).toFixed(5)}, ${Number(n.lon).toFixed(5)}` : ''} Hash Prefix${n.hash_size ? '' + n.public_key.slice(0, n.hash_size * 2).toUpperCase() + ' (' + n.hash_size + '-byte)' : 'Unknown'}${n.hash_size_inconsistent ? ' ⚠️ varies' : ''} @@ -880,10 +905,9 @@ // Status filter (active/stale) if (statusFilter === 'active' || statusFilter === 'stale') { filtered = filtered.filter(n => { - const role = (n.role || 'companion').toLowerCase(); - const t = n.last_heard || n.last_seen; - const lastMs = t ? new Date(t).getTime() : 0; - return getNodeStatus(role, lastMs) === statusFilter; + const si = getStatusInfo(n); + if (statusFilter === 'active') return si.status === 'active' || si.status === 'relaying'; + return si.status === statusFilter; }); } nodes = filtered; @@ -971,6 +995,7 @@ Public Key Role Last Seen + St. Adverts @@ -1102,18 +1127,19 @@ const dupMap = buildDupNameMap(_allNodes); tbody.innerHTML = sorted.map(n => { - const roleColor = ROLE_COLORS[n.role] || '#6b7280'; const isClaimed = myKeys.has(n.public_key); - const lastSeenTime = n.last_heard || n.last_seen; - const status = getNodeStatus(n.role || 'companion', lastSeenTime ? new Date(lastSeenTime).getTime() : 0); - const lastSeenClass = status === 'active' ? 'last-seen-active' : 'last-seen-stale'; + const si = getStatusInfo(n); + const lastSeenClass = si.status === 'relaying' ? 'last-seen-active' : si.status === 'active' ? (si.role === 'repeater' ? 'last-seen-idle' : 'last-seen-active') : 'last-seen-stale'; + const statusEmoji = si.statusLabel.split(' ')[0]; + const statusTooltip = si.statusLabel.replace(/^.\s/, '') + (si.explanation ? ' — ' + si.explanation : ''); const cs = _fleetSkew && _fleetSkew[n.public_key]; const skewBadgeHtml = cs && cs.severity && cs.severity !== 'ok' ? renderSkewBadge(cs.severity, cs.medianSkewSec) : ''; return ` ${favStar(n.public_key, 'node-fav')}${isClaimed ? '★ ' : ''}${n.name || '(unnamed)'}${dupNameBadge(n.name, n.public_key, dupMap)}${skewBadgeHtml} ${truncate(n.public_key, 16)} - ${n.role} + ${n.role} ${renderNodeTimestampHtml(n.last_heard || n.last_seen)} + ${statusEmoji} ${n.advert_count || 0} `; }).join(''); @@ -1155,6 +1181,7 @@ // Status calculation via shared helper const lastHeard = stats.lastHeard; n._lastHeard = lastHeard || n.last_seen; + n.stats = stats; const si = getStatusInfo(n); const roleColor = si.roleColor; const totalPackets = stats.totalTransmissions || stats.totalPackets || n.advert_count || 0; @@ -1190,6 +1217,9 @@
Packets Today
${stats.packetsToday || 0}
${stats.avgSnr != null ? `
Avg SNR
${Number(stats.avgSnr).toFixed(1)} dB
` : ''} ${stats.avgHops ? `
Avg Hops
${stats.avgHops}
` : ''} + ${si.role === 'repeater' ? `
Relay (1h)
${typeof si.relayCount1h === 'number' ? si.relayCount1h + ' packet' + (si.relayCount1h === 1 ? '' : 's') : '—'}
` : ''} + ${si.role === 'repeater' ? `
Relay (24h)
${typeof si.relayCount24h === 'number' ? si.relayCount24h + ' packet' + (si.relayCount24h === 1 ? '' : 's') : '—'}
` : ''} + ${si.role === 'repeater' && si.lastRelayed ? `
Last Relayed
${renderNodeTimestampHtml(si.lastRelayed)}
` : ''} ${hasLoc ? `
Location
${Number(n.lat).toFixed(5)}, ${Number(n.lon).toFixed(5)}
` : ''}
diff --git a/public/observers.js b/public/observers.js index 3619eaf7..4c2e6a40 100644 --- a/public/observers.js +++ b/public/observers.js @@ -6,6 +6,8 @@ let wsHandler = null; let refreshTimer = null; let regionChangeHandler = null; + let _serverTimeAtFetch = null; // server clock at last API response + let _clientTimeAtFetch = null; // client clock at last API response function init(app) { app.innerHTML = ` @@ -57,6 +59,10 @@ try { const data = await api('/observers', { ttl: CLIENT_TTL.observers }); observers = data.observers || []; + if (data.server_time) { + _serverTimeAtFetch = new Date(data.server_time).getTime(); + _clientTimeAtFetch = Date.now(); + } render(); } catch (e) { document.getElementById('obsContent').innerHTML = @@ -64,14 +70,16 @@ } } - // NOTE: Comparing server timestamps to Date.now() can skew if client/server - // clocks differ. We add ±30s tolerance to thresholds to reduce false positives. + // Use server clock as reference to eliminate client/server clock skew (#463). + // _serverTimeAtFetch + elapsed gives a corrected "now" in server time. function healthStatus(lastSeen) { if (!lastSeen) return { cls: 'health-red', label: 'Unknown' }; - const ago = Date.now() - new Date(lastSeen).getTime(); - const tolerance = 30000; // 30s tolerance for clock skew - if (ago < 600000 + tolerance) return { cls: 'health-green', label: 'Online' }; // < 10 min + tolerance - if (ago < 3600000 + tolerance) return { cls: 'health-yellow', label: 'Stale' }; // < 1 hour + tolerance + const serverNow = (_serverTimeAtFetch !== null) + ? _serverTimeAtFetch + (Date.now() - _clientTimeAtFetch) + : Date.now(); + const ago = serverNow - new Date(lastSeen).getTime(); + if (ago < 600000) return { cls: 'health-green', label: 'Online' }; // < 10 min + if (ago < 3600000) return { cls: 'health-yellow', label: 'Stale' }; // < 1 hour return { cls: 'health-red', label: 'Offline' }; } diff --git a/public/packets.js b/public/packets.js index e0c56fd5..748f2d16 100644 --- a/public/packets.js +++ b/public/packets.js @@ -1589,7 +1589,7 @@ const types = filters.type.split(',').map(Number); displayPackets = displayPackets.filter(p => types.includes(p.payload_type)); } - if (filters.observer) { + if (filters.observer && !groupByHash) { const obsIds = new Set(filters.observer.split(',')); displayPackets = displayPackets.filter(p => { if (obsIds.has(p.observer_id)) return true; diff --git a/public/roles.js b/public/roles.js index bbeeeb88..00281c45 100644 --- a/public/roles.js +++ b/public/roles.js @@ -84,12 +84,16 @@ }; }; - // Simplified two-state helper: returns 'active' or 'stale' - window.getNodeStatus = function (role, lastSeenMs) { + // Three-state helper for repeaters: returns 'relaying', 'active', or 'stale' + window.getNodeStatus = function (role, lastSeenMs, relayCount24h) { var isInfra = role === 'repeater' || role === 'room'; var staleMs = isInfra ? HEALTH_THRESHOLDS.infraSilentMs : HEALTH_THRESHOLDS.nodeSilentMs; var age = typeof lastSeenMs === 'number' ? (Date.now() - lastSeenMs) : Infinity; - return age < staleMs ? 'active' : 'stale'; + if (age >= staleMs) return 'stale'; + if (role === 'repeater') { + return (typeof relayCount24h === 'number' && relayCount24h > 0) ? 'relaying' : 'active'; + } + return 'active'; }; // ─── Tile URLs ─── diff --git a/public/style.css b/public/style.css index 68374c8a..bfcbaa01 100644 --- a/public/style.css +++ b/public/style.css @@ -1621,6 +1621,7 @@ tr[data-hops]:hover { background: rgba(59,130,246,0.1); } .marker-stale { opacity: 0.7; filter: grayscale(90%) brightness(0.8); } .last-seen-active { color: var(--status-green); } .last-seen-stale { color: var(--text-muted); } +.last-seen-idle { color: var(--status-yellow); } /* === Node Analytics === */ .analytics-stats { display: grid; grid-template-columns: repeat(3, 1fr); gap: 8px; margin-bottom: 16px; } diff --git a/test-frontend-helpers.js b/test-frontend-helpers.js index 46fdb7f1..80584d94 100644 --- a/test-frontend-helpers.js +++ b/test-frontend-helpers.js @@ -326,10 +326,10 @@ console.log('\n=== nodes.js: getStatusTooltip / getStatusInfo (extracted) ==='); if (ex.getStatusInfo) { const gsi = ex.getStatusInfo; - test('active repeater status', () => { + test('active repeater status (idle — heard but no relay traffic)', () => { const info = gsi({ role: 'repeater', last_heard: new Date().toISOString() }); assert.strictEqual(info.status, 'active'); - assert.ok(info.statusLabel.includes('Active')); + assert.ok(info.statusLabel.includes('Idle')); }); test('stale companion status (old date)', () => { const old = new Date(Date.now() - 48 * 3600000).toISOString(); @@ -346,6 +346,19 @@ console.log('\n=== nodes.js: getStatusTooltip / getStatusInfo (extracted) ==='); const info = gsi({ role: 'repeater', last_heard: d }); assert.strictEqual(info.status, 'active'); }); + test('relaying repeater statusLabel and explanation', () => { + const info = gsi({ role: 'repeater', last_heard: new Date().toISOString(), + stats: { relay_count_24h: 42, relay_count_1h: 5, last_relayed: new Date().toISOString() } }); + assert.strictEqual(info.status, 'relaying'); + assert.ok(info.statusLabel.includes('Relaying'), 'statusLabel should include Relaying'); + assert.ok(info.explanation.includes('42 packet'), 'explanation should include packet count'); + }); + test('relaying repeater with relay_count_24h === 1 uses singular', () => { + const info = gsi({ role: 'repeater', last_heard: new Date().toISOString(), + stats: { relay_count_24h: 1 } }); + assert.ok(info.explanation.includes('1 packet'), 'should have singular'); + assert.ok(!info.explanation.includes('1 packets'), 'should not have plural'); + }); } if (ex.renderNodeBadges) { @@ -3904,9 +3917,9 @@ console.log('\n=== nodes.js: getStatusInfo edge cases ==='); assert.strictEqual(info.status, 'stale'); }); - test('getStatusInfo returns explanation for active node', () => { + test('getStatusInfo returns explanation for active repeater (idle)', () => { const info = gsi({ role: 'repeater', last_heard: new Date().toISOString() }); - assert.ok(info.explanation.includes('Last heard')); + assert.ok(info.explanation.includes('no relay traffic')); }); test('getStatusInfo returns explanation for stale companion', () => { diff --git a/test-repeater-liveness.js b/test-repeater-liveness.js new file mode 100644 index 00000000..39902ad0 --- /dev/null +++ b/test-repeater-liveness.js @@ -0,0 +1,79 @@ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); + +// Minimal browser environment +const ctx = { + window: {}, + console, + document: { + readyState: 'complete', + documentElement: { getAttribute: () => null }, + getElementById: () => null, + createElement: () => ({ textContent: '' }), + head: { appendChild: () => {} } + }, + navigator: {}, + fetch: () => Promise.resolve({ json: () => Promise.resolve({}) }), + Date, +}; +vm.createContext(ctx); +vm.runInContext(fs.readFileSync('public/roles.js', 'utf8'), ctx); +// Mirror browser semantics: in a real browser window === globalThis, so bare +// NOTE: mirrors browser global; update if HEALTH_THRESHOLDS moves out of window scope in roles.js +ctx.HEALTH_THRESHOLDS = ctx.window.HEALTH_THRESHOLDS; + +// Run tests inside the VM context to preserve closures +const testCode = ` +let pass = 0, fail = 0; +function test(name, fn) { + try { fn(); pass++; console.log(' ok:', name); } + catch (e) { fail++; console.log('FAIL:', name, '—', e.message); } +} +function assert(cond, msg) { if (!cond) throw new Error(msg || 'assertion failed'); } + +const now = Date.now(); +const recentMs = now - 1000; // 1 second ago — always active +const staleMs = now - (window.HEALTH_THRESHOLDS.infraSilentMs + 1); // just past silent threshold + +// --- Repeater three-state --- +test('repeater + recent + relay > 0 → relaying', + () => assert(window.getNodeStatus('repeater', recentMs, 5) === 'relaying')); + +test('repeater + recent + relay == 0 → active (idle)', + () => assert(window.getNodeStatus('repeater', recentMs, 0) === 'active')); + +test('repeater + stale + relay > 0 → stale (stale beats relay)', + () => assert(window.getNodeStatus('repeater', staleMs, 99) === 'stale')); + +test('repeater + stale + relay == 0 → stale', + () => assert(window.getNodeStatus('repeater', staleMs, 0) === 'stale')); + +// --- Non-repeater roles unaffected --- +test('companion + recent + relay 0 → active', + () => assert(window.getNodeStatus('companion', recentMs, 0) === 'active')); + +test('companion + recent + relay > 0 → active (relay ignored)', + () => assert(window.getNodeStatus('companion', recentMs, 99) === 'active')); + +test('room + recent + relay 0 → active', + () => assert(window.getNodeStatus('room', recentMs, 0) === 'active')); + +test('sensor + recent + relay 0 → active', + () => assert(window.getNodeStatus('sensor', recentMs, 0) === 'active')); + +// --- Backward compatibility: omitting third arg --- +test('getNodeStatus(repeater, recent) with no relay arg → active (not relaying)', + () => assert(window.getNodeStatus('repeater', recentMs) === 'active')); + +test('getNodeStatus(companion, recent) with no relay arg → active', + () => assert(window.getNodeStatus('companion', recentMs) === 'active')); + +window.testResults = { pass, fail }; +`; + +vm.runInContext(testCode, ctx); + +const { pass, fail } = ctx.window.testResults; +console.log(`\n${pass} passed, ${fail} failed`); +if (fail > 0) process.exit(1);