diff --git a/cmd/meshd/main.go b/cmd/meshd/main.go index 7e0f0d9..c0b48ec 100644 --- a/cmd/meshd/main.go +++ b/cmd/meshd/main.go @@ -2000,15 +2000,20 @@ func parseJoinCommandArgs(args []string) (inviteURL string, ownerDID string, noS const peerListDaemonLookupTimeout = 750 * time.Millisecond +var ( + errPeerListDaemonUnavailable = errors.New("meshd daemon is not running or unavailable") + errPeerListDaemonMismatch = errors.New("meshd daemon is serving a different profile or network") + errPeerListSnapshotNotReady = errors.New("meshd peer snapshot is not ready") +) + type peerListCommandDependencies struct { - loadIdentity func(string) (*did.DID, error) - loadDaemonStatus func(context.Context, string) (*daemon.Status, error) + loadDaemonStatus func(context.Context, string) (*daemon.Status, error) + daemonLookupTimeout time.Duration } // cmdPeerList lists all peers in the current mesh network. func cmdPeerList(ctx context.Context, args []string, flagProfile string) error { return cmdPeerListWithDependencies(ctx, args, flagProfile, peerListCommandDependencies{ - loadIdentity: loadIdentity, loadDaemonStatus: loadPeerListDaemonStatus, }) } @@ -2027,161 +2032,33 @@ func cmdPeerListWithDependencies(ctx context.Context, args []string, flagProfile return fmt.Errorf("not in a network. Use 'meshd network join' first.") } - // The running daemon already owns a materialized, last-good view of the - // mesh. Prefer it before unlocking identity state or contacting the DWN. - // Legacy daemon responses and snapshots for another profile/network are - // deliberately ignored and fall through to the remote path below. - if deps.loadDaemonStatus != nil { - if status, statusErr := deps.loadDaemonStatus(ctx, daemon.DefaultSocketPath()); statusErr == nil { - if rows, warning, ok := peerListRowsFromDaemonStatus(ns, status); ok { - if warning != "" { - fmt.Fprintf(os.Stderr, "Warning: %s\n", warning) - } - printPeerListRows(ns.NetworkName, rows) - return nil - } - } + if deps.loadDaemonStatus == nil { + return fmt.Errorf("%w; run 'meshd up'", errPeerListDaemonUnavailable) } - - identityLoader := deps.loadIdentity - if identityLoader == nil { - identityLoader = loadIdentity + lookupTimeout := deps.daemonLookupTimeout + if lookupTimeout <= 0 { + lookupTimeout = peerListDaemonLookupTimeout } - identity, err := identityLoader(stateDir) - if err != nil { - return err + lookupCtx, cancel := context.WithTimeout(ctx, lookupTimeout) + defer cancel() + status, statusErr := deps.loadDaemonStatus(lookupCtx, daemon.DefaultSocketPath()) + if statusErr != nil { + return fmt.Errorf("%w: %w; run 'meshd up'", errPeerListDaemonUnavailable, statusErr) } - selfNodeDID := networkNodeDID(ns, identity.URI) - meta := resolveIdentityMetadata(flagProfile, identity.URI) - selfOwnerDID := networkOwnerDID(ns, firstNonEmpty(meta.OwnerDID, identity.URI)) - operationIdentity, err := loadDWNOperationIdentity(stateDir, meta, identity) + rows, warning, err := peerListRowsFromDaemonStatus(ns, status) if err != nil { return err } - signer := dwnSigner(operationIdentity) - agent := dwn.NewSimpleAgent(ns.AnchorEndpoint, signer) - api := dwn.NewDwnAPI(agent) - - readAuth, err := walletDWNAuthForOperation(stateDir, meta, dwn.InterfaceRecordsQuery, protocols.MeshProtocolURI, "", ns.NetworkRecordID, false) - if err != nil { - return err - } - // Determine protocol role for queries. Delegated sessions read as the - // owner (no role); member-associated nodes read as network/member. - queryRole := protocolRoleForAuth(readAuth, readProtocolRole(ns.AnchorDID, identity.URI, ns.MemberRecordID)) - delegateSession := delegateSessionForCLIBestEffort(ctx, stateDir, meta, ns, operationIdentity, readAuth) - encMgr := newEncryptionKeyManager(identity) - if resp, err := loadControlStateForCLI(ctx, ns, identity, operationIdentity, encMgr, readAuth, delegateSession); err == nil { - if refreshed, _, saveErr := refreshLocalMembershipMetadataFromMap(stateDir, ns, resp); saveErr == nil && refreshed != nil { - ns = refreshed - selfOwnerDID = networkOwnerDID(ns, firstNonEmpty(meta.OwnerDID, identity.URI)) - } - printPeerListRows(ns.NetworkName, peerListRowsFromMapResponse(ns, resp, selfNodeDID, selfOwnerDID)) - return nil - } - - // Query owner-provisioned node records (network/node). - records, status, err := api.Query(ctx, ns.AnchorDID, dwn.QueryParams{ - Filter: dwn.RecordsFilter{ - Protocol: protocols.MeshProtocolURI, - ProtocolPath: "network/node", - ContextID: ns.NetworkRecordID, - }, - DateSort: "createdAscending", - PermissionGrantID: readAuth.PermissionGrantID, - DelegatedGrant: readAuth.DelegatedGrant, - }, queryRole) - if err != nil { - return fmt.Errorf("querying peers: %w", err) - } - - if status.Code != 200 { - return fmt.Errorf("query failed: %d %s", status.Code, status.Detail) - } - - // Also query member-associated node records (network/member/node). - // Nested protocol queries must use the direct parent member context. - memberRecords, mStatus, mErr := api.Query(ctx, ns.AnchorDID, dwn.QueryParams{ - Filter: dwn.RecordsFilter{ - Protocol: protocols.MeshProtocolURI, - ProtocolPath: "network/member", - ContextID: ns.NetworkRecordID, - }, - DateSort: "createdAscending", - PermissionGrantID: readAuth.PermissionGrantID, - DelegatedGrant: readAuth.DelegatedGrant, - }, queryRole) - if mErr == nil && mStatus.Code == 200 { - for _, memberRecord := range memberRecords { - memberNodeRecords, mnStatus, mnErr := api.Query(ctx, ns.AnchorDID, dwn.QueryParams{ - Filter: dwn.RecordsFilter{ - Protocol: protocols.MeshProtocolURI, - ProtocolPath: "network/member/node", - ContextID: ns.NetworkRecordID + "/" + memberRecord.ID, - }, - DateSort: "createdAscending", - PermissionGrantID: readAuth.PermissionGrantID, - DelegatedGrant: readAuth.DelegatedGrant, - }, queryRole) - if mnErr == nil && mnStatus.Code == 200 { - records = append(records, memberNodeRecords...) - } - } - } - - if len(records) == 0 { - fmt.Println("No peers found.") - return nil - } - - var rows []peerListRow - for _, r := range records { - peerDID := r.Recipient - displayDID := peerDID - if displayDID == "" { - displayDID = "(unknown)" - } - device := peerListDevice(peerDID, selfNodeDID) - - var node struct { - MeshIP string `json:"meshIP"` - Label string `json:"label"` - MemberDID string `json:"memberDID"` - ExpiresAt string `json:"expiresAt"` - } - if err := r.Data().JSON(ctx, &node); err != nil { - // Data may not be inline (encrypted records need context key). - rows = append(rows, peerListRow{ - NodeDID: displayDID, - MeshIP: peerListMeshIP(ns.MeshCIDR, peerDID, ""), - Device: device, - Owner: peerListOwner(peerDID, "", selfNodeDID, selfOwnerDID), - Label: "(encrypted)", - Expires: "unknown", - Path: r.ProtocolPath, - }) - continue - } - rows = append(rows, peerListRow{ - NodeDID: displayDID, - MeshIP: peerListMeshIP(ns.MeshCIDR, peerDID, node.MeshIP), - Device: device, - Owner: peerListOwner(peerDID, node.MemberDID, selfNodeDID, selfOwnerDID), - Label: node.Label, - Expires: node.ExpiresAt, - Path: r.ProtocolPath, - }) + if warning != "" { + fmt.Fprintf(os.Stderr, "Warning: %s\n", warning) } printPeerListRows(ns.NetworkName, rows) - return nil } func loadPeerListDaemonStatus(ctx context.Context, socketPath string) (*daemon.Status, error) { - lookupCtx, cancel := context.WithTimeout(ctx, peerListDaemonLookupTimeout) - defer cancel() - return daemon.NewClient(socketPath).GetStatus(lookupCtx) + return daemon.NewClient(socketPath).GetStatus(ctx) } func loadControlStateForCLI(ctx context.Context, ns *state.NetworkState, identity *did.DID, signerIdentity *did.DID, encMgr *dwncrypto.EncryptionKeyManager, readAuth dwn.MessageAuth, delegateSession *mesh.DelegateSession) (*control.MapResponse, error) { @@ -2309,28 +2186,37 @@ func peerListRowsFromMapResponse(ns *state.NetworkState, resp *control.MapRespon return rows } -func peerListRowsFromDaemonStatus(ns *state.NetworkState, status *daemon.Status) ([]peerListRow, string, bool) { - if ns == nil || status == nil || !status.Running { - return nil, "", false +func peerListRowsFromDaemonStatus(ns *state.NetworkState, status *daemon.Status) ([]peerListRow, string, error) { + if status == nil || !status.Running { + return nil, "", errPeerListDaemonUnavailable } - // NodeDID was added to network.json after the first releases. Without it - // there is no identity-free way to prove that the daemon snapshot belongs - // to this profile, so legacy state must use the existing identity path. + // Never unlock identity state or query the DWN from this read path. The + // persisted node and network IDs are the complete local trust boundary for + // accepting the daemon snapshot. + if ns == nil { + return nil, "", fmt.Errorf("%w: local network context is missing", errPeerListDaemonMismatch) + } selfNodeDID := strings.TrimSpace(ns.NodeDID) if selfNodeDID == "" || strings.TrimSpace(ns.NetworkRecordID) == "" { - return nil, "", false + return nil, "", fmt.Errorf("%w: local network context is incomplete", errPeerListDaemonMismatch) } - if status.NetworkRecordID != ns.NetworkRecordID || status.Self == nil || status.Self.NodeDID != selfNodeDID { - return nil, "", false + if status.NetworkRecordID != ns.NetworkRecordID { + return nil, "", fmt.Errorf("%w: daemon network %q does not match local network %q", errPeerListDaemonMismatch, status.NetworkRecordID, ns.NetworkRecordID) + } + if status.Self != nil && strings.TrimSpace(status.Self.NodeDID) != selfNodeDID { + return nil, "", fmt.Errorf("%w: daemon node %q does not match local node %q", errPeerListDaemonMismatch, status.Self.NodeDID, selfNodeDID) } snapshot := status.Snapshot - if snapshot == nil || snapshot.Generation == 0 || strings.TrimSpace(snapshot.RefreshedAt) == "" { - return nil, "", false + if status.Self == nil || snapshot == nil || snapshot.Generation == 0 || strings.TrimSpace(snapshot.RefreshedAt) == "" { + if snapshot != nil && strings.TrimSpace(snapshot.LastError) != "" { + return nil, "", fmt.Errorf("%w: %s", errPeerListSnapshotNotReady, strings.TrimSpace(snapshot.LastError)) + } + return nil, "", fmt.Errorf("%w; wait for synchronization and check 'meshd status'", errPeerListSnapshotNotReady) } if _, err := time.Parse(time.RFC3339Nano, snapshot.RefreshedAt); err != nil { - return nil, "", false + return nil, "", fmt.Errorf("%w: invalid refresh timestamp", errPeerListSnapshotNotReady) } selfOwnerDID := firstNonEmpty(status.OwnerDID, ns.EffectiveOwnerDID(selfNodeDID)) @@ -2365,7 +2251,7 @@ func peerListRowsFromDaemonStatus(ns *state.NetworkState, status *daemon.Status) }) } if len(rows) == 0 || rows[0].NodeDID != selfNodeDID { - return nil, "", false + return nil, "", fmt.Errorf("%w: snapshot does not begin with the local node", errPeerListDaemonMismatch) } warning := "" @@ -2376,7 +2262,7 @@ func peerListRowsFromDaemonStatus(ns *state.NetworkState, status *daemon.Status) lastError, ) } - return rows, warning, true + return rows, warning, nil } func printPeerListRows(networkName string, rows []peerListRow) { diff --git a/cmd/meshd/peer_list_local_test.go b/cmd/meshd/peer_list_local_test.go index a01b354..b1dfe8a 100644 --- a/cmd/meshd/peer_list_local_test.go +++ b/cmd/meshd/peer_list_local_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/enboxorg/meshd/internal/daemon" - "github.com/enboxorg/meshd/internal/did" "github.com/enboxorg/meshd/internal/engine" "github.com/enboxorg/meshd/internal/state" ) @@ -58,9 +57,9 @@ func TestPeerListRowsFromDaemonStatus(t *testing.T) { }, } - rows, warning, ok := peerListRowsFromDaemonStatus(ns, status) - if !ok { - t.Fatal("peerListRowsFromDaemonStatus rejected ready matching snapshot") + rows, warning, err := peerListRowsFromDaemonStatus(ns, status) + if err != nil { + t.Fatalf("peerListRowsFromDaemonStatus rejected ready matching snapshot: %v", err) } want := []peerListRow{ { @@ -111,17 +110,20 @@ func TestPeerListRowsFromDaemonStatusRejectsUntrustedOrUnreadySnapshot(t *testin } tests := []struct { - name string - mutate func(*state.NetworkState, *daemon.Status) (*state.NetworkState, *daemon.Status) + name string + wantErr error + mutate func(*state.NetworkState, *daemon.Status) (*state.NetworkState, *daemon.Status) }{ { - name: "absent daemon status", + name: "absent daemon status", + wantErr: errPeerListDaemonUnavailable, mutate: func(ns *state.NetworkState, _ *daemon.Status) (*state.NetworkState, *daemon.Status) { return ns, nil }, }, { - name: "old daemon response", + name: "old daemon response", + wantErr: errPeerListSnapshotNotReady, mutate: func(ns *state.NetworkState, status *daemon.Status) (*state.NetworkState, *daemon.Status) { status.Self = nil status.Snapshot = nil @@ -129,49 +131,56 @@ func TestPeerListRowsFromDaemonStatusRejectsUntrustedOrUnreadySnapshot(t *testin }, }, { - name: "legacy state without node DID", + name: "legacy state without node DID", + wantErr: errPeerListDaemonMismatch, mutate: func(ns *state.NetworkState, status *daemon.Status) (*state.NetworkState, *daemon.Status) { ns.NodeDID = "" return ns, status }, }, { - name: "network mismatch", + name: "network mismatch", + wantErr: errPeerListDaemonMismatch, mutate: func(ns *state.NetworkState, status *daemon.Status) (*state.NetworkState, *daemon.Status) { status.NetworkRecordID = "other-network" return ns, status }, }, { - name: "self mismatch", + name: "self mismatch", + wantErr: errPeerListDaemonMismatch, mutate: func(ns *state.NetworkState, status *daemon.Status) (*state.NetworkState, *daemon.Status) { status.Self.NodeDID = "did:jwk:other-profile" return ns, status }, }, { - name: "zero generation", + name: "zero generation", + wantErr: errPeerListSnapshotNotReady, mutate: func(ns *state.NetworkState, status *daemon.Status) (*state.NetworkState, *daemon.Status) { status.Snapshot.Generation = 0 return ns, status }, }, { - name: "missing refresh time", + name: "missing refresh time", + wantErr: errPeerListSnapshotNotReady, mutate: func(ns *state.NetworkState, status *daemon.Status) (*state.NetworkState, *daemon.Status) { status.Snapshot.RefreshedAt = "" return ns, status }, }, { - name: "malformed refresh time", + name: "malformed refresh time", + wantErr: errPeerListSnapshotNotReady, mutate: func(ns *state.NetworkState, status *daemon.Status) (*state.NetworkState, *daemon.Status) { status.Snapshot.RefreshedAt = "not-a-time" return ns, status }, }, { - name: "not running", + name: "not running", + wantErr: errPeerListDaemonUnavailable, mutate: func(ns *state.NetworkState, status *daemon.Status) (*state.NetworkState, *daemon.Status) { status.Running = false return ns, status @@ -182,8 +191,9 @@ func TestPeerListRowsFromDaemonStatusRejectsUntrustedOrUnreadySnapshot(t *testin for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ns, status := tc.mutate(readyState(), readyStatus()) - if rows, warning, ok := peerListRowsFromDaemonStatus(ns, status); ok || rows != nil || warning != "" { - t.Fatalf("result = (%+v, %q, %v), want remote fallback", rows, warning, ok) + rows, warning, err := peerListRowsFromDaemonStatus(ns, status) + if !errors.Is(err, tc.wantErr) || rows != nil || warning != "" { + t.Fatalf("result = (%+v, %q, %v), want local error %v", rows, warning, err, tc.wantErr) } }) } @@ -355,7 +365,7 @@ func TestDaemonRefreshState(t *testing.T) { } } -func TestCmdPeerListUsesDaemonSnapshotBeforeIdentity(t *testing.T) { +func TestCmdPeerListUsesOnlyDaemonSnapshot(t *testing.T) { stateDir := t.TempDir() t.Setenv("MESHD_STATE_DIR", stateDir) ns := &state.NetworkState{ @@ -369,14 +379,9 @@ func TestCmdPeerListUsesDaemonSnapshotBeforeIdentity(t *testing.T) { t.Fatalf("SaveNetworkState: %v", err) } - var identityLoads atomic.Int32 var statusLoads atomic.Int32 output, err := captureStdout(t, func() error { return cmdPeerListWithDependencies(context.Background(), nil, "", peerListCommandDependencies{ - loadIdentity: func(string) (*did.DID, error) { - identityLoads.Add(1) - return nil, errors.New("identity must not be loaded") - }, loadDaemonStatus: func(context.Context, string) (*daemon.Status, error) { statusLoads.Add(1) return &daemon.Status{ @@ -404,9 +409,6 @@ func TestCmdPeerListUsesDaemonSnapshotBeforeIdentity(t *testing.T) { if err != nil { t.Fatalf("cmdPeerListWithDependencies: %v", err) } - if got := identityLoads.Load(); got != 0 { - t.Fatalf("identity loads = %d, want zero", got) - } if got := statusLoads.Load(); got != 1 { t.Fatalf("daemon status loads = %d, want one", got) } @@ -417,7 +419,7 @@ func TestCmdPeerListUsesDaemonSnapshotBeforeIdentity(t *testing.T) { } } -func TestCmdPeerListFallsBackToIdentityForMismatchedSnapshot(t *testing.T) { +func TestCmdPeerListRejectsMismatchedDaemonWithoutRemoteFallback(t *testing.T) { stateDir := t.TempDir() t.Setenv("MESHD_STATE_DIR", stateDir) if err := state.SaveNetworkState(stateDir, &state.NetworkState{ @@ -428,14 +430,10 @@ func TestCmdPeerListFallsBackToIdentityForMismatchedSnapshot(t *testing.T) { t.Fatalf("SaveNetworkState: %v", err) } - wantErr := errors.New("identity fallback reached") - var identityLoads atomic.Int32 + var statusLoads atomic.Int32 err := cmdPeerListWithDependencies(context.Background(), nil, "", peerListCommandDependencies{ - loadIdentity: func(string) (*did.DID, error) { - identityLoads.Add(1) - return nil, wantErr - }, loadDaemonStatus: func(context.Context, string) (*daemon.Status, error) { + statusLoads.Add(1) return &daemon.Status{ Running: true, NetworkRecordID: "other-network", @@ -447,10 +445,86 @@ func TestCmdPeerListFallsBackToIdentityForMismatchedSnapshot(t *testing.T) { }, nil }, }) - if !errors.Is(err, wantErr) { - t.Fatalf("cmdPeerListWithDependencies error = %v, want identity fallback", err) + if !errors.Is(err, errPeerListDaemonMismatch) { + t.Fatalf("cmdPeerListWithDependencies error = %v, want profile/network mismatch", err) + } + if got := statusLoads.Load(); got != 1 { + t.Fatalf("daemon status loads = %d, want one", got) + } +} + +func TestCmdPeerListReturnsLocalSnapshotNotReady(t *testing.T) { + stateDir := t.TempDir() + t.Setenv("MESHD_STATE_DIR", stateDir) + if err := state.SaveNetworkState(stateDir, &state.NetworkState{ + NetworkRecordID: "network-1", + NetworkName: "home", + NodeDID: "did:jwk:self", + }); err != nil { + t.Fatalf("SaveNetworkState: %v", err) + } + + err := cmdPeerListWithDependencies(context.Background(), nil, "", peerListCommandDependencies{ + loadDaemonStatus: func(context.Context, string) (*daemon.Status, error) { + return &daemon.Status{ + Running: true, + NetworkRecordID: "network-1", + Self: &daemon.PeerStatus{NodeDID: "did:jwk:self"}, + Snapshot: &daemon.SnapshotStatus{LastError: "initial synchronization is rate limited"}, + }, nil + }, + }) + if !errors.Is(err, errPeerListSnapshotNotReady) || !strings.Contains(err.Error(), "rate limited") { + t.Fatalf("cmdPeerListWithDependencies error = %v, want local not-ready detail", err) + } +} + +func TestCmdPeerListReturnsDaemonUnavailableWithoutRemoteFallback(t *testing.T) { + stateDir := t.TempDir() + t.Setenv("MESHD_STATE_DIR", stateDir) + if err := state.SaveNetworkState(stateDir, &state.NetworkState{ + NetworkRecordID: "network-1", + NetworkName: "home", + NodeDID: "did:jwk:self", + }); err != nil { + t.Fatalf("SaveNetworkState: %v", err) + } + + wantErr := errors.New("socket unavailable") + err := cmdPeerListWithDependencies(context.Background(), nil, "", peerListCommandDependencies{ + loadDaemonStatus: func(context.Context, string) (*daemon.Status, error) { + return nil, wantErr + }, + }) + if !errors.Is(err, errPeerListDaemonUnavailable) || !errors.Is(err, wantErr) || !strings.Contains(err.Error(), "meshd up") { + t.Fatalf("cmdPeerListWithDependencies error = %v, want unavailable error and start hint", err) + } +} + +func TestCmdPeerListDaemonLookupIsBounded(t *testing.T) { + stateDir := t.TempDir() + t.Setenv("MESHD_STATE_DIR", stateDir) + if err := state.SaveNetworkState(stateDir, &state.NetworkState{ + NetworkRecordID: "network-1", + NetworkName: "home", + NodeDID: "did:jwk:self", + }); err != nil { + t.Fatalf("SaveNetworkState: %v", err) + } + + started := time.Now() + err := cmdPeerListWithDependencies(context.Background(), nil, "", peerListCommandDependencies{ + daemonLookupTimeout: 20 * time.Millisecond, + loadDaemonStatus: func(ctx context.Context, _ string) (*daemon.Status, error) { + <-ctx.Done() + return nil, ctx.Err() + }, + }) + elapsed := time.Since(started) + if !errors.Is(err, errPeerListDaemonUnavailable) || !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("cmdPeerListWithDependencies error = %v, want bounded unavailable deadline", err) } - if got := identityLoads.Load(); got != 1 { - t.Fatalf("identity loads = %d, want one", got) + if elapsed > 500*time.Millisecond { + t.Fatalf("daemon lookup took %s, want bounded local failure", elapsed) } } diff --git a/docs/topology-materializer.md b/docs/topology-materializer.md new file mode 100644 index 0000000..8c5c1dc --- /dev/null +++ b/docs/topology-materializer.md @@ -0,0 +1,96 @@ +# Topology materialization + +`meshd` treats the daemon's in-memory network map as the local read model. +Peer-list, tray, and status requests read that model over the daemon socket; +they never query the DWN in the request path. If the daemon is unavailable or +its first snapshot is not ready, the CLI reports that state immediately instead +of starting a second control-plane load. + +The control-plane update path has two modes: + +1. A full reconciliation establishes an authoritative raw-record baseline. +2. A live DWN subscription stages ordered record changes and applies them to a + clone of that baseline. + +The incremental path is transactional. It deep-copies an event before the +subscription acknowledges it, applies DWN write/delete ordering and squash +semantics, hydrates only a winning write whose payload was not inlined, builds +the parsed mesh response, converts it to a meshnet network map, and commits the +raw and parsed states together. A failed conversion leaves the last-good map +and staged prefix unchanged for retry. A failed projection also preserves the +last-good map, but invalidates the baseline so the next retry repairs it with a +full reconciliation. + +## Full reconciliation triggers + +A full load is the repair path, not the normal event path. It is required at +startup, for delivery-key changes, after a subscription cursor gap or terminal +failure, when the bounded event queue overflows, when an event is ambiguous or +invalid, or when no complete baseline exists. A slow periodic full load remains +as anti-entropy protection against missed server events. Rate limits and +transport failures retain pending work and honor coordinator backoff before a +repair is attempted. + +The full-load cut is captured before remote reads begin. Events staged after +that cut survive baseline installation, so a snapshot cannot overwrite a +concurrent update. A newer repair marker aborts publication because the load +cannot prove that it covered the missing tail. Query entries with out-of-line +payloads are completed with a targeted authenticated `RecordsRead`; inline +entries require no extra request. + +Every topology query is cursor-paginated. Parent-scoped member and node-child +queries run through a bounded worker pool and are assembled deterministically. +A reconciliation is bounded to 64 pages per logical query, 25,000 requests, +10,000 records, and 64 MiB of retained record/cursor data. Exceeding any bound, +repeating a cursor, or receiving an incomplete response fails the transaction +and leaves the last-good map installed. + +## Bounds and recovery + +The staged queue is bounded by both event count and retained bytes, and inbound +WebSocket messages have a separate hard size limit. DWN HTTP responses are +also rejected above 128 MiB before JSON or binary payload decoding. Overflow +or poison data is +acknowledged to avoid a reconnect loop, but invalidates the incremental +baseline and schedules a full repair. The raw baseline is replaced by each +anti-entropy load, which also bounds tombstone lifetime. + +Parsed contributions are cached by canonical record identity. Successful +outcomes are immutable for that CID and are reused across unrelated updates and +anti-entropy runs. Opaque/key-unavailable outcomes are retried after key-cache +invalidation while their same logical slot can continue contributing a +last-good member, node, node-info, endpoint, relay, or ACL. A cold unreadable +ACL or configured relay fails closed; an authoritative empty set remains a +valid deletion. Descriptor-only node ghosts never receive fallback addresses or +keys: opaque peers stay out of the engine map, while opaque self fails unless a +recipient-matched typed last-good membership exists. Missing or deleted self +membership commits an epoch-expired, zero-peer down map so meshnet tears down +WireGuard while subscriptions remain available for renewal. + +The local materializer follows the DWN base-state lattice: + +- writes are ordered by `(messageTimestamp, messageCID)`; +- only a strictly newer delete can transition a live write to a tombstone; +- a plain tombstone can only become a strictly newer prune, and prune is + terminal; +- squash records replace strictly older records in the same protocol path and + parent context while retaining equal or newer records. + +The nearest future peer-membership expiry has its own lifecycle-owned timer. It +reprojects the raw baseline locally, without a DWN request, so expired peers are +removed from the engine map at the deadline. Self expiry remains owned by +meshnet's generation-fenced key-expiry timer, which fails the engine closed. + +Cold starts deliberately rebuild from the authoritative DWN instead of loading +a persistent disk cache. Persisting encrypted raw records would add cache +versioning and stale-secret handling without reducing steady-state requests; +the subscription-backed in-memory baseline removes those requests while the +daemon is running. + +Delivery records addressed to this node have their own live subscription. +Audience records and encryption-protocol grant-key records are not covered by +the network-context subscription today. A new audience key referenced by a +topology record is resolved on demand, and independent audience changes are +retried by anti-entropy. Grant-key sets are currently loaded with the delegate +session; independent grant-key changes require a session reload until a +dedicated subscription and atomic key-set replacement are implemented. diff --git a/internal/control/audiencekeycache.go b/internal/control/audiencekeycache.go index 7dbbd45..291a1e8 100644 --- a/internal/control/audiencekeycache.go +++ b/internal/control/audiencekeycache.go @@ -5,6 +5,21 @@ import ( "errors" "fmt" "sync" + "time" + + "github.com/enboxorg/meshd/internal/dwn" +) + +const ( + defaultAudienceKeyFailureTTL = 30 * time.Second + maxAudienceKeyFailures = 256 +) + +var ( + errAudienceKeyDeliveryAbsent = errors.New("audience key delivery absent") + errAudienceRecordAbsent = errors.New("audience record absent") + errAudienceSealUnavailable = errors.New("audience seal route unavailable") + errAudienceDeliveryUnavailable = errors.New("audience delivery route unavailable") ) // audienceKeyCacheKey identifies one role-audience key. A DWNClient is bound @@ -17,19 +32,29 @@ type audienceKeyCacheKey struct { } type audienceKeyCall struct { - done chan struct{} - err error + done chan struct{} + err error + failureGeneration uint64 +} + +type audienceKeyFailure struct { + err error + expiresAt time.Time } // audienceKeyCache retains successfully delivered role-audience private keys // for the lifetime of a DWNClient and coalesces concurrent misses. Key IDs are // public-key thumbprints, so audience rotation naturally selects a new entry. -// Failed lookups are deliberately not cached: a delivery can arrive later, or -// a transient DWN error can recover on the next map refresh. +// Only authoritative delivery absence is cached briefly; transient failures +// are never cached, and full/delivery invalidation clears absence entries. type audienceKeyCache struct { - mu sync.Mutex - keys map[audienceKeyCacheKey][]byte - inflight map[audienceKeyCacheKey]*audienceKeyCall + mu sync.Mutex + keys map[audienceKeyCacheKey][]byte + inflight map[audienceKeyCacheKey]*audienceKeyCall + failures map[audienceKeyCacheKey]audienceKeyFailure + failureGeneration uint64 + failureTTL time.Duration + now func() time.Time } // get takes ownership of the buffer returned by load and zeroes it after @@ -46,7 +71,16 @@ func (c *audienceKeyCache) get( c.mu.Unlock() return result, nil } - if call, ok := c.inflight[key]; ok { + now := c.timeNowLocked() + if failure, ok := c.failures[key]; ok { + if now.Before(failure.expiresAt) { + c.mu.Unlock() + return nil, failure.err + } + delete(c.failures, key) + } + generation := c.failureGeneration + if call, ok := c.inflight[key]; ok && call.failureGeneration == generation { done := call.done c.mu.Unlock() select { @@ -70,7 +104,7 @@ func (c *audienceKeyCache) get( if c.inflight == nil { c.inflight = make(map[audienceKeyCacheKey]*audienceKeyCall) } - call := &audienceKeyCall{done: make(chan struct{})} + call := &audienceKeyCall{done: make(chan struct{}), failureGeneration: generation} c.inflight[key] = call c.mu.Unlock() @@ -85,9 +119,16 @@ func (c *audienceKeyCache) get( c.keys = make(map[audienceKeyCacheKey][]byte) } c.keys[key] = append([]byte(nil), privateKey...) + delete(c.failures, key) + } else if call.failureGeneration == c.failureGeneration && stableAudienceKeyFailure(err) { + if _, alreadyAvailable := c.keys[key]; !alreadyAvailable { + c.cacheFailureLocked(key, err) + } } call.err = err - delete(c.inflight, key) + if c.inflight[key] == call { + delete(c.inflight, key) + } close(call.done) c.mu.Unlock() @@ -100,3 +141,64 @@ func (c *audienceKeyCache) get( return result, nil } } + +func (c *audienceKeyCache) invalidateFailures() { + c.mu.Lock() + c.failureGeneration++ + c.failures = nil + c.mu.Unlock() +} + +func (c *audienceKeyCache) timeNowLocked() time.Time { + if c.now != nil { + return c.now() + } + return time.Now() +} + +func (c *audienceKeyCache) cacheFailureLocked(key audienceKeyCacheKey, err error) { + now := c.timeNowLocked() + ttl := c.failureTTL + if ttl <= 0 { + ttl = defaultAudienceKeyFailureTTL + } + if c.failures == nil { + c.failures = make(map[audienceKeyCacheKey]audienceKeyFailure) + } + for cachedKey, failure := range c.failures { + if !now.Before(failure.expiresAt) { + delete(c.failures, cachedKey) + } + } + if len(c.failures) >= maxAudienceKeyFailures { + var oldestKey audienceKeyCacheKey + var oldest time.Time + for cachedKey, failure := range c.failures { + if oldest.IsZero() || failure.expiresAt.Before(oldest) || + (failure.expiresAt.Equal(oldest) && audienceKeyCacheKeyLess(cachedKey, oldestKey)) { + oldestKey = cachedKey + oldest = failure.expiresAt + } + } + delete(c.failures, oldestKey) + } + c.failures[key] = audienceKeyFailure{err: err, expiresAt: now.Add(ttl)} +} + +func audienceKeyCacheKeyLess(a, b audienceKeyCacheKey) bool { + if a.protocol != b.protocol { + return a.protocol < b.protocol + } + if a.rolePath != b.rolePath { + return a.rolePath < b.rolePath + } + return a.keyID < b.keyID +} + +func stableAudienceKeyFailure(err error) bool { + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, dwn.ErrRateLimited) || errors.Is(err, dwn.ErrTransport) { + return false + } + return errors.Is(err, errAudienceKeyDeliveryAbsent) +} diff --git a/internal/control/audiencekeycache_test.go b/internal/control/audiencekeycache_test.go index dc195bb..1e8c807 100644 --- a/internal/control/audiencekeycache_test.go +++ b/internal/control/audiencekeycache_test.go @@ -9,6 +9,9 @@ import ( "sync/atomic" "testing" "time" + + "github.com/enboxorg/meshd/internal/dwn" + dwncrypto "github.com/enboxorg/meshd/internal/dwn/crypto" ) func TestAudienceKeyCacheReusesSuccessfulLookupAndCopiesKey(t *testing.T) { @@ -88,6 +91,130 @@ func TestAudienceKeyCacheDoesNotCacheFailedLookup(t *testing.T) { } } +func TestAudienceKeyCacheCachesOnlyStableAbsenceUntilTTL(t *testing.T) { + now := time.Date(2026, 7, 11, 12, 0, 0, 0, time.UTC) + cache := audienceKeyCache{failureTTL: time.Minute, now: func() time.Time { return now }} + key := audienceKeyCacheKey{protocol: "p", rolePath: "r", keyID: "missing"} + wantErr := fmt.Errorf("%w: no delivery record", errAudienceKeyDeliveryAbsent) + var loads atomic.Int32 + loader := func(context.Context) ([]byte, error) { + loads.Add(1) + return nil, wantErr + } + for range 2 { + if _, err := cache.get(context.Background(), key, loader); !errors.Is(err, errAudienceKeyDeliveryAbsent) { + t.Fatalf("stable miss error = %v", err) + } + } + if got := loads.Load(); got != 1 { + t.Fatalf("stable miss loader calls = %d, want 1", got) + } + now = now.Add(time.Minute) + if _, err := cache.get(context.Background(), key, loader); !errors.Is(err, errAudienceKeyDeliveryAbsent) { + t.Fatalf("expired stable miss error = %v", err) + } + if got := loads.Load(); got != 2 { + t.Fatalf("expired stable miss loader calls = %d, want 2", got) + } +} + +func TestAudienceKeyCacheInvalidationRetainsSuccessAndNeverCachesTransient(t *testing.T) { + cache := audienceKeyCache{} + successKey := audienceKeyCacheKey{protocol: "p", rolePath: "r", keyID: "success"} + if _, err := cache.get(context.Background(), successKey, func(context.Context) ([]byte, error) { + return []byte{1, 2, 3}, nil + }); err != nil { + t.Fatal(err) + } + missingKey := audienceKeyCacheKey{protocol: "p", rolePath: "r", keyID: "missing"} + if _, err := cache.get(context.Background(), missingKey, func(context.Context) ([]byte, error) { + return nil, fmt.Errorf("%w: absent", errAudienceKeyDeliveryAbsent) + }); !errors.Is(err, errAudienceKeyDeliveryAbsent) { + t.Fatalf("stable failure = %v", err) + } + cache.invalidateFailures() + if _, err := cache.get(context.Background(), successKey, func(context.Context) ([]byte, error) { + return nil, errors.New("successful key was invalidated") + }); err != nil { + t.Fatalf("retained success: %v", err) + } + var retryLoads atomic.Int32 + if _, err := cache.get(context.Background(), missingKey, func(context.Context) ([]byte, error) { + retryLoads.Add(1) + return []byte{4, 5, 6}, nil + }); err != nil { + t.Fatalf("invalidated absence retry: %v", err) + } + if retryLoads.Load() != 1 { + t.Fatalf("invalidated absence loads = %d", retryLoads.Load()) + } + + for _, transient := range []error{context.Canceled, context.DeadlineExceeded, dwn.ErrRateLimited, dwn.ErrTransport} { + key := audienceKeyCacheKey{protocol: "p", rolePath: "r", keyID: transient.Error()} + var loads atomic.Int32 + for range 2 { + if _, err := cache.get(context.Background(), key, func(context.Context) ([]byte, error) { + loads.Add(1) + return nil, errors.Join(fmt.Errorf("%w: absent", errAudienceKeyDeliveryAbsent), transient) + }); !errors.Is(err, transient) { + t.Fatalf("transient %v error = %v", transient, err) + } + } + if loads.Load() != 2 { + t.Fatalf("transient %v was cached; loads=%d", transient, loads.Load()) + } + } +} + +func TestRoleAudiencePrivateKeyCachesStableUnavailableUntilTTLAndInvalidation(t *testing.T) { + now := time.Date(2026, 7, 11, 12, 0, 0, 0, time.UTC) + client := newMaterializerTestClient() + client.roleAudienceKeys.failureTTL = time.Minute + client.roleAudienceKeys.now = func() time.Time { return now } + info := &dwncrypto.RoleAudienceInfo{Protocol: "p", RolePath: "network/node", KeyID: "missing"} + key := audienceKeyCacheKey{protocol: info.Protocol, rolePath: info.RolePath, keyID: info.KeyID} + + if _, err := client.roleAudiencePrivateKey(context.Background(), info); !errors.Is(err, errAudienceKeyDeliveryAbsent) { + t.Fatalf("first stable miss = %v", err) + } + client.roleAudienceKeys.mu.Lock() + firstExpiry := client.roleAudienceKeys.failures[key].expiresAt + client.roleAudienceKeys.mu.Unlock() + now = now.Add(30 * time.Second) + if _, err := client.roleAudiencePrivateKey(context.Background(), info); !errors.Is(err, errAudienceKeyDeliveryAbsent) { + t.Fatalf("cached stable miss = %v", err) + } + client.roleAudienceKeys.mu.Lock() + cachedExpiry := client.roleAudienceKeys.failures[key].expiresAt + client.roleAudienceKeys.mu.Unlock() + if !cachedExpiry.Equal(firstExpiry) { + t.Fatalf("cached miss re-ran routes: expiry %v -> %v", firstExpiry, cachedExpiry) + } + + now = firstExpiry + if _, err := client.roleAudiencePrivateKey(context.Background(), info); !errors.Is(err, errAudienceKeyDeliveryAbsent) { + t.Fatalf("expired stable miss = %v", err) + } + client.roleAudienceKeys.mu.Lock() + expiredRetry := client.roleAudienceKeys.failures[key].expiresAt + client.roleAudienceKeys.mu.Unlock() + if !expiredRetry.After(firstExpiry) { + t.Fatalf("TTL expiry did not retry routes: %v <= %v", expiredRetry, firstExpiry) + } + + client.roleAudienceKeys.invalidateFailures() + now = now.Add(time.Second) + if _, err := client.roleAudiencePrivateKey(context.Background(), info); !errors.Is(err, errAudienceKeyDeliveryAbsent) { + t.Fatalf("invalidated stable miss = %v", err) + } + client.roleAudienceKeys.mu.Lock() + invalidatedRetry := client.roleAudienceKeys.failures[key].expiresAt + client.roleAudienceKeys.mu.Unlock() + if !invalidatedRetry.After(expiredRetry) { + t.Fatalf("invalidation did not retry routes: %v <= %v", invalidatedRetry, expiredRetry) + } +} + func TestAudienceKeyCacheCoalescesConcurrentLookup(t *testing.T) { var cache audienceKeyCache key := audienceKeyCacheKey{protocol: "p", rolePath: "r", keyID: "k"} diff --git a/internal/control/audiencesource.go b/internal/control/audiencesource.go index 93a9e14..0cda40b 100644 --- a/internal/control/audiencesource.go +++ b/internal/control/audiencesource.go @@ -5,6 +5,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "fmt" "log/slog" "sort" @@ -130,7 +131,7 @@ func (s *SealedAudienceSource) Current(ctx context.Context, protocol, rolePath, // this reader can already derive seal keys for. func (s *SealedAudienceSource) AudiencePrivateKeyByKeyID(ctx context.Context, protocol, rolePath, keyID string) ([]byte, error) { if s.sealKeys == nil { - return nil, fmt.Errorf("no seal keys available") + return nil, fmt.Errorf("%w: no seal keys available", errAudienceSealUnavailable) } rec, err := s.audienceByKeyID(ctx, protocol, rolePath, keyID) if err != nil { @@ -209,7 +210,7 @@ func (s *SealedAudienceSource) audienceByKeyID(ctx context.Context, protocol, ro return nil, err } if len(records) == 0 { - return nil, fmt.Errorf("no audience record for keyId %s (%s %s)", keyID, protocol, rolePath) + return nil, fmt.Errorf("%w: no audience record for keyId %s (%s %s)", errAudienceRecordAbsent, keyID, protocol, rolePath) } return records[0], nil } @@ -227,6 +228,9 @@ func (s *SealedAudienceSource) queryAudience(ctx context.Context, filterTags map } entries, err := dwn.QueryEntries(reply) if err != nil { + if !errors.Is(err, dwn.ErrRateLimited) { + err = errors.Join(dwn.ErrTransport, err) + } return nil, fmt.Errorf("parsing audience query: %w", err) } diff --git a/internal/control/control_test.go b/internal/control/control_test.go index 28a4e15..6249e22 100644 --- a/internal/control/control_test.go +++ b/internal/control/control_test.go @@ -216,7 +216,7 @@ func TestBuildMapResponseSkipsExpiredPeer(t *testing.T) { } } -func TestBuildMapResponseReturnsNilWhenSelfExpired(t *testing.T) { +func TestBuildMapResponseEmitsDownMapWhenSelfExpired(t *testing.T) { selfDID, _ := testDIDJWK(t) peerDID, _ := testDIDJWK(t) c := NewDWNClient("https://dwn.example", "did:example:anchor", "network-1", selfDID, nil) @@ -229,8 +229,10 @@ func TestBuildMapResponseReturnsNilWhenSelfExpired(t *testing.T) { } c.nodes[peerDID] = &NodeRecord{DID: peerDID, MeshIP: "10.200.0.3", RecordID: "peer-record"} - if resp := c.buildMapResponse(); resp != nil { - t.Fatalf("buildMapResponse returned %#v, want nil", resp) + resp := c.buildMapResponse() + if resp == nil || resp.Node == nil || resp.Node.DID != selfDID || len(resp.Peers) != 0 || + resp.Node.ExpiresAt != c.nodes[selfDID].ExpiresAt { + t.Fatalf("expired self response = %#v, want expired self and zero peers", resp) } } @@ -243,7 +245,7 @@ func TestNodeRecordExpired(t *testing.T) { }{ "empty": {expiresAt: "", want: false}, "future": {expiresAt: now.Add(time.Minute).Format(time.RFC3339), want: false}, - "exact now": {expiresAt: now.Format(time.RFC3339), want: false}, + "exact now": {expiresAt: now.Format(time.RFC3339), want: true}, "past": {expiresAt: now.Add(-time.Minute).Format(time.RFC3339), want: true}, "malformed": {expiresAt: "not-a-time", want: false}, } diff --git a/internal/control/data.go b/internal/control/data.go index 4906561..d031cb0 100644 --- a/internal/control/data.go +++ b/internal/control/data.go @@ -38,6 +38,13 @@ type NodeRecord struct { Endpoints []EndpointData `json:"-"` RecordID string `json:"-"` + // Opaque marks a descriptor-only ghost whose encrypted payload was not + // readable. Revoked marks the synthetic, expired self node emitted when a + // complete authoritative projection contains no self membership record. + // Neither marker is serialized into DWN record data. + Opaque bool `json:"-"` + Revoked bool `json:"-"` + // MemberRecordID is the parent member record ID, if this node is // under a member (network/member/node path). Empty for owner-provisioned // top-level nodes (network/node path). diff --git a/internal/control/delta_queue.go b/internal/control/delta_queue.go new file mode 100644 index 0000000..ea2491d --- /dev/null +++ b/internal/control/delta_queue.go @@ -0,0 +1,538 @@ +package control + +import ( + "context" + "errors" + "fmt" + + "github.com/enboxorg/meshd/internal/dwn" + dwncrypto "github.com/enboxorg/meshd/internal/dwn/crypto" + "github.com/enboxorg/meshd/protocols" +) + +const ( + maxPendingTopologyEvents = 4096 + maxPendingTopologyBytes = 8 << 20 + pendingTopologyFixedBytes = 64 +) + +// ErrFullReconciliationRequired means the local raw cache cannot prove that +// it has a complete topology history and must be replaced by a full snapshot. +var ErrFullReconciliationRequired = errors.New("full reconciliation required") + +// PendingStateValidator validates a fully projected local state before its +// raw/parsed state and captured event prefix are committed. The callback runs +// while the client's load transaction is serialized and must not call back +// into methods that acquire loadMu. +type PendingStateValidator func(*MapResponse) error + +type pendingTopologyEvent struct { + sequence uint64 + message *dwn.SubscriptionMessage + retainedBytes int +} + +// beginFullReconciliation captures the topology sequence immediately before a +// remote snapshot begins. installRawBaseline uses it as a cut: events staged +// after this sequence survive the baseline installation. +func (c *DWNClient) beginFullReconciliation() uint64 { + c.deltaMu.Lock() + defer c.deltaMu.Unlock() + return c.topologySequence +} + +// installRawBaseline installs an independently owned complete raw snapshot and +// removes only events and repair markers covered by that snapshot. The caller +// serializes complete loads through loadMu. +func (c *DWNClient) installRawBaseline(set *rawMeshRecordSet, through uint64) { + c.deltaMu.Lock() + defer c.deltaMu.Unlock() + if set == nil { + c.rawBaseline = nil + c.clearPendingTopologyLocked() + c.markFullReconciliationLocked() + return + } + + c.rawBaseline = set.clone() + c.trimPendingTopologyPrefixLocked(through) + if c.fullReconciliation && c.repairSequence <= through { + c.fullReconciliation = false + c.repairSequence = 0 + } +} + +// completeFullReconciliation atomically publishes one validated full snapshot. +// A repair marker newer than the fetch cut invalidates the candidate, while +// ordinary post-cut events remain queued for the next incremental refresh. +func (c *DWNClient) completeFullReconciliation( + ctx context.Context, + candidate *rawMeshRecordSet, + projection *rawMeshMaterialization, + through uint64, +) error { + if candidate == nil || projection == nil { + return fmt.Errorf("completing full reconciliation: nil candidate or projection") + } + installed := candidate.clone() + c.deltaMu.Lock() + if err := ctx.Err(); err != nil { + c.deltaMu.Unlock() + return errors.Join(ErrFullReconciliationRequired, err) + } + if c.fullReconciliation && c.repairSequence > through { + c.deltaMu.Unlock() + return ErrFullReconciliationRequired + } + + c.mu.Lock() + if err := ctx.Err(); err != nil { + c.mu.Unlock() + c.deltaMu.Unlock() + return errors.Join(ErrFullReconciliationRequired, err) + } + c.commitRawMeshMaterializationLocked(projection) + c.rawBaseline = installed + c.trimPendingTopologyPrefixLocked(through) + if c.fullReconciliation && c.repairSequence <= through { + c.fullReconciliation = false + c.repairSequence = 0 + } + c.mu.Unlock() + c.deltaMu.Unlock() + c.commitRawMeshMaterializationCounters(projection) + return nil +} + +// StageTopologyEvent retains a deep copy of one map-affecting event. Poison or +// ambiguous frames are ACKed but mark the cache for repair so they cannot cause +// an infinite reconnect loop. The bounded queue remains safe under a stalled +// consumer. +func (c *DWNClient) StageTopologyEvent(message *dwn.SubscriptionMessage) error { + if rawTopologyEventAffectsDecryptContext(message) { + c.invalidateRawParsedOutcomes() + c.deliveredAudienceKeys.invalidateFailures() + c.roleAudienceKeys.invalidateFailures() + c.RequireFullReconciliation() + return nil + } + relevant, certain := rawTopologyEventAffectsMap(message) + if !certain { + c.deltaMu.Lock() + c.clearPendingTopologyLocked() + c.markFullReconciliationLocked() + c.deltaMu.Unlock() + return nil + } + if !relevant { + return nil + } + eventBytes := pendingTopologyMessageBytes(message) + if eventBytes > maxPendingTopologyBytes { + c.deltaMu.Lock() + c.clearPendingTopologyLocked() + c.markFullReconciliationLocked() + c.deltaMu.Unlock() + return nil + } + cloned, err := clonePendingTopologyMessage(message) + if err != nil { + c.deltaMu.Lock() + c.clearPendingTopologyLocked() + c.markFullReconciliationLocked() + c.deltaMu.Unlock() + return nil + } + + c.deltaMu.Lock() + defer c.deltaMu.Unlock() + // Account the retained clone, not the caller-owned message. Rechecking also + // keeps the bound exact if a caller mutated its message between sizing and + // cloning (concurrent mutation remains outside the API contract). + eventBytes = pendingTopologyMessageBytes(cloned) + if len(c.pendingTopology) >= maxPendingTopologyEvents || + eventBytes > maxPendingTopologyBytes || c.pendingTopologyBytes > maxPendingTopologyBytes-eventBytes { + c.clearPendingTopologyLocked() + c.markFullReconciliationLocked() + return nil + } + c.topologySequence++ + c.pendingTopology = append(c.pendingTopology, pendingTopologyEvent{ + sequence: c.topologySequence, + message: cloned, + retainedBytes: eventBytes, + }) + c.pendingTopologyBytes += eventBytes + return nil +} + +func rawTopologyEventAffectsDecryptContext(message *dwn.SubscriptionMessage) bool { + if message == nil || message.Type != dwn.SubscriptionEventType || message.Event == nil || len(message.Event.Message) == 0 { + return false + } + method, err := classifyRawRecordMessage(message.Event.Message) + if err != nil { + return false + } + var protocol, path string + if method == "Write" { + write, _, err := unwrapRawRecordMessage(message.Event.Message, "recordsWrite") + if err != nil || write.Descriptor == nil { + return false + } + protocol, path = write.Descriptor.Protocol, write.Descriptor.ProtocolPath + } else { + if len(message.Event.InitialWrite) == 0 { + return false + } + initial, _, err := unwrapRawRecordMessage(message.Event.InitialWrite, "recordsWrite") + if err != nil || initial.Descriptor == nil { + return false + } + protocol, path = initial.Descriptor.Protocol, initial.Descriptor.ProtocolPath + } + return (protocol == protocols.MeshProtocolURI && + (path == dwncrypto.EncryptionControlAudiencePath || path == dwncrypto.EncryptionControlDeliveryPath)) || + (protocol == dwncrypto.EncryptionProtocolURI && path == dwncrypto.GrantKeyProtocolPath) +} + +// RequireFullReconciliation invalidates delta continuity after a subscription +// gap or another condition that a local replay cannot repair. +func (c *DWNClient) RequireFullReconciliation() { + c.deltaMu.Lock() + c.clearPendingTopologyLocked() + c.markFullReconciliationLocked() + c.deltaMu.Unlock() +} + +func (c *DWNClient) markFullReconciliationLocked() { + c.topologySequence++ + c.repairSequence = c.topologySequence + c.fullReconciliation = true +} + +// beginPendingTopology snapshots a baseline and deeply copied queue prefix. +// The returned sequence fences precisely which events a successful apply may +// remove while later concurrently staged events remain queued. +func (c *DWNClient) beginPendingTopology() (*rawMeshRecordSet, []pendingTopologyEvent, uint64, error) { + c.deltaMu.Lock() + defer c.deltaMu.Unlock() + if c.fullReconciliation || c.rawBaseline == nil { + return nil, nil, 0, ErrFullReconciliationRequired + } + baseline := c.rawBaseline.clone() + events := make([]pendingTopologyEvent, len(c.pendingTopology)) + var through uint64 + for i, event := range c.pendingTopology { + message, err := clonePendingTopologyMessage(event.message) + if err != nil { + c.clearPendingTopologyLocked() + c.markFullReconciliationLocked() + return nil, nil, 0, ErrFullReconciliationRequired + } + events[i] = pendingTopologyEvent{ + sequence: event.sequence, + message: message, + retainedBytes: event.retainedBytes, + } + through = event.sequence + } + return baseline, events, through, nil +} + +func (c *DWNClient) completePendingTopology(ctx context.Context, candidate *rawMeshRecordSet, projection *rawMeshMaterialization, through uint64) error { + c.deltaMu.Lock() + defer c.deltaMu.Unlock() + if c.fullReconciliation || c.rawBaseline == nil { + return ErrFullReconciliationRequired + } + if err := ctx.Err(); err != nil { + c.clearPendingTopologyLocked() + c.markFullReconciliationLocked() + return errors.Join(ErrFullReconciliationRequired, err) + } + if through == 0 { + return nil + } + prefix := 0 + for prefix < len(c.pendingTopology) && c.pendingTopology[prefix].sequence <= through { + prefix++ + } + if prefix == 0 || c.pendingTopology[prefix-1].sequence != through { + c.clearPendingTopologyLocked() + c.markFullReconciliationLocked() + return ErrFullReconciliationRequired + } + c.mu.Lock() + c.commitRawMeshMaterializationLocked(projection) + c.rawBaseline = candidate + c.trimPendingTopologyPrefixLocked(through) + c.mu.Unlock() + c.commitRawMeshMaterializationCounters(projection) + return nil +} + +// ApplyPendingState builds and commits the current map from local state. +// Callers that must validate a derived representation before commit should use +// ApplyPendingStateValidated. +func (c *DWNClient) ApplyPendingState(ctx context.Context) (*MapResponse, error) { + return c.ApplyPendingStateValidated(ctx, nil) +} + +// ApplyPendingStateValidated builds the current map from local state, +// hydrating only a write whose subscription frame omitted its data. The +// validator runs after complete projection but before the raw baseline, parsed +// state, counters, or captured queue prefix are committed. A validation error +// leaves the complete candidate pending for a later retry. +func (c *DWNClient) ApplyPendingStateValidated(ctx context.Context, validate PendingStateValidator) (*MapResponse, error) { + if ctx == nil { + return nil, fmt.Errorf("applying pending topology: nil context") + } + if err := ctx.Err(); err != nil { + return nil, err + } + + c.loadMu.Lock() + defer c.loadMu.Unlock() + + candidate, events, through, err := c.beginPendingTopology() + if err != nil { + return nil, err + } + if len(events) == 0 { + response := c.buildMapResponse() + if response == nil { + c.RequireFullReconciliation() + return nil, ErrFullReconciliationRequired + } + if validate != nil { + if err := validate(response); err != nil { + return response, err + } + } + // Fence publication against repair or cancellation racing validation. + if err := c.completePendingTopology(ctx, candidate, nil, 0); err != nil { + return nil, err + } + return response, nil + } + + for _, event := range events { + if err := c.applyPendingTopologyEvent(ctx, candidate, event.message); err != nil { + c.RequireFullReconciliation() + return nil, errors.Join(ErrFullReconciliationRequired, err) + } + } + projection, err := c.projectRawMeshRecordSetWithDecryptors(ctx, candidate, c.makeDecryptor) + if err != nil { + c.RequireFullReconciliation() + return nil, errors.Join(ErrFullReconciliationRequired, err) + } + if validate != nil { + if err := validate(projection.response); err != nil { + return projection.response, err + } + } + if err := c.completePendingTopology(ctx, candidate, projection, through); err != nil { + return nil, err + } + return projection.response, nil +} + +func (c *DWNClient) applyPendingTopologyEvent(ctx context.Context, candidate *rawMeshRecordSet, message *dwn.SubscriptionMessage) error { + if err := ctx.Err(); err != nil { + return err + } + if _, err := candidate.applySubscriptionMessage(message, ""); err == nil { + return nil + } else if !errors.Is(err, errRawMeshRecordDataUnavailable) { + return err + } + return c.hydrateAndApplyTopologyWrite(ctx, candidate, message) +} + +func (c *DWNClient) hydrateAndApplyTopologyWrite(ctx context.Context, candidate *rawMeshRecordSet, message *dwn.SubscriptionMessage) error { + recordID, err := topologyWriteRecordID(message) + if err != nil { + return err + } + if c.anchorDWN == nil { + return fmt.Errorf("hydrating topology record %s: no anchor DWN client", recordID) + } + result, err := c.anchorDWN.RecordsReadWithAuth( + ctx, + c.anchorTenant, + dwn.RecordsFilter{RecordID: recordID}, + c.readAuth(c.protocolRole), + ) + if err != nil { + return fmt.Errorf("hydrating topology record %s: %w", recordID, err) + } + raw, err := rawMeshRecordReadEntry(result) + if err != nil { + return fmt.Errorf("hydrating topology record %s: %w", recordID, err) + } + original, err := normalizeRawMeshRecordIdentity(message.Event.Message, "") + if err != nil { + return fmt.Errorf("hydrating topology record %s: invalid original write: %w", recordID, err) + } + record, err := normalizeRawMeshRecord(raw, "") + if err != nil { + return fmt.Errorf("hydrating topology record %s: %w", recordID, err) + } + if record.recordID != original.recordID || record.protocol != original.protocol || + record.protocolPath != original.protocolPath || record.contextID != original.contextID || + record.parentID != original.parentID || record.recipient != original.recipient || + record.dateCreated != original.dateCreated { + return fmt.Errorf("hydrating topology record %s: read returned a different immutable record slot", recordID) + } + if compareRawMeshRecordRevision(record.head(), original.head()) < 0 { + return fmt.Errorf("hydrating topology record %s: read returned stale head %s before event %s", + recordID, record.messageCID, original.messageCID) + } + + // Apply the read head by its own canonical identity. Replaying it with the + // subscription frame would compare a newer head against the original cursor CID. + candidate.mu.Lock() + candidate.initLocked(1) + candidate.applyWriteLocked(record) + candidate.mu.Unlock() + return nil +} + +func rawTopologyEventAffectsMap(message *dwn.SubscriptionMessage) (relevant, certain bool) { + if message != nil && message.IsLatestBaseState != nil && !*message.IsLatestBaseState { + return false, true + } + if message == nil || message.Type != dwn.SubscriptionEventType || message.Event == nil || len(message.Event.Message) == 0 { + return false, false + } + method, err := classifyRawRecordMessage(message.Event.Message) + if err != nil { + return false, false + } + if method == "Delete" { + if len(message.Event.InitialWrite) == 0 { + return true, true + } + deletion, err := normalizeRawMeshRecordDelete(message.Event.Message) + if err != nil { + return false, false + } + initial, err := normalizeRawMeshRecordIdentity(message.Event.InitialWrite, "") + if err != nil || initial.recordID != deletion.recordID { + return false, false + } + if initial.protocol != protocols.MeshProtocolURI { + return false, true + } + return rawMeshMapProtocolPath(initial.protocolPath), true + } + write, _, err := unwrapRawRecordMessage(message.Event.Message, "recordsWrite") + if err != nil || write.Descriptor == nil || write.Descriptor.Protocol == "" || write.Descriptor.ProtocolPath == "" { + return false, false + } + if write.Descriptor.Protocol != protocols.MeshProtocolURI { + return false, true + } + return rawMeshMapProtocolPath(write.Descriptor.ProtocolPath), true +} + +func rawMeshMapProtocolPath(path string) bool { + switch path { + case "network", "network/member", "network/node", "network/member/node", + "network/relay", "network/aclPolicy", "network/node/nodeInfo", + "network/member/node/nodeInfo", "network/node/endpoint", + "network/member/node/endpoint": + return true + default: + return false + } +} + +func topologyWriteRecordID(message *dwn.SubscriptionMessage) (string, error) { + if message == nil || message.Event == nil { + return "", fmt.Errorf("topology write is missing its event") + } + write, _, err := unwrapRawRecordMessage(message.Event.Message, "recordsWrite") + if err != nil { + return "", err + } + if write.RecordID == "" { + return "", fmt.Errorf("topology write is missing recordId") + } + return write.RecordID, nil +} + +func pendingTopologyMessageBytes(message *dwn.SubscriptionMessage) int { + if message == nil { + return 0 + } + size := pendingTopologyFixedBytes + len(message.Type) + len(message.Seq) + + len(message.MessageCID) + len(message.Protocol) + len(message.EncodedData) + if message.Cursor != nil { + size += len(message.Cursor.StreamID) + len(message.Cursor.Epoch) + + len(message.Cursor.Position) + len(message.Cursor.MessageCID) + } + if message.Event != nil { + size += len(message.Event.Message) + len(message.Event.InitialWrite) + } + if message.Error != nil { + size += len(message.Error.Code) + len(message.Error.Detail) + } + return size +} + +func pendingTopologyEventsBytes(events []pendingTopologyEvent) int { + total := 0 + for _, event := range events { + total += pendingTopologyMessageBytes(event.message) + } + return total +} + +func (c *DWNClient) trimPendingTopologyPrefixLocked(through uint64) { + first := 0 + removedBytes := 0 + for first < len(c.pendingTopology) && c.pendingTopology[first].sequence <= through { + removedBytes += c.pendingTopology[first].retainedBytes + first++ + } + if first == 0 { + return + } + remaining := append([]pendingTopologyEvent(nil), c.pendingTopology[first:]...) + clear(c.pendingTopology) + c.pendingTopology = remaining + c.pendingTopologyBytes -= removedBytes +} + +func (c *DWNClient) clearPendingTopologyLocked() { + clear(c.pendingTopology) + c.pendingTopology = nil + c.pendingTopologyBytes = 0 +} + +func clonePendingTopologyMessage(message *dwn.SubscriptionMessage) (*dwn.SubscriptionMessage, error) { + if message == nil || message.Type != dwn.SubscriptionEventType || message.Event == nil || len(message.Event.Message) == 0 { + return nil, fmt.Errorf("invalid topology event") + } + cloned := *message + if message.Cursor != nil { + cursor := *message.Cursor + cloned.Cursor = &cursor + } + event := *message.Event + event.Message = cloneRawJSON(message.Event.Message) + event.InitialWrite = cloneRawJSON(message.Event.InitialWrite) + cloned.Event = &event + if message.Error != nil { + wireError := *message.Error + cloned.Error = &wireError + } + if message.IsLatestBaseState != nil { + latest := *message.IsLatestBaseState + cloned.IsLatestBaseState = &latest + } + return &cloned, nil +} diff --git a/internal/control/delta_queue_test.go b/internal/control/delta_queue_test.go new file mode 100644 index 0000000..6d92095 --- /dev/null +++ b/internal/control/delta_queue_test.go @@ -0,0 +1,974 @@ +package control + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/enboxorg/meshd/internal/dwn" + "github.com/enboxorg/meshd/protocols" +) + +func TestApplyPendingStateWriteDeleteUsesNoRemoteRequest(t *testing.T) { + client := newMaterializerTestClient() + set := installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + + peerWrite := materializerRecord(t, materializerRecordSpec{ + id: "peer-node", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.2", Label: "peer"}, + timestamp: "2026-07-11T12:01:00Z", + }) + event := rawRecordTestSubscription(peerWrite, "") + event.Cursor = &dwn.ProgressToken{StreamID: "topology", Epoch: "epoch", Position: "1"} + if err := client.StageTopologyEvent(event); err != nil { + t.Fatal(err) + } + for i := range event.Event.Message { + event.Event.Message[i] = 'x' + } + event.Cursor.Position = "mutated" + *event.IsLatestBaseState = false + + response, err := client.ApplyPendingState(context.Background()) + if err != nil { + t.Fatalf("apply write: %v", err) + } + if len(response.Peers) != 1 || response.Peers[0].DID != materializerPeerDID { + t.Fatalf("write peers = %#v", response.Peers) + } + if _, ok := client.rawBaseline.get("peer-node"); !ok { + t.Fatal("committed raw baseline is missing peer write") + } + + deleteEvent := rawRecordTestSubscription( + rawRecordTestDelete(t, "peer-node", "2026-07-11T12:02:00Z"), + "", + ) + if err := client.StageTopologyEvent(deleteEvent); err != nil { + t.Fatal(err) + } + response, err = client.ApplyPendingState(context.Background()) + if err != nil { + t.Fatalf("apply delete: %v", err) + } + if len(response.Peers) != 0 { + t.Fatalf("delete peers = %#v", response.Peers) + } + if _, ok := client.rawBaseline.get("peer-node"); ok { + t.Fatal("committed raw baseline retained deleted peer") + } + if response, err = client.ApplyPendingState(context.Background()); err != nil || response.Node == nil { + t.Fatalf("no-pending local response = %#v, %v", response, err) + } + if client.anchorDWN != nil { + t.Fatal("zero-remote test unexpectedly configured an anchor client") + } + + // Keep the initially returned set demonstrably independent from the cache. + externalPeerWrite := materializerRecord(t, materializerRecordSpec{ + id: "peer-node", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.2", Label: "external"}, + timestamp: "2026-07-11T12:04:00Z", + }) + if _, err := set.addEntries([]json.RawMessage{externalPeerWrite}, ""); err != nil { + t.Fatal(err) + } + if _, ok := client.rawBaseline.get("peer-node"); ok { + t.Fatal("external baseline mutation reached the installed cache") + } +} + +func TestApplyPendingStateHydratesNewerHeadWithOriginalCursorCID(t *testing.T) { + owner, signer, _, _ := sealedTestOwner(t) + endpointData, err := json.Marshal(EndpointData{ + LocalEndpoints: []string{"192.0.2.55:4242"}, + DiscoKey: "hydrated-disco", + UpdatedAt: "2026-07-11T12:04:00Z", + }) + if err != nil { + t.Fatal(err) + } + dateCreated := "2026-07-11T12:03:00Z" + originalEntry := deltaRecordWithoutEncodedData(t, materializerRecord(t, materializerRecordSpec{ + id: "self-endpoint", path: "network/node/endpoint", + parentContext: materializerNetworkID + "/self-node", + data: EndpointData{}, timestamp: dateCreated, + })) + readEntry := deltaRecordWithoutEncodedData(t, materializerRecord(t, materializerRecordSpec{ + id: "self-endpoint", path: "network/node/endpoint", + parentContext: materializerNetworkID + "/self-node", + data: EndpointData{}, dateCreated: dateCreated, timestamp: "2026-07-11T12:04:00Z", + })) + originalCID, err := computeRawRecordMessageCID(originalEntry) + if err != nil { + t.Fatal(err) + } + readCID, err := computeRawRecordMessageCID(readEntry) + if err != nil { + t.Fatal(err) + } + if readCID == originalCID { + t.Fatal("newer RecordsRead head reused the original event CID") + } + + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + var request dwn.JsonRpcRequest + if err := json.Unmarshal([]byte(r.Header.Get("dwn-request")), &request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + response := &dwn.JsonRpcResponse{ + JSONRPC: "2.0", + ID: request.ID, + Result: &dwn.JsonRpcResult{ + Reply: &dwn.DwnReply{ + Status: dwn.Status{Code: http.StatusOK, Detail: "OK"}, + Entry: readEntry, + }, + }, + } + wire, err := json.Marshal(response) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("dwn-response", string(wire)) + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(endpointData) + })) + defer server.Close() + + client := NewDWNClient( + server.URL, + owner.URI, + materializerNetworkID, + materializerSelfDID, + signer, + ) + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + event := rawRecordTestSubscription(originalEntry, "") + event.MessageCID = originalCID + event.Cursor = &dwn.ProgressToken{ + StreamID: "topology", Epoch: "epoch", Position: "1", MessageCID: originalCID, + } + if err := client.StageTopologyEvent(event); err != nil { + t.Fatal(err) + } + + response, err := client.ApplyPendingState(context.Background()) + if err != nil { + t.Fatalf("ApplyPendingState: %v", err) + } + if got := requests.Load(); got != 1 { + t.Fatalf("RecordsRead requests = %d, want exactly 1", got) + } + if len(response.Node.Endpoints) != 1 || response.Node.Endpoints[0] != "192.0.2.55:4242" { + t.Fatalf("hydrated endpoints = %#v", response.Node.Endpoints) + } + if response.Node.DiscoKey != "hydrated-disco" { + t.Fatalf("hydrated disco key = %q", response.Node.DiscoKey) + } + committed, ok := client.rawBaseline.get("self-endpoint") + if !ok || committed.messageCID != readCID || committed.messageTimestamp != "2026-07-11T12:04:00Z" { + t.Fatalf("committed hydrated head = %#v, want newer CID %q", committed, readCID) + } + if _, err := client.ApplyPendingState(context.Background()); err != nil { + t.Fatal(err) + } + if got := requests.Load(); got != 1 { + t.Fatalf("no-pending apply made another request: %d", got) + } +} + +func TestApplyPendingStateHydrationHTTP500DefersFullRebuild(t *testing.T) { + owner, signer, _, _ := sealedTestOwner(t) + entry := deltaRecordWithoutEncodedData(t, materializerRecord(t, materializerRecordSpec{ + id: "self-endpoint-500", path: "network/node/endpoint", + parentContext: materializerNetworkID + "/self-node", data: EndpointData{}, + timestamp: "2026-07-11T12:05:00Z", + })) + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + var request dwn.JsonRpcRequest + if err := json.Unmarshal([]byte(r.Header.Get("dwn-request")), &request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(dwn.JsonRpcResponse{ + JSONRPC: "2.0", ID: request.ID, Result: &dwn.JsonRpcResult{Reply: &dwn.DwnReply{ + Status: dwn.Status{Code: http.StatusInternalServerError, Detail: "forced failure"}, + }}, + }) + })) + defer server.Close() + client := NewDWNClient(server.URL, owner.URI, materializerNetworkID, materializerSelfDID, signer) + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + oldRaw := client.rawBaseline + if err := client.StageTopologyEvent(rawRecordTestSubscription(entry, "")); err != nil { + t.Fatal(err) + } + + _, err := client.ApplyPendingState(context.Background()) + if !errors.Is(err, ErrFullReconciliationRequired) || !errors.Is(err, dwn.ErrTransport) { + t.Fatalf("hydration error = %v, want full-repair + transport", err) + } + if requests.Load() != 1 || client.rawBaseline != oldRaw { + t.Fatalf("hydration failure requests=%d raw=%p/old=%p", requests.Load(), client.rawBaseline, oldRaw) + } +} + +func TestDeltaQueueOverflowMalformedAndIrrelevantFrames(t *testing.T) { + client := newMaterializerTestClient() + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + + irrelevant := materializerRecord(t, materializerRecordSpec{ + id: "invite", path: "network/invite", parentContext: materializerNetworkID, + data: json.RawMessage("not-json"), timestamp: "2026-07-11T12:00:00Z", + }) + beforeSequence := client.topologySequence + if err := client.StageTopologyEvent(rawRecordTestSubscription(irrelevant, "")); err != nil { + t.Fatal(err) + } + if len(client.pendingTopology) != 0 || client.topologySequence != beforeSequence { + t.Fatalf("irrelevant frame queued: len=%d sequence=%d", len(client.pendingTopology), client.topologySequence) + } + + counted := rawRecordTestSubscription( + rawRecordTestDelete(t, "counted", "2026-07-11T12:00:00Z"), "", + ) + for i := 0; i < maxPendingTopologyEvents; i++ { + if err := client.StageTopologyEvent(counted); err != nil { + t.Fatal(err) + } + } + if got := len(client.pendingTopology); got != maxPendingTopologyEvents { + t.Fatalf("queue length = %d", got) + } + if err := client.StageTopologyEvent(rawRecordTestSubscription( + rawRecordTestDelete(t, "overflow", "2026-07-11T13:00:00Z"), "", + )); err != nil { + t.Fatal(err) + } + if !client.fullReconciliation || len(client.pendingTopology) != 0 || client.pendingTopologyBytes != 0 { + t.Fatalf("overflow repair=%v queue=%d bytes=%d", client.fullReconciliation, len(client.pendingTopology), client.pendingTopologyBytes) + } + if _, err := client.ApplyPendingState(context.Background()); !errors.Is(err, ErrFullReconciliationRequired) { + t.Fatalf("overflow apply error = %v", err) + } + + through := client.beginFullReconciliation() + client.installRawBaseline(newDeltaTestRawSet(t, materializerBaseRecords(t)), through) + if client.fullReconciliation { + t.Fatal("covered overflow repair marker survived baseline") + } + if err := client.StageTopologyEvent(&dwn.SubscriptionMessage{Type: dwn.SubscriptionEventType}); err != nil { + t.Fatal(err) + } + if !client.fullReconciliation { + t.Fatal("malformed poison frame did not request full reconciliation") + } +} + +func TestDeltaQueueIgnoresIrrelevantDeleteInitialWrite(t *testing.T) { + client := newMaterializerTestClient() + timestamp := "2026-07-11T12:00:00Z" + initialWrite := rawRecordTestJSON(t, map[string]any{ + "recordId": "invite-delete", + "contextId": materializerNetworkID + "/invite-delete", + "descriptor": map[string]any{ + "interface": "Records", + "method": "Write", + "protocol": protocols.MeshProtocolURI, + "protocolPath": "network/invite", + "parentId": materializerNetworkID, + "dateCreated": timestamp, + "messageTimestamp": timestamp, + }, + }) + event := rawRecordTestSubscription(rawRecordTestDelete(t, "invite-delete", "2026-07-11T12:01:00Z"), "") + event.Event.InitialWrite = initialWrite + beforeSequence := client.topologySequence + + if err := client.StageTopologyEvent(event); err != nil { + t.Fatal(err) + } + if client.fullReconciliation || len(client.pendingTopology) != 0 || client.topologySequence != beforeSequence { + t.Fatalf("irrelevant delete queued: repair=%v queue=%d sequence=%d", client.fullReconciliation, len(client.pendingTopology), client.topologySequence) + } +} + +func TestDeltaSequenceAndBaselineCutPreserveNewerEvents(t *testing.T) { + client := newMaterializerTestClient() + set := installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + + if err := client.StageTopologyEvent(rawRecordTestSubscription( + rawRecordTestDelete(t, "first", "2026-07-11T12:01:00Z"), "", + )); err != nil { + t.Fatal(err) + } + firstSequence := client.pendingTopology[0].sequence + if _, err := client.ApplyPendingState(context.Background()); err != nil { + t.Fatal(err) + } + if len(client.pendingTopology) != 0 { + t.Fatal("applied prefix remained queued") + } + assertPendingTopologyByteInvariant(t, client) + + if err := client.StageTopologyEvent(rawRecordTestSubscription( + rawRecordTestDelete(t, "covered", "2026-07-11T12:02:00Z"), "", + )); err != nil { + t.Fatal(err) + } + secondSequence := client.pendingTopology[0].sequence + if secondSequence <= firstSequence { + t.Fatalf("sequence reset across drained batches: first=%d second=%d", firstSequence, secondSequence) + } + through := client.beginFullReconciliation() + if err := client.StageTopologyEvent(rawRecordTestSubscription( + rawRecordTestDelete(t, "after-cut", "2026-07-11T12:03:00Z"), "", + )); err != nil { + t.Fatal(err) + } + afterCutSequence := client.pendingTopology[1].sequence + afterCutBytes := pendingTopologyMessageBytes(client.pendingTopology[1].message) + client.installRawBaseline(set, through) + if len(client.pendingTopology) != 1 || client.pendingTopology[0].sequence != afterCutSequence { + t.Fatalf("baseline cut lost concurrent tail: %#v", client.pendingTopology) + } + if got := pendingTopologyEventsBytes(client.pendingTopology); got != afterCutBytes { + t.Fatalf("baseline cut bytes = %d, want tail %d", got, afterCutBytes) + } + if _, err := client.ApplyPendingState(context.Background()); err != nil { + t.Fatal(err) + } + if got := pendingTopologyEventsBytes(client.pendingTopology); got != 0 { + t.Fatalf("applied prefix bytes = %d, want 0", got) + } + assertPendingTopologyByteInvariant(t, client) +} + +func TestApplyPendingStateRequiredRecordFailureRollsBack(t *testing.T) { + client := newMaterializerTestClient() + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + oldNetwork := client.network + oldNode := client.nodes[materializerSelfDID] + oldACL := client.acl + oldBaseline := client.rawBaseline + oldUnreadable := client.UnreadableEndpointCount() + oldDropped := client.DroppedPeerCount() + + invalidNetwork := materializerRecord(t, materializerRecordSpec{ + id: materializerNetworkID, path: "network", data: json.RawMessage("not-json"), + timestamp: "2026-07-11T12:02:00Z", + }) + if err := client.StageTopologyEvent(rawRecordTestSubscription(invalidNetwork, "")); err != nil { + t.Fatal(err) + } + if _, err := client.ApplyPendingState(context.Background()); !errors.Is(err, ErrFullReconciliationRequired) { + t.Fatalf("ApplyPendingState error = %v", err) + } + if !client.fullReconciliation { + t.Fatal("required-record projection failure did not request repair") + } + if client.network != oldNetwork || client.nodes[materializerSelfDID] != oldNode || + client.acl != oldACL || client.rawBaseline != oldBaseline { + t.Fatal("required-record projection failure replaced last-good pointers") + } + if client.UnreadableEndpointCount() != oldUnreadable || client.DroppedPeerCount() != oldDropped { + t.Fatal("required-record projection failure transferred observability counters") + } +} + +func TestApplyPendingStateSelectedRecordFailureUsesLastGoodPolicy(t *testing.T) { + t.Run("ACL retains prior policy", func(t *testing.T) { + client := newMaterializerTestClient() + records := append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "acl", path: "network/aclPolicy", parentContext: materializerNetworkID, + data: ACLPolicyData{Version: 1, DefaultAction: "accept"}, timestamp: "2026-07-11T12:01:00Z", + })) + installDeltaTestBaseline(t, client, records) + oldBaseline := client.rawBaseline + oldACL := client.acl + update := materializerRecord(t, materializerRecordSpec{ + id: "acl", path: "network/aclPolicy", parentContext: materializerNetworkID, + data: json.RawMessage("not-json"), timestamp: "2026-07-11T12:02:00Z", + }) + if err := client.StageTopologyEvent(rawRecordTestSubscription(update, "")); err != nil { + t.Fatal(err) + } + if _, err := client.ApplyPendingState(context.Background()); err != nil { + t.Fatalf("ApplyPendingState: %v", err) + } + if client.fullReconciliation || len(client.pendingTopology) != 0 { + t.Fatalf("ACL replacement caused repair=%v queue=%d", client.fullReconciliation, len(client.pendingTopology)) + } + if client.rawBaseline == oldBaseline { + t.Fatal("ACL replacement did not advance the raw baseline") + } + if oldACL == nil || client.acl == nil || oldACL.Version != 1 || client.acl.Version != 1 || + oldACL.DefaultAction != "accept" || client.acl.DefaultAction != "accept" { + t.Fatalf("last-good ACL was not preserved: old=%#v current=%#v", oldACL, client.acl) + } + }) + + t.Run("endpoint is skipped", func(t *testing.T) { + client := newMaterializerTestClient() + records := append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "endpoint", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", + data: EndpointData{LocalEndpoints: []string{"old-endpoint"}}, timestamp: "2026-07-11T12:01:00Z", + })) + installDeltaTestBaseline(t, client, records) + oldBaseline := client.rawBaseline + oldNode := client.nodes[materializerSelfDID] + oldUnreadable := client.UnreadableEndpointCount() + update := materializerRecord(t, materializerRecordSpec{ + id: "endpoint", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", + data: json.RawMessage("not-json"), timestamp: "2026-07-11T12:02:00Z", + }) + if err := client.StageTopologyEvent(rawRecordTestSubscription(update, "")); err != nil { + t.Fatal(err) + } + response, err := client.ApplyPendingState(context.Background()) + if err != nil { + t.Fatalf("ApplyPendingState: %v", err) + } + if client.fullReconciliation || len(client.pendingTopology) != 0 { + t.Fatalf("endpoint replacement caused repair=%v queue=%d", client.fullReconciliation, len(client.pendingTopology)) + } + if client.rawBaseline == oldBaseline { + t.Fatal("endpoint replacement did not advance the raw baseline") + } + if response == nil || response.Node == nil || len(response.Node.Endpoints) != 0 || + len(client.nodes[materializerSelfDID].Endpoints) != 0 { + t.Fatalf("malformed endpoint was not skipped: response=%#v node=%#v", response, client.nodes[materializerSelfDID]) + } + if oldNode == nil || len(oldNode.Endpoints) != 1 || len(oldNode.Endpoints[0].LocalEndpoints) != 1 || + oldNode.Endpoints[0].LocalEndpoints[0] != "old-endpoint" { + t.Fatalf("projection mutated the prior node: %#v", oldNode) + } + if client.UnreadableEndpointCount() <= oldUnreadable { + t.Fatal("skipped endpoint did not increment unreadable endpoint count") + } + }) +} + +func TestPendingCommitFencePreservesLastGoodOnRepairAndCancellation(t *testing.T) { + t.Run("repair between projection and commit", func(t *testing.T) { + client, candidate, projection, through, oldNode, oldBaseline := prepareBlockedDeltaCommit(t) + oldDropped := client.DroppedPeerCount() + + release := make(chan struct{}) + repaired := make(chan struct{}) + go func() { + <-release + client.RequireFullReconciliation() + close(repaired) + }() + close(release) + <-repaired + + err := client.completePendingTopology(context.Background(), candidate, projection, through) + if !errors.Is(err, ErrFullReconciliationRequired) { + t.Fatalf("complete error = %v", err) + } + if client.nodes[materializerSelfDID] != oldNode || client.rawBaseline != oldBaseline { + t.Fatal("repair fence committed projected state") + } + if client.DroppedPeerCount() != oldDropped { + t.Fatal("repair fence transferred projected counters") + } + }) + + t.Run("context canceled at commit fence", func(t *testing.T) { + client, candidate, projection, through, oldNode, oldBaseline := prepareBlockedDeltaCommit(t) + oldDropped := client.DroppedPeerCount() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := client.completePendingTopology(ctx, candidate, projection, through) + if !errors.Is(err, context.Canceled) || !errors.Is(err, ErrFullReconciliationRequired) { + t.Fatalf("complete error = %v", err) + } + if client.nodes[materializerSelfDID] != oldNode || client.rawBaseline != oldBaseline { + t.Fatal("canceled fence committed projected state") + } + if client.DroppedPeerCount() != oldDropped { + t.Fatal("canceled fence transferred projected counters") + } + }) +} + +func prepareBlockedDeltaCommit(t *testing.T) ( + *DWNClient, + *rawMeshRecordSet, + *rawMeshMaterialization, + uint64, + *NodeRecord, + *rawMeshRecordSet, +) { + t.Helper() + client := newMaterializerTestClient() + records := []json.RawMessage{ + materializerRecord(t, materializerRecordSpec{ + id: materializerNetworkID, path: "network", + data: NetworkConfig{Name: "no-fallback"}, timestamp: "2026-07-11T12:00:00Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "self-node", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, data: NodeRecord{MeshIP: "10.200.1.1"}, + timestamp: "2026-07-11T12:00:01Z", + }), + } + installDeltaTestBaseline(t, client, records) + oldNode := client.nodes[materializerSelfDID] + oldBaseline := client.rawBaseline + + peer := materializerRecord(t, materializerRecordSpec{ + id: "peer-node", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerPeerDID, data: NodeRecord{Label: "no-ip"}, + timestamp: "2026-07-11T12:01:00Z", + }) + if err := client.StageTopologyEvent(rawRecordTestSubscription(peer, "")); err != nil { + t.Fatal(err) + } + candidate, events, through, err := client.beginPendingTopology() + if err != nil { + t.Fatal(err) + } + for _, event := range events { + if err := client.applyPendingTopologyEvent(context.Background(), candidate, event.message); err != nil { + t.Fatal(err) + } + } + projection, err := client.projectRawMeshRecordSetWithDecryptors(context.Background(), candidate, client.makeDecryptor) + if err != nil { + t.Fatal(err) + } + if projection.builder.DroppedPeerCount() == 0 { + t.Fatal("fixture did not produce a projected counter delta") + } + return client, candidate, projection, through, oldNode, oldBaseline +} + +func TestApplyPendingStateValidatedCommitsOnlyAfterValidation(t *testing.T) { + client := newMaterializerTestClient() + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + oldBaseline := client.rawBaseline + oldNode := client.nodes[materializerSelfDID] + + peerWrite := materializerRecord(t, materializerRecordSpec{ + id: "validated-peer", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.2"}, + timestamp: "2026-07-11T12:05:00Z", + }) + if err := client.StageTopologyEvent(rawRecordTestSubscription(peerWrite, "")); err != nil { + t.Fatal(err) + } + validationErr := errors.New("network map rejected") + response, err := client.ApplyPendingStateValidated(context.Background(), func(candidate *MapResponse) error { + if len(candidate.Peers) != 1 { + t.Fatalf("candidate peers = %#v", candidate.Peers) + } + return validationErr + }) + if !errors.Is(err, validationErr) || response == nil { + t.Fatalf("validated apply = (%#v, %v)", response, err) + } + if client.rawBaseline != oldBaseline || client.nodes[materializerSelfDID] != oldNode || len(client.pendingTopology) != 1 { + t.Fatal("validation failure advanced last-good state or queue") + } + if _, err := client.ApplyPendingStateValidated(context.Background(), func(*MapResponse) error { return nil }); err != nil { + t.Fatalf("validated retry: %v", err) + } + if client.rawBaseline == oldBaseline || len(client.pendingTopology) != 0 { + t.Fatal("successful validation did not commit the captured prefix") + } +} + +func TestApplyPendingStateValidatedPreservesConcurrentTail(t *testing.T) { + client := newMaterializerTestClient() + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + first := rawRecordTestSubscription(rawRecordTestDelete(t, "first-tail", "2026-07-11T12:05:00Z"), "") + second := rawRecordTestSubscription(rawRecordTestDelete(t, "second-tail", "2026-07-11T12:06:00Z"), "") + if err := client.StageTopologyEvent(first); err != nil { + t.Fatal(err) + } + entered := make(chan struct{}) + release := make(chan struct{}) + done := make(chan error, 1) + go func() { + _, err := client.ApplyPendingStateValidated(context.Background(), func(*MapResponse) error { + close(entered) + <-release + return nil + }) + done <- err + }() + <-entered + if err := client.StageTopologyEvent(second); err != nil { + t.Fatal(err) + } + secondSequence := client.pendingTopology[len(client.pendingTopology)-1].sequence + close(release) + if err := <-done; err != nil { + t.Fatal(err) + } + if len(client.pendingTopology) != 1 || client.pendingTopology[0].sequence != secondSequence { + t.Fatalf("concurrent tail = %#v", client.pendingTopology) + } + assertPendingTopologyByteInvariant(t, client) +} + +func TestApplyPendingStateValidatedRepairAndCancellationFence(t *testing.T) { + for _, test := range []struct { + name string + fence func(*DWNClient, context.CancelFunc) + want error + }{ + {name: "repair", fence: func(client *DWNClient, _ context.CancelFunc) { client.RequireFullReconciliation() }, want: ErrFullReconciliationRequired}, + {name: "cancel", fence: func(_ *DWNClient, cancel context.CancelFunc) { cancel() }, want: context.Canceled}, + } { + t.Run(test.name, func(t *testing.T) { + client := newMaterializerTestClient() + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + oldBaseline := client.rawBaseline + if err := client.StageTopologyEvent(rawRecordTestSubscription(rawRecordTestDelete(t, "fenced", "2026-07-11T12:05:00Z"), "")); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + entered := make(chan struct{}) + release := make(chan struct{}) + done := make(chan error, 1) + go func() { + _, err := client.ApplyPendingStateValidated(ctx, func(*MapResponse) error { + close(entered) + <-release + return nil + }) + done <- err + }() + <-entered + test.fence(client, cancel) + close(release) + err := <-done + if !errors.Is(err, test.want) || !errors.Is(err, ErrFullReconciliationRequired) { + t.Fatalf("fenced apply error = %v", err) + } + if client.rawBaseline != oldBaseline { + t.Fatal("fenced validation committed raw state") + } + }) + } +} + +func TestApplyPendingStateValidatedNoPendingRepairFence(t *testing.T) { + client := newMaterializerTestClient() + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + oldBaseline := client.rawBaseline + entered := make(chan struct{}) + release := make(chan struct{}) + done := make(chan error, 1) + go func() { + _, err := client.ApplyPendingStateValidated(context.Background(), func(*MapResponse) error { + close(entered) + <-release + return nil + }) + done <- err + }() + <-entered + client.RequireFullReconciliation() + close(release) + if err := <-done; !errors.Is(err, ErrFullReconciliationRequired) { + t.Fatalf("no-pending fenced apply error = %v", err) + } + if client.rawBaseline != oldBaseline { + t.Fatal("no-pending fence changed baseline") + } +} + +func TestTopologyQueueByteBudgetBoundaryAndAccounting(t *testing.T) { + client := newMaterializerTestClient() + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + event := rawRecordTestSubscription(rawRecordTestDelete(t, "byte-budget", "2026-07-11T12:05:00Z"), "") + baseBytes := pendingTopologyMessageBytes(event) + event.EncodedData = strings.Repeat("x", maxPendingTopologyBytes-baseBytes) + if got := pendingTopologyMessageBytes(event); got != maxPendingTopologyBytes { + t.Fatalf("boundary event bytes = %d", got) + } + if err := client.StageTopologyEvent(event); err != nil { + t.Fatal(err) + } + if got := pendingTopologyEventsBytes(client.pendingTopology); got != maxPendingTopologyBytes || client.fullReconciliation { + t.Fatalf("boundary queue bytes=%d repair=%v", got, client.fullReconciliation) + } + assertPendingTopologyByteInvariant(t, client) + if err := client.StageTopologyEvent(rawRecordTestSubscription(rawRecordTestDelete(t, "overflow-byte", "2026-07-11T12:06:00Z"), "")); err != nil { + t.Fatal(err) + } + if !client.fullReconciliation || len(client.pendingTopology) != 0 || pendingTopologyEventsBytes(client.pendingTopology) != 0 { + t.Fatal("byte overflow did not clear queue and request repair") + } + assertPendingTopologyByteInvariant(t, client) + + through := client.beginFullReconciliation() + client.installRawBaseline(newDeltaTestRawSet(t, materializerBaseRecords(t)), through) + if err := client.StageTopologyEvent(rawRecordTestSubscription(rawRecordTestDelete(t, "before-oversized", "2026-07-11T12:06:30Z"), "")); err != nil { + t.Fatal(err) + } + oversized := rawRecordTestSubscription(rawRecordTestDelete(t, "oversized-byte", "2026-07-11T12:07:00Z"), "") + oversized.EncodedData = strings.Repeat("y", maxPendingTopologyBytes-pendingTopologyMessageBytes(oversized)+1) + if err := client.StageTopologyEvent(oversized); err != nil { + t.Fatal(err) + } + if !client.fullReconciliation || len(client.pendingTopology) != 0 || client.pendingTopologyBytes != 0 { + t.Fatal("oversized single event was retained") + } + assertPendingTopologyByteInvariant(t, client) + + through = client.beginFullReconciliation() + client.installRawBaseline(newDeltaTestRawSet(t, materializerBaseRecords(t)), through) + metadataHeavy := rawRecordTestSubscription(rawRecordTestDelete(t, "metadata-budget", "2026-07-11T12:08:00Z"), "") + metadataHeavy.Seq = strings.Repeat("s", 4096) + metadataHeavy.MessageCID = strings.Repeat("m", 4096) + metadataHeavy.Protocol = strings.Repeat("p", 512<<10) + metadataHeavy.Cursor = &dwn.ProgressToken{ + StreamID: strings.Repeat("i", 512<<10), + Epoch: strings.Repeat("e", 512<<10), + Position: strings.Repeat("9", 512<<10), + } + metadataHeavy.Error = &dwn.SubscriptionError{ + Code: strings.Repeat("c", 4096), + Detail: strings.Repeat("d", 512<<10), + } + metadataBytes := pendingTopologyFixedBytes + len(metadataHeavy.Type) + len(metadataHeavy.Seq) + + len(metadataHeavy.MessageCID) + len(metadataHeavy.Protocol) + len(metadataHeavy.EncodedData) + + len(metadataHeavy.Cursor.StreamID) + len(metadataHeavy.Cursor.Epoch) + + len(metadataHeavy.Cursor.Position) + len(metadataHeavy.Cursor.MessageCID) + + len(metadataHeavy.Event.Message) + len(metadataHeavy.Event.InitialWrite) + + len(metadataHeavy.Error.Code) + len(metadataHeavy.Error.Detail) + if got := pendingTopologyMessageBytes(metadataHeavy); got != metadataBytes { + t.Fatalf("metadata event bytes = %d, want exact retained size %d", got, metadataBytes) + } + if metadataBytes >= maxPendingTopologyBytes { + t.Fatalf("metadata fixture already exceeds budget: %d", metadataBytes) + } + metadataHeavy.Cursor.MessageCID = strings.Repeat("r", maxPendingTopologyBytes-metadataBytes) + if got := pendingTopologyMessageBytes(metadataHeavy); got != maxPendingTopologyBytes { + t.Fatalf("metadata boundary bytes = %d", got) + } + if err := client.StageTopologyEvent(metadataHeavy); err != nil { + t.Fatal(err) + } + metadataHeavy.Cursor.MessageCID = "mutated after staging" + if got := pendingTopologyEventsBytes(client.pendingTopology); got != maxPendingTopologyBytes || client.fullReconciliation { + t.Fatalf("metadata boundary queue bytes=%d repair=%v", got, client.fullReconciliation) + } + assertPendingTopologyByteInvariant(t, client) + if err := client.StageTopologyEvent(rawRecordTestSubscription(rawRecordTestDelete(t, "metadata-overflow", "2026-07-11T12:09:00Z"), "")); err != nil { + t.Fatal(err) + } + if !client.fullReconciliation || len(client.pendingTopology) != 0 || client.pendingTopologyBytes != 0 { + t.Fatal("metadata byte overflow did not clear queue and request repair") + } + assertPendingTopologyByteInvariant(t, client) +} + +func TestDeltaQueueByteAccountingClearPaths(t *testing.T) { + client := newMaterializerTestClient() + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + event := rawRecordTestSubscription( + rawRecordTestDelete(t, "accounting-clear", "2026-07-11T12:10:00Z"), "", + ) + + if err := client.StageTopologyEvent(event); err != nil { + t.Fatal(err) + } + assertPendingTopologyByteInvariant(t, client) + client.RequireFullReconciliation() + assertPendingTopologyByteInvariant(t, client) + if !client.fullReconciliation { + t.Fatal("explicit repair did not set the repair marker") + } + + through := client.beginFullReconciliation() + client.installRawBaseline(newDeltaTestRawSet(t, materializerBaseRecords(t)), through) + if err := client.StageTopologyEvent(event); err != nil { + t.Fatal(err) + } + assertPendingTopologyByteInvariant(t, client) + client.deltaMu.Lock() + client.pendingTopology[0].message.Event = nil + client.deltaMu.Unlock() + if _, _, _, err := client.beginPendingTopology(); !errors.Is(err, ErrFullReconciliationRequired) { + t.Fatalf("clone failure = %v, want full repair", err) + } + assertPendingTopologyByteInvariant(t, client) + + through = client.beginFullReconciliation() + client.installRawBaseline(newDeltaTestRawSet(t, materializerBaseRecords(t)), through) + if err := client.StageTopologyEvent(event); err != nil { + t.Fatal(err) + } + candidate, _, pendingThrough, err := client.beginPendingTopology() + if err != nil { + t.Fatal(err) + } + canceled, cancel := context.WithCancel(context.Background()) + cancel() + if err := client.completePendingTopology(canceled, candidate, nil, pendingThrough); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled completion = %v", err) + } + assertPendingTopologyByteInvariant(t, client) + + through = client.beginFullReconciliation() + client.installRawBaseline(newDeltaTestRawSet(t, materializerBaseRecords(t)), through) + if err := client.StageTopologyEvent(event); err != nil { + t.Fatal(err) + } + client.installRawBaseline(nil, client.beginFullReconciliation()) + assertPendingTopologyByteInvariant(t, client) +} + +func TestFullReconciliationByteAccountingPreservesConcurrentTail(t *testing.T) { + client := newMaterializerTestClient() + set := installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + covered := rawRecordTestSubscription( + rawRecordTestDelete(t, "covered-accounting", "2026-07-11T12:11:00Z"), "", + ) + if err := client.StageTopologyEvent(covered); err != nil { + t.Fatal(err) + } + through := client.beginFullReconciliation() + tail := rawRecordTestSubscription( + rawRecordTestDelete(t, "concurrent-accounting-tail", "2026-07-11T12:12:00Z"), "", + ) + + const workers = 8 + const perWorker = 32 + start := make(chan struct{}) + errs := make(chan error, workers) + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for range perWorker { + if err := client.StageTopologyEvent(tail); err != nil { + errs <- err + return + } + } + }() + } + close(start) + client.installRawBaseline(set, through) + wg.Wait() + close(errs) + for err := range errs { + t.Fatal(err) + } + assertPendingTopologyByteInvariant(t, client) + client.deltaMu.Lock() + if got, want := len(client.pendingTopology), workers*perWorker; got != want { + client.deltaMu.Unlock() + t.Fatalf("post-cut tail count = %d, want %d", got, want) + } + if got, want := client.pendingTopologyBytes, workers*perWorker*pendingTopologyMessageBytes(tail); got != want { + client.deltaMu.Unlock() + t.Fatalf("post-cut tail bytes = %d, want %d", got, want) + } + client.deltaMu.Unlock() + + commitThrough := client.beginFullReconciliation() + postCommit := rawRecordTestSubscription( + rawRecordTestDelete(t, "post-commit-accounting", "2026-07-11T12:13:00Z"), "", + ) + if err := client.StageTopologyEvent(postCommit); err != nil { + t.Fatal(err) + } + candidate := newDeltaTestRawSet(t, materializerBaseRecords(t)) + projection, err := client.projectRawMeshRecordSetWithDecryptors(context.Background(), candidate, client.makeDecryptor) + if err != nil { + t.Fatal(err) + } + if err := client.completeFullReconciliation(context.Background(), candidate, projection, commitThrough); err != nil { + t.Fatal(err) + } + assertPendingTopologyByteInvariant(t, client) + client.deltaMu.Lock() + if got := len(client.pendingTopology); got != 1 { + client.deltaMu.Unlock() + t.Fatalf("full commit tail count = %d, want 1", got) + } + if got, want := client.pendingTopologyBytes, pendingTopologyMessageBytes(postCommit); got != want { + client.deltaMu.Unlock() + t.Fatalf("full commit tail bytes = %d, want %d", got, want) + } + client.deltaMu.Unlock() +} + +func assertPendingTopologyByteInvariant(t *testing.T, client *DWNClient) { + t.Helper() + client.deltaMu.Lock() + defer client.deltaMu.Unlock() + want := 0 + for i, event := range client.pendingTopology { + eventBytes := pendingTopologyMessageBytes(event.message) + if event.retainedBytes != eventBytes { + t.Fatalf("event %d retained bytes = %d, want %d", i, event.retainedBytes, eventBytes) + } + want += eventBytes + } + if client.pendingTopologyBytes != want { + t.Fatalf("pending topology bytes = %d, want %d", client.pendingTopologyBytes, want) + } + if client.pendingTopologyBytes < 0 || client.pendingTopologyBytes > maxPendingTopologyBytes { + t.Fatalf("pending topology byte bound = %d", client.pendingTopologyBytes) + } +} + +func installDeltaTestBaseline(t *testing.T, client *DWNClient, records []json.RawMessage) *rawMeshRecordSet { + t.Helper() + set := newDeltaTestRawSet(t, records) + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatalf("materialize baseline: %v", err) + } + through := client.beginFullReconciliation() + client.installRawBaseline(set, through) + return set +} + +func newDeltaTestRawSet(t *testing.T, records []json.RawMessage) *rawMeshRecordSet { + t.Helper() + set, err := newRawMeshRecordSet(records, "") + if err != nil { + t.Fatalf("newRawMeshRecordSet: %v", err) + } + return set +} + +func deltaRecordWithoutEncodedData(t *testing.T, raw json.RawMessage) json.RawMessage { + t.Helper() + var object map[string]json.RawMessage + if err := json.Unmarshal(raw, &object); err != nil { + t.Fatal(err) + } + delete(object, "encodedData") + stripped, err := json.Marshal(object) + if err != nil { + t.Fatal(err) + } + return stripped +} diff --git a/internal/control/dwnclient.go b/internal/control/dwnclient.go index 360c992..33129e1 100644 --- a/internal/control/dwnclient.go +++ b/internal/control/dwnclient.go @@ -1,6 +1,7 @@ package control import ( + "bytes" "context" "crypto/sha256" "encoding/base64" @@ -10,6 +11,7 @@ import ( "errors" "fmt" "log/slog" + "net/http" "net/netip" "sort" "strings" @@ -32,13 +34,15 @@ const DefaultPeerStaleThreshold = 5 * time.Minute // Sentinel errors. var ( - ErrNoNetwork = errors.New("network record not found") - ErrNoEntry = errors.New("no data found in entry") + ErrNoNetwork = errors.New("network record not found") + ErrNoEntry = errors.New("no data found in entry") + errRawCaptureReadNotFound = errors.New("RecordsRead entry not found") ) func shouldAbortStateLoad(ctx context.Context, err error) bool { return ctx.Err() != nil || errors.Is(err, dwn.ErrRateLimited) || + errors.Is(err, dwn.ErrTransport) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) } @@ -51,6 +55,8 @@ func endpointFailureClass(err error) string { return "context" case errors.Is(err, dwn.ErrTransport): return "transport" + case errors.Is(err, errAudienceKeyDeliveryAbsent): + return "key-unavailable" } message := strings.ToLower(err.Error()) if strings.Contains(message, "no delivery record") || @@ -175,11 +181,25 @@ type DWNClient struct { // coalesces concurrent lookups. It has its own lock because record parsing // can run while c.mu is held. deliveredAudienceKeys audienceKeyCache + // roleAudienceKeys memoizes the combined seal-or-delivery resolution per + // tuple, preventing one absent audience query per encrypted record. + roleAudienceKeys audienceKeyCache // loadMu serializes complete state refreshes and blocks snapshot readers // until a refresh either commits or rolls back. loadMu sync.Mutex - mu sync.RWMutex + + // deltaMu protects the last-good raw snapshot and staged topology events. + deltaMu sync.Mutex + rawBaseline *rawMeshRecordSet + pendingTopology []pendingTopologyEvent + pendingTopologyBytes int + topologySequence uint64 + repairSequence uint64 + fullReconciliation bool + mu sync.RWMutex + rawParsedGeneration uint64 + rawParsedOutcomes map[rawParsedOutcomeKey]rawParsedOutcome network *NetworkConfig members map[string]*MemberRecord @@ -190,12 +210,12 @@ type DWNClient struct { // peerEndpoints caches resolved DID → DWN endpoint mappings. peerEndpoints map[string]*PeerEndpointInfo - // undecryptablePeers counts (cumulatively, across all state loads) node - // records whose data payload could not be decrypted — the visible symptom - // of a missing role-audience key delivery (issue #187). Tracked separately - // from droppedPeers because a record can fail to decrypt yet still surface a - // mesh IP via fallback, or decrypt yet lack a usable IP. + // undecryptablePeers counts distinct node failure episodes/classes. + // nodeFailures suppresses repeated projection warnings for unchanged opaque + // records until the record recovers or the failure class changes. The map is + // protected by mu because node parsing already runs under that lock. undecryptablePeers atomic.Int64 + nodeFailures map[string]string // unreadableEndpoints counts endpoint records that could not be parsed or // decrypted. endpointFailures suppresses repeated warnings until recovery; @@ -250,14 +270,14 @@ func NewDWNClient( members: make(map[string]*MemberRecord), nodes: make(map[string]*NodeRecord), peerEndpoints: make(map[string]*PeerEndpointInfo), + nodeFailures: make(map[string]string), endpointFailures: make(map[string]string), } } -// UndecryptablePeerCount returns the cumulative number of node records this -// client has failed to decrypt since it was created. A non-zero, growing value -// means role-audience keys are not reaching this node (issue #187): its peers -// exist in the DWN but cannot be read, so they never enter the mesh map. +// UndecryptablePeerCount returns the number of distinct node failure episodes +// or failure-class transitions since this client was created. A non-zero, +// growing value means records remain unreadable or are failing in new ways. func (c *DWNClient) UndecryptablePeerCount() int64 { return c.undecryptablePeers.Load() } @@ -335,313 +355,589 @@ func (c *DWNClient) restoreState(snapshot dwnClientStateSnapshot) { // Both paths have nodeInfo and endpoint child records. LoadState queries // all paths and merges them into a unified node map keyed by DID. func (c *DWNClient) LoadState(ctx context.Context) (*MapResponse, error) { - c.loadMu.Lock() - previous := c.stateSnapshot() - committed := false - defer func() { - if !committed { - c.restoreState(previous) - } - c.loadMu.Unlock() - }() + return c.LoadStateValidated(ctx, nil) +} - // Determine the protocol role for queries. The anchor (network owner) - // can read as author without a role. Non-anchor nodes use their node - // role. Both network/node and network/member roles grant read access - // to all record types, so we use network/node universally for - // non-anchor reads. - role := c.protocolRole +// LoadStateValidated reads a complete remote snapshot and validates its +// projected response before making the parsed state, raw baseline, or covered +// topology-event prefix durable. The validator runs while loadMu is held and +// must not call methods that acquire loadMu. +func (c *DWNClient) LoadStateValidated(ctx context.Context, validate PendingStateValidator) (*MapResponse, error) { + if ctx == nil { + return nil, fmt.Errorf("loading full state: nil context") + } + if err := ctx.Err(); err != nil { + return nil, err + } - // 1. Read network config. - c.logger.DebugContext(ctx, "reading network record", - slog.String("recordId", c.networkRecordID), - ) + c.loadMu.Lock() + defer c.loadMu.Unlock() + c.invalidateRawParsedOutcomes() + c.deliveredAudienceKeys.invalidateFailures() + c.roleAudienceKeys.invalidateFailures() - netResp, err := c.anchorDWN.RecordsReadWithAuth(ctx, c.anchorTenant, dwn.RecordsFilter{ + through := c.beginFullReconciliation() + budget := &fullStateFetchBudget{} + role := c.protocolRole + fetchStarted := time.Now() + loadCommitted := false + logger := c.logger + if logger == nil { + logger = slog.Default() + } + defer func() { + requests, records, retainedBytes := budget.snapshot() + logger.DebugContext(ctx, "full state reconciliation finished", + slog.Bool("committed", loadCommitted), + slog.Int("requests", requests), + slog.Int("records", records), + slog.Int("retainedBytes", retainedBytes), + slog.Duration("duration", time.Since(fetchStarted)), + ) + }() + if err := budget.takeRequest(); err != nil { + return nil, err + } + networkResult, err := c.anchorDWN.RecordsReadWithAuth(ctx, c.anchorTenant, dwn.RecordsFilter{ RecordID: c.networkRecordID, }, c.readAuth(role)) if err != nil { return nil, fmt.Errorf("reading network: %w", err) } - if netResp.Reply == nil || netResp.Reply.Status.Code != 200 { - code, detail := 0, "nil reply" - if netResp.Reply != nil { - code = netResp.Reply.Status.Code - detail = netResp.Reply.Status.Detail + if networkResult != nil && networkResult.Reply != nil && networkResult.Reply.Status.Code == http.StatusNotFound { + return nil, fmt.Errorf("%w: %d %s", ErrNoNetwork, networkResult.Reply.Status.Code, networkResult.Reply.Status.Detail) + } + if statusErr := rawCaptureReadStatusError(networkResult); statusErr != nil { + if errors.Is(statusErr, dwn.ErrRateLimited) { + return nil, fmt.Errorf("reading network: %w", statusErr) } - return nil, fmt.Errorf("%w: %d %s", ErrNoNetwork, code, detail) + return nil, fmt.Errorf("reading network: %w", errors.Join(dwn.ErrTransport, statusErr)) } - - var network NetworkConfig - // Network record is NOT encrypted (publicly readable anchor). - if err := ParseEntryData(netResp.Reply.Entry, &network, nil); err != nil { - return nil, fmt.Errorf("parsing network: %w", err) + networkEntry, err := rawMeshRecordReadEntry(networkResult) + if err != nil { + return nil, fmt.Errorf("reading network entry: %w", err) + } + if err := budget.retain([]json.RawMessage{networkEntry}, nil); err != nil { + return nil, fmt.Errorf("retaining network entry: %w", err) + } + candidate, err := newRawMeshRecordSet([]json.RawMessage{networkEntry}, "network") + if err != nil { + return nil, fmt.Errorf("normalizing network entry: %w", err) } - c.mu.Lock() - c.network = &network - c.mu.Unlock() + queryPath := func(protocolPath, contextID, dateSort string) error { + entries, err := c.queryAllRawMeshRecords(ctx, dwn.RecordsFilter{ + Protocol: protocols.MeshProtocolURI, + ProtocolPath: protocolPath, + ContextID: contextID, + }, dateSort, role, budget) + if err != nil { + return fmt.Errorf("querying %s under %s: %w", protocolPath, contextID, err) + } + if _, err := candidate.addEntries(entries, protocolPath); err != nil { + return fmt.Errorf("normalizing %s under %s: %w", protocolPath, contextID, err) + } + return nil + } + queryGroup := func(specs []fullStateQuerySpec) error { + batches, err := c.queryRawMeshRecordGroup(ctx, specs, role, budget) + if err != nil { + return err + } + for i, entries := range batches { + spec := specs[i] + if _, err := candidate.addEntries(entries, spec.protocolPath); err != nil { + return fmt.Errorf("normalizing %s under %s: %w", spec.protocolPath, spec.contextID, err) + } + } + return nil + } - // 2. Query owner-provisioned node records (network/node). - c.logger.DebugContext(ctx, "querying owner-provisioned nodes") + if err := queryPath("network/node", c.networkRecordID, "createdAscending"); err != nil { + return nil, err + } + if err := queryPath("network/member", c.networkRecordID, "createdAscending"); err != nil { + return nil, err + } + memberContexts := fullStateMemberContexts(candidate, c.networkRecordID) + memberQueries := make([]fullStateQuerySpec, len(memberContexts)) + for i, memberContext := range memberContexts { + memberQueries[i] = fullStateQuerySpec{ + protocolPath: "network/member/node", contextID: memberContext, dateSort: "createdAscending", + } + } + if err := queryGroup(memberQueries); err != nil { + return nil, err + } + if err := queryPath("network/relay", c.networkRecordID, "createdAscending"); err != nil { + return nil, err + } + if err := queryPath("network/aclPolicy", c.networkRecordID, "createdDescending"); err != nil { + return nil, err + } + nodeContexts := fullStateNodeContexts(candidate, c.networkRecordID) + nodeQueries := make([]fullStateQuerySpec, 0, 2*len(nodeContexts)) + for _, node := range nodeContexts { + nodeQueries = append(nodeQueries, + fullStateQuerySpec{protocolPath: node.infoPath, contextID: node.contextID, dateSort: "createdDescending"}, + fullStateQuerySpec{protocolPath: node.endpointPath, contextID: node.contextID, dateSort: "createdDescending"}, + ) + } + if err := queryGroup(nodeQueries); err != nil { + return nil, err + } - nodesResp, err := c.anchorDWN.RecordsQueryWithAuth(ctx, c.anchorTenant, dwn.RecordsFilter{ - Protocol: protocols.MeshProtocolURI, - ProtocolPath: "network/node", - ContextID: c.networkRecordID, - }, "createdAscending", nil, c.readAuth(role)) + projection, err := c.projectRawMeshRecordSetWithDecryptors(ctx, candidate, c.makeDecryptor) if err != nil { - return nil, fmt.Errorf("querying nodes: %w", err) + return nil, fmt.Errorf("projecting full state: %w", err) + } + if validate != nil { + if err := validate(projection.response); err != nil { + return projection.response, err + } } + if err := c.completeFullReconciliation(ctx, candidate, projection, through); err != nil { + return projection.response, err + } + loadCommitted = true + return projection.response, nil +} - nodeEntries, err := dwn.QueryResult(nodesResp) - if err != nil { - return nil, fmt.Errorf("parsing nodes: %w", err) +const ( + fullStateQueryPageSize = 256 + fullStateQueryWorkers = 8 + fullStateMaxQueryPages = 64 + fullStateMaxRecords = 10_000 + fullStateMaxBytes = 64 << 20 + fullStateMaxRequests = 25_000 +) + +type fullStateFetchBudget struct { + mu sync.Mutex + requests int + records int + bytes int +} + +func (b *fullStateFetchBudget) takeRequest() error { + b.mu.Lock() + defer b.mu.Unlock() + if b.requests >= fullStateMaxRequests { + return fmt.Errorf("full-state request limit exceeded: %d", fullStateMaxRequests) } + b.requests++ + return nil +} - nodeDecryptor := c.makeDecryptor(ctx, "network/node") - var nodeLoadErr error - c.mu.Lock() - clear(c.nodes) // Clear stale nodes before repopulating. - for _, entry := range nodeEntries { - if err := c.loadNodeEntry(ctx, entry, nodeDecryptor, ""); shouldAbortStateLoad(ctx, err) { - nodeLoadErr = err - break +func (b *fullStateFetchBudget) retain(entries []json.RawMessage, cursor json.RawMessage) error { + b.mu.Lock() + defer b.mu.Unlock() + if len(entries) > fullStateMaxRecords-b.records { + return fmt.Errorf("full-state record limit exceeded: %d", fullStateMaxRecords) + } + retainedBytes := len(cursor) + for _, entry := range entries { + if len(entry) > fullStateMaxBytes-retainedBytes { + return fmt.Errorf("full-state byte limit exceeded: %d", fullStateMaxBytes) } + retainedBytes += len(entry) } - c.mu.Unlock() - if nodeLoadErr != nil { - return nil, fmt.Errorf("decrypting nodes: %w", nodeLoadErr) + if retainedBytes > fullStateMaxBytes-b.bytes { + return fmt.Errorf("full-state byte limit exceeded: %d", fullStateMaxBytes) } + b.records += len(entries) + b.bytes += retainedBytes + return nil +} - c.logger.DebugContext(ctx, "loaded owner-provisioned nodes", slog.Int("count", len(nodeEntries))) +func (b *fullStateFetchBudget) snapshot() (requests, records, retainedBytes int) { + b.mu.Lock() + defer b.mu.Unlock() + return b.requests, b.records, b.bytes +} - // 3. Query member records (network/member) to discover members. - c.logger.DebugContext(ctx, "querying members") +func canonicalFullStateCursor(raw json.RawMessage) (json.RawMessage, string, error) { + if len(raw) == 0 { + return nil, "", nil + } + var compact bytes.Buffer + if err := json.Compact(&compact, raw); err != nil { + return nil, "", fmt.Errorf("decoding pagination cursor: %w", err) + } + canonical := cloneRawJSON(compact.Bytes()) + if bytes.Equal(canonical, []byte("null")) { + return nil, "", nil + } + return canonical, string(canonical), nil +} - membersReady := true - membersResp, err := c.anchorDWN.RecordsQueryWithAuth(ctx, c.anchorTenant, dwn.RecordsFilter{ - Protocol: protocols.MeshProtocolURI, - ProtocolPath: "network/member", - ContextID: c.networkRecordID, - }, "createdAscending", nil, c.readAuth(role)) - if err != nil { - if shouldAbortStateLoad(ctx, err) { - return nil, fmt.Errorf("querying members: %w", err) +func fullStateMemberContexts(set *rawMeshRecordSet, networkRecordID string) []string { + seen := make(map[string]struct{}) + for _, record := range set.all() { + if record.protocol != protocols.MeshProtocolURI || record.protocolPath != "network/member" || + !rawRecordIsDirectChild(record, networkRecordID) { + continue } - c.logger.DebugContext(ctx, "querying members failed", slog.Any("error", err)) - membersReady = false + seen[record.contextID] = struct{}{} } - var memberEntries []json.RawMessage - if membersReady { - memberEntries, err = dwn.QueryResult(membersResp) - if err != nil { - if shouldAbortStateLoad(ctx, err) { - return nil, fmt.Errorf("parsing members: %w", err) + contexts := make([]string, 0, len(seen)) + for contextID := range seen { + contexts = append(contexts, contextID) + } + sort.Strings(contexts) + return contexts +} + +type fullStateNodeContext struct { + contextID string + infoPath string + endpointPath string +} + +func fullStateNodeContexts(set *rawMeshRecordSet, networkRecordID string) []fullStateNodeContext { + memberContexts := make(map[string]struct{}) + for _, contextID := range fullStateMemberContexts(set, networkRecordID) { + memberContexts[contextID] = struct{}{} + } + seen := make(map[string]fullStateNodeContext) + for _, record := range set.all() { + if record.protocol != protocols.MeshProtocolURI { + continue + } + var node fullStateNodeContext + switch record.protocolPath { + case "network/node": + if !rawRecordIsDirectChild(record, networkRecordID) { + continue } - c.logger.DebugContext(ctx, "parsing member results", slog.Any("error", err)) - membersReady = false - } - } - - if membersReady { - memberDecryptor := c.makeDecryptor(ctx, "network/member") - var memberLoadErr error - c.mu.Lock() - clear(c.members) // Clear stale members before repopulating. - for _, entry := range memberEntries { - meta := extractEntryMetadata(entry) - memberDID := meta.Recipient - - var member MemberRecord - if err := ParseEntryData(entry, &member, memberDecryptor); err != nil { - c.logger.DebugContext(ctx, "parsing member entry", - slog.Any("error", err), - slog.String("memberDID", memberDID), - ) - if shouldAbortStateLoad(ctx, err) { - memberLoadErr = err - break - } - // Track a permanently unreadable member from its public metadata so - // later key delivery can still target it. - if memberDID != "" { - member.DID = memberDID - member.RecordID = meta.RecordID - c.members[memberDID] = &member - } + node = fullStateNodeContext{ + contextID: record.contextID, infoPath: "network/node/nodeInfo", endpointPath: "network/node/endpoint", + } + case "network/member/node": + if _, ok := memberContexts[record.parentContextID]; !ok || !rawRecordIsDirectChild(record, record.parentContextID) { continue } - member.DID = memberDID - member.RecordID = meta.RecordID - if memberDID != "" { - c.members[memberDID] = &member + node = fullStateNodeContext{ + contextID: record.contextID, infoPath: "network/member/node/nodeInfo", endpointPath: "network/member/node/endpoint", } + default: + continue } - c.mu.Unlock() - if memberLoadErr != nil { - return nil, fmt.Errorf("decrypting members: %w", memberLoadErr) - } - - c.logger.DebugContext(ctx, "loaded members", slog.Int("count", len(memberEntries))) + seen[node.infoPath+"\x00"+node.contextID] = node + } + nodes := make([]fullStateNodeContext, 0, len(seen)) + for _, node := range seen { + nodes = append(nodes, node) } + sort.Slice(nodes, func(i, j int) bool { + if nodes[i].infoPath != nodes[j].infoPath { + return nodes[i].infoPath < nodes[j].infoPath + } + return nodes[i].contextID < nodes[j].contextID + }) + return nodes +} - // 4. Query member-associated node records (network/member/node). - c.logger.DebugContext(ctx, "querying member nodes") +type fullStatePageQuery func(context.Context, *dwn.Pagination) (*dwn.DwnReply, error) + +func (c *DWNClient) queryAllRawMeshRecords( + ctx context.Context, + filter dwn.RecordsFilter, + dateSort string, + role string, + budget *fullStateFetchBudget, +) ([]json.RawMessage, error) { + return c.queryAllRawMeshRecordsWith(ctx, filter, role, budget, func(ctx context.Context, pagination *dwn.Pagination) (*dwn.DwnReply, error) { + return c.anchorDWN.RecordsQueryWithAuth( + ctx, c.anchorTenant, filter, dateSort, pagination, c.readAuth(role), + ) + }) +} - memberNodeCount := 0 - for _, query := range c.memberNodeParentQueries() { - memberNodesResp, err := c.anchorDWN.RecordsQueryWithAuth(ctx, c.anchorTenant, dwn.RecordsFilter{ - Protocol: protocols.MeshProtocolURI, - ProtocolPath: "network/member/node", - ContextID: query.ParentContextID, - }, "createdAscending", nil, c.readAuth(role)) +func (c *DWNClient) queryAllRawMeshRecordsWith( + ctx context.Context, + filter dwn.RecordsFilter, + role string, + budget *fullStateFetchBudget, + query fullStatePageQuery, +) ([]json.RawMessage, error) { + var entries []json.RawMessage + var cursor json.RawMessage + seenCursors := make(map[string]struct{}) + for page := 0; page < fullStateMaxQueryPages; page++ { + if err := budget.takeRequest(); err != nil { + return nil, err + } + reply, err := query(ctx, &dwn.Pagination{Limit: fullStateQueryPageSize, Cursor: cloneRawJSON(cursor)}) if err != nil { - if shouldAbortStateLoad(ctx, err) { - return nil, fmt.Errorf("querying member nodes for %s: %w", query.MemberRecordID, err) - } - c.logger.DebugContext(ctx, "querying member nodes failed", - slog.String("memberRecordId", query.MemberRecordID), - slog.Any("error", err), - ) - continue + return nil, fmt.Errorf("requesting page %d: %w", page+1, err) } - - memberNodeEntries, err := dwn.QueryResult(memberNodesResp) + pageEntries, err := dwn.QueryEntries(reply) if err != nil { - if shouldAbortStateLoad(ctx, err) { - return nil, fmt.Errorf("parsing member nodes for %s: %w", query.MemberRecordID, err) + if errors.Is(err, dwn.ErrRateLimited) { + return nil, fmt.Errorf("reading page %d: %w", page+1, err) } - c.logger.DebugContext(ctx, "parsing member node results", - slog.String("memberRecordId", query.MemberRecordID), - slog.Any("error", err), - ) - continue - } - - memberNodeDecryptor := c.makeDecryptor(ctx, "network/member/node") - var memberNodeLoadErr error - c.mu.Lock() - for _, entry := range memberNodeEntries { - if err := c.loadNodeEntry(ctx, entry, memberNodeDecryptor, query.MemberRecordID); shouldAbortStateLoad(ctx, err) { - memberNodeLoadErr = err - break + if reply == nil || reply.Status.Code != http.StatusOK { + err = errors.Join(dwn.ErrTransport, err) } + return nil, fmt.Errorf("reading page %d: %w", page+1, err) } - c.mu.Unlock() - if memberNodeLoadErr != nil { - return nil, fmt.Errorf("decrypting member nodes for %s: %w", query.MemberRecordID, memberNodeLoadErr) + hydrated, err := c.hydrateRawCaptureEntries(ctx, pageEntries, filter.ProtocolPath, filter.ContextID, role, budget) + if err != nil { + return nil, fmt.Errorf("hydrating page %d: %w", page+1, err) + } + nextCursor, cursorKey, err := canonicalFullStateCursor(reply.Cursor) + if err != nil { + return nil, fmt.Errorf("page %d: %w", page+1, err) } - memberNodeCount += len(memberNodeEntries) + if err := budget.retain(hydrated, nextCursor); err != nil { + return nil, fmt.Errorf("page %d: %w", page+1, err) + } + entries = append(entries, hydrated...) + if cursorKey == "" { + return entries, nil + } + if _, duplicate := seenCursors[cursorKey]; duplicate { + return nil, fmt.Errorf("page %d repeated pagination cursor", page+1) + } + seenCursors[cursorKey] = struct{}{} + cursor = nextCursor } - c.logger.DebugContext(ctx, "loaded member nodes", slog.Int("count", memberNodeCount)) + return nil, fmt.Errorf("query page limit exceeded: %d", fullStateMaxQueryPages) +} - // 5. Query relay records. - relayResp, err := c.anchorDWN.RecordsQueryWithAuth(ctx, c.anchorTenant, dwn.RecordsFilter{ - Protocol: protocols.MeshProtocolURI, - ProtocolPath: "network/relay", - ContextID: c.networkRecordID, - }, "createdAscending", nil, c.readAuth(role)) - if err != nil { - return nil, fmt.Errorf("querying relays: %w", err) +type fullStateQuerySpec struct { + protocolPath string + contextID string + dateSort string +} + +func (c *DWNClient) queryRawMeshRecordGroup( + ctx context.Context, + specs []fullStateQuerySpec, + role string, + budget *fullStateFetchBudget, +) ([][]json.RawMessage, error) { + if len(specs) == 0 { + return nil, nil + } + groupCtx, cancel := context.WithCancel(ctx) + defer cancel() + results := make([][]json.RawMessage, len(specs)) + jobs := make(chan int, len(specs)) + for i := range specs { + jobs <- i + } + close(jobs) + + workerCount := min(fullStateQueryWorkers, len(specs)) + var workers sync.WaitGroup + var firstErr error + var errorOnce sync.Once + for range workerCount { + workers.Add(1) + go func() { + defer workers.Done() + for index := range jobs { + if groupCtx.Err() != nil { + return + } + spec := specs[index] + entries, err := c.queryAllRawMeshRecords(groupCtx, dwn.RecordsFilter{ + Protocol: protocols.MeshProtocolURI, ProtocolPath: spec.protocolPath, ContextID: spec.contextID, + }, spec.dateSort, role, budget) + if err != nil { + errorOnce.Do(func() { + firstErr = fmt.Errorf("querying %s under %s: %w", spec.protocolPath, spec.contextID, err) + cancel() + }) + return + } + results[index] = entries + } + }() + } + workers.Wait() + if firstErr != nil { + return nil, firstErr } + if err := ctx.Err(); err != nil { + return nil, err + } + return results, nil +} - relayEntries, err := dwn.QueryResult(relayResp) +func rawMeshRecordReadEntry(result *dwn.RecordsReadResult) (json.RawMessage, error) { + if err := rawCaptureReadStatusError(result); err != nil { + return nil, err + } + entry, err := dwn.ReadEntry(result.Reply) + if err != nil { + return nil, err + } + encodedData := "" + if len(result.Data) != 0 && !rawRecordHasEncodedData(entry) { + encodedData = base64.RawURLEncoding.EncodeToString(result.Data) + } + entry, err = injectSubscriptionEncodedData(entry, encodedData) if err != nil { - return nil, fmt.Errorf("parsing relays: %w", err) + return nil, err } + if !rawRecordHasEncodedData(entry) { + return nil, fmt.Errorf("RecordsRead result has no encoded data") + } + return entry, nil +} - relayDecryptor := c.makeDecryptor(ctx, "network/relay") - var relayLoadErr error - c.mu.Lock() - c.relays = nil // Clear stale relays before repopulating. - for _, entry := range relayEntries { - var relay RelayData - if err := ParseEntryData(entry, &relay, relayDecryptor); err != nil { - c.logger.DebugContext(ctx, "parsing relay entry", slog.Any("error", err)) - if shouldAbortStateLoad(ctx, err) { - relayLoadErr = err - break - } +type rawCaptureRecordIdentity struct { + recordID string + protocol string + protocolPath string + contextID string + parentContextID string + parentID string + recipient string + dateCreated string + messageTimestamp string + revision time.Time + messageCID string +} + +// hydrateRawCaptureEntries fills query entries whose data was omitted because +// it exceeded the DWN inline-data limit. The targeted read is authenticated in +// exactly the same way as the query. Hydration is all-or-nothing: callers keep +// the original query slice when any read or validation fails. +func (c *DWNClient) hydrateRawCaptureEntries(ctx context.Context, entries []json.RawMessage, pathHint, contextID, role string, budget *fullStateFetchBudget) ([]json.RawMessage, error) { + var hydrated []json.RawMessage + for i, entry := range entries { + if rawRecordHasEncodedData(entry) { continue } - c.relays = append(c.relays, &relay) - } - c.mu.Unlock() - if relayLoadErr != nil { - return nil, fmt.Errorf("decrypting relays: %w", relayLoadErr) - } - // 6. Query ACL policy record. - aclReady := true - aclResp, err := c.anchorDWN.RecordsQueryWithAuth(ctx, c.anchorTenant, dwn.RecordsFilter{ - Protocol: protocols.MeshProtocolURI, - ProtocolPath: "network/aclPolicy", - ContextID: c.networkRecordID, - }, "createdDescending", nil, c.readAuth(role)) - if err != nil { - if shouldAbortStateLoad(ctx, err) { - return nil, fmt.Errorf("querying ACL policy: %w", err) + expected, err := rawCaptureIdentity(entry) + if err != nil { + return nil, fmt.Errorf("identifying %s query entry %d: %w", pathHint, i, err) } - c.logger.DebugContext(ctx, "querying ACL policy failed", slog.Any("error", err)) - aclReady = false - } - var aclEntries []json.RawMessage - if aclReady { - aclEntries, err = dwn.QueryResult(aclResp) + if expected.protocol != protocols.MeshProtocolURI || expected.protocolPath != pathHint || expected.parentContextID != contextID { + return nil, fmt.Errorf("query entry %s does not match requested protocol/path %s %s", expected.recordID, protocols.MeshProtocolURI, pathHint) + } + + if err := budget.takeRequest(); err != nil { + return nil, err + } + result, err := c.anchorDWN.RecordsReadWithAuth(ctx, c.anchorTenant, dwn.RecordsFilter{ + RecordID: expected.recordID, + }, c.readAuth(role)) if err != nil { - if shouldAbortStateLoad(ctx, err) { - return nil, fmt.Errorf("parsing ACL policy results: %w", err) - } - c.logger.DebugContext(ctx, "parsing ACL policy results", slog.Any("error", err)) - aclReady = false + return nil, fmt.Errorf("hydrating %s record %s: %w", pathHint, expected.recordID, err) } - } - if aclReady && len(aclEntries) > 0 { - // ACL policies are squashed snapshots; take the newest visible entry. - aclDecryptor := c.makeDecryptor(ctx, "network/aclPolicy") - var policy ACLPolicyData - if err := ParseEntryData(aclEntries[0], &policy, aclDecryptor); err != nil { - if shouldAbortStateLoad(ctx, err) { - return nil, fmt.Errorf("decrypting ACL policy: %w", err) + if err := rawCaptureReadStatusError(result); err != nil { + if !errors.Is(err, dwn.ErrRateLimited) && !errors.Is(err, dwn.ErrTransport) && + !errors.Is(err, errRawCaptureReadNotFound) { + err = errors.Join(dwn.ErrTransport, err) } - c.logger.DebugContext(ctx, "parsing ACL policy entry", slog.Any("error", err)) - } else { - c.mu.Lock() - c.acl = &policy - c.mu.Unlock() - c.logger.DebugContext(ctx, "loaded ACL policy", - slog.Int("version", policy.Version), - slog.Int("rules", len(policy.Rules)), - ) + return nil, fmt.Errorf("hydrating %s record %s: %w", pathHint, expected.recordID, err) + } + readEntry, err := rawMeshRecordReadEntry(result) + if err != nil { + return nil, fmt.Errorf("hydrating %s record %s: %w", pathHint, expected.recordID, err) + } + actual, err := rawCaptureIdentity(readEntry) + if err != nil { + return nil, fmt.Errorf("validating hydrated %s record %s: %w", pathHint, expected.recordID, err) + } + if !sameRawCaptureSlot(actual, expected) { + return nil, fmt.Errorf("hydrated %s record %s moved to a different immutable slot", pathHint, expected.recordID) + } + if compareRawCaptureRevision(actual, expected) < 0 { + return nil, fmt.Errorf("hydrated %s record %s is older than the queried revision", pathHint, expected.recordID) } - } - // 7. Query nodeInfo records under each node's direct parent context. - if err := c.loadNodeChildRecords(ctx, "nodeInfo", role, c.loadNodeInfoEntry); err != nil { - return nil, fmt.Errorf("loading node info: %w", err) + if hydrated == nil { + hydrated = append([]json.RawMessage(nil), entries...) + } + hydrated[i] = readEntry } + if hydrated == nil { + return entries, nil + } + return hydrated, nil +} - // 8. Query endpoint records under each node's direct parent context. - if err := c.loadNodeChildRecords(ctx, "endpoint", role, c.loadEndpointEntry); err != nil { - return nil, fmt.Errorf("loading endpoints: %w", err) +func rawCaptureReadStatusError(result *dwn.RecordsReadResult) error { + if result == nil || result.Reply == nil { + return fmt.Errorf("%w: empty RecordsRead result", dwn.ErrTransport) + } + status := result.Reply.Status + if status.Code == http.StatusTooManyRequests { + return &dwn.RateLimitError{RetryAfter: rawCaptureRetryAfter(status.Detail), Detail: status.Detail} + } + if status.Code == http.StatusNotFound { + return fmt.Errorf("%w: %d %s", errRawCaptureReadNotFound, status.Code, status.Detail) } + if status.Code != http.StatusOK { + return fmt.Errorf("%w: read failed: %d %s", dwn.ErrTransport, status.Code, status.Detail) + } + return nil +} - hasACL := c.acl != nil - c.logger.DebugContext(ctx, "mesh state loaded", - slog.String("network", network.Name), - slog.Int("nodes", len(c.nodes)), - slog.Int("members", len(c.members)), - slog.Int("relays", len(c.relays)), - slog.Bool("aclPolicy", hasACL), - ) +func sameRawCaptureSlot(a, b rawCaptureRecordIdentity) bool { + return a.recordID == b.recordID && + a.protocol == b.protocol && + a.protocolPath == b.protocolPath && + a.contextID == b.contextID && + a.parentContextID == b.parentContextID && + a.parentID == b.parentID && + a.recipient == b.recipient && + a.dateCreated == b.dateCreated +} + +func compareRawCaptureRevision(a, b rawCaptureRecordIdentity) int { + if a.revision.Before(b.revision) { + return -1 + } + if a.revision.After(b.revision) { + return 1 + } + return strings.Compare(a.messageCID, b.messageCID) +} - resp := c.buildMapResponse() - if resp == nil { - return nil, fmt.Errorf("self DID %q not found in network node records", c.selfDID) +func rawCaptureRetryAfter(detail string) time.Duration { + const marker = "retry after " + lower := strings.ToLower(detail) + index := strings.LastIndex(lower, marker) + if index < 0 { + return time.Second } - committed = true - return resp, nil + fields := strings.Fields(strings.TrimSpace(detail[index+len(marker):])) + if len(fields) == 0 { + return time.Second + } + delay, err := time.ParseDuration(strings.TrimRight(fields[0], ".,;")) + if err != nil || delay < 0 { + return time.Second + } + return delay +} + +func rawCaptureIdentity(entry json.RawMessage) (rawCaptureRecordIdentity, error) { + record, err := normalizeRawMeshRecordIdentity(entry, "") + if err != nil { + return rawCaptureRecordIdentity{}, err + } + return rawCaptureRecordIdentity{ + recordID: record.recordID, + protocol: record.protocol, + protocolPath: record.protocolPath, + contextID: record.contextID, + parentContextID: record.parentContextID, + parentID: record.parentID, + recipient: record.recipient, + dateCreated: record.dateCreated, + messageTimestamp: record.messageTimestamp, + revision: record.revision, + messageCID: record.messageCID, + }, nil } type memberNodeParentQuery struct { @@ -676,19 +972,28 @@ func (c *DWNClient) loadNodeEntry(ctx context.Context, entry json.RawMessage, de meta := extractEntryMetadata(entry) nodeDID := meta.Recipient + failureKey := meta.RecordID + if failureKey == "" { + failureKey = nodeDID + } + var node NodeRecord if err := ParseEntryData(entry, &node, decryptor); err != nil { - // An undecryptable peer node record is the visible symptom of a - // missing role-audience key delivery (issue #187): the record exists - // but this node was never handed the key to read it. Surface it at - // Warn (not Debug) and count it so operators can see peers silently - // dropping out of the mesh. - c.undecryptablePeers.Add(1) - c.logger.WarnContext(ctx, "node record could not be decrypted; peer will be invisible until a role-audience key is delivered", - slog.Any("error", err), - slog.String("nodeDID", nodeDID), - slog.String("memberRecordId", memberRecordID), - ) + if c.nodeFailures == nil { + c.nodeFailures = make(map[string]string) + } + failureClass := endpointFailureClass(err) + if previous, warned := c.nodeFailures[failureKey]; !warned || previous != failureClass { + c.undecryptablePeers.Add(1) + c.logger.WarnContext(ctx, "node record could not be loaded; peer will be invisible until key delivery or record recovery", + slog.Any("error", err), + slog.String("failureClass", failureClass), + slog.String("nodeDID", nodeDID), + slog.String("recordId", meta.RecordID), + slog.String("memberRecordId", memberRecordID), + ) + } + c.nodeFailures[failureKey] = failureClass // Even if we can't decrypt the data payload, track the node DID // from the unencrypted recipient field. This allows peer discovery // and auto key delivery to work even before context key exchange. @@ -696,6 +1001,7 @@ func (c *DWNClient) loadNodeEntry(ctx context.Context, entry json.RawMessage, de node.DID = nodeDID node.RecordID = meta.RecordID node.MemberRecordID = memberRecordID + node.Opaque = true c.nodes[nodeDID] = &node } return err @@ -707,6 +1013,14 @@ func (c *DWNClient) loadNodeEntry(ctx context.Context, entry json.RawMessage, de if nodeDID != "" { c.nodes[nodeDID] = &node } + if _, recovering := c.nodeFailures[failureKey]; recovering { + delete(c.nodeFailures, failureKey) + c.logger.InfoContext(ctx, "node record is readable again", + slog.String("nodeDID", nodeDID), + slog.String("recordId", meta.RecordID), + slog.String("memberRecordId", memberRecordID), + ) + } return nil } @@ -749,10 +1063,16 @@ func (c *DWNClient) nodeChildRecordQueries(childType string) []nodeChildRecordQu type nodeChildRecordHandler func(ctx context.Context, entry json.RawMessage, decryptor EntryDecryptor) error +type nodeChildRecordCapture func(protocolPath string, entries []json.RawMessage, err error) error + func (c *DWNClient) loadNodeChildRecords(ctx context.Context, childType string, role string, handler nodeChildRecordHandler) error { + return c.loadNodeChildRecordsWithCapture(ctx, childType, role, handler, nil) +} + +func (c *DWNClient) loadNodeChildRecordsWithCapture(ctx context.Context, childType string, role string, handler nodeChildRecordHandler, capture nodeChildRecordCapture) error { total := 0 for _, query := range c.nodeChildRecordQueries(childType) { - count, err := c.loadChildRecords(ctx, query.ProtocolPath, query.ParentContextID, role, handler) + count, err := c.loadChildRecordsWithCapture(ctx, query.ProtocolPath, query.ParentContextID, role, handler, capture) if err != nil { return fmt.Errorf("%s under %s: %w", query.ProtocolPath, query.ParentContextID, err) } @@ -768,6 +1088,10 @@ func (c *DWNClient) loadNodeChildRecords(ctx context.Context, childType string, // loadChildRecords queries child records at the given protocol path under a // direct parent context and processes each entry with the provided handler. func (c *DWNClient) loadChildRecords(ctx context.Context, protocolPath string, parentContextID string, role string, handler nodeChildRecordHandler) (int, error) { + return c.loadChildRecordsWithCapture(ctx, protocolPath, parentContextID, role, handler, nil) +} + +func (c *DWNClient) loadChildRecordsWithCapture(ctx context.Context, protocolPath string, parentContextID string, role string, handler nodeChildRecordHandler, capture nodeChildRecordCapture) (int, error) { resp, err := c.anchorDWN.RecordsQueryWithAuth(ctx, c.anchorTenant, dwn.RecordsFilter{ Protocol: protocols.MeshProtocolURI, ProtocolPath: protocolPath, @@ -777,6 +1101,11 @@ func (c *DWNClient) loadChildRecords(ctx context.Context, protocolPath string, p if shouldAbortStateLoad(ctx, err) { return 0, fmt.Errorf("querying child records: %w", err) } + if capture != nil { + if captureErr := capture(protocolPath, nil, err); captureErr != nil { + return 0, captureErr + } + } c.logger.DebugContext(ctx, "querying child records failed", slog.String("path", protocolPath), slog.String("parentContextId", parentContextID), @@ -790,6 +1119,11 @@ func (c *DWNClient) loadChildRecords(ctx context.Context, protocolPath string, p if shouldAbortStateLoad(ctx, err) { return 0, fmt.Errorf("parsing child record results: %w", err) } + if capture != nil { + if captureErr := capture(protocolPath, nil, err); captureErr != nil { + return 0, captureErr + } + } c.logger.DebugContext(ctx, "parsing child record results", slog.String("path", protocolPath), slog.String("parentContextId", parentContextID), @@ -798,6 +1132,12 @@ func (c *DWNClient) loadChildRecords(ctx context.Context, protocolPath string, p return 0, nil } + if capture != nil { + if captureErr := capture(protocolPath, entries, nil); captureErr != nil { + return 0, captureErr + } + } + decryptor := c.makeDecryptor(ctx, protocolPath) var loadErr error c.mu.Lock() @@ -845,9 +1185,9 @@ func (c *DWNClient) loadNodeInfoEntry(ctx context.Context, entry json.RawMessage // Caller must hold c.mu. func (c *DWNClient) loadEndpointEntry(ctx context.Context, entry json.RawMessage, decryptor EntryDecryptor) error { meta := extractEntryMetadata(entry) - failureKey := meta.RecordID + failureKey := meta.ParentID if failureKey == "" { - failureKey = meta.ParentID + failureKey = meta.RecordID } var parentNode *NodeRecord @@ -974,6 +1314,11 @@ func (c *DWNClient) buildMapResponse() *MapResponse { DERPMap: c.buildDERPMap(), DNSConfig: c.buildDNSConfig(), } + if c.acl != nil { + resp.PacketFilter = c.buildFilterRules() + } else { + resp.PacketFilter = defaultFilterRules() + } // Keep response ordering deterministic for logs, status output, and tests. // Node IDs themselves are derived from each DID below; assigning IDs from @@ -988,14 +1333,36 @@ func (c *DWNClient) buildMapResponse() *MapResponse { now := time.Now().UTC() for _, did := range dids { rec := c.nodes[did] - if nodeRecordExpired(rec, now) { + if rec == nil { + continue + } + if rec.Opaque { if did == c.selfDID { - c.logger.Debug("self node membership is expired", - slog.String("did", did), - slog.String("expiresAt", rec.ExpiresAt), - ) + c.logger.Warn("self node record is opaque; refusing descriptor-only network identity", + slog.String("did", did), slog.String("recordId", rec.RecordID)) return nil } + c.droppedPeers.Add(1) + c.logger.Warn("dropping opaque peer before fallback identity derivation", + slog.String("did", did), slog.String("recordId", rec.RecordID)) + continue + } + expired := nodeRecordExpired(rec, now) + if did == c.selfDID && (rec.Revoked || expired) { + c.logger.Debug("self node membership is inactive", + slog.String("did", did), + slog.String("expiresAt", rec.ExpiresAt), + slog.Bool("revoked", rec.Revoked), + ) + nodeID, stableID := nodeIdentityForDID(c.networkRecordID, did) + node := nodeRecordToNode(nodeID, did, rec) + node.StableID = stableID + c.applyFallbackMeshIP(node) + resp.Node = node + resp.Peers = nil + return resp + } + if expired || rec.Revoked { c.logger.Debug("skipping expired peer in network map", slog.String("did", did), slog.String("expiresAt", rec.ExpiresAt), @@ -1033,12 +1400,6 @@ func (c *DWNClient) buildMapResponse() *MapResponse { return nil } - if c.acl != nil { - resp.PacketFilter = c.buildFilterRules() - } else { - resp.PacketFilter = defaultFilterRules() - } - return resp } @@ -1067,7 +1428,7 @@ func nodeRecordExpired(rec *NodeRecord, now time.Time) bool { if err != nil { return false } - return now.After(expiresAt) + return !now.Before(expiresAt) } func (c *DWNClient) applyFallbackMeshIP(node *Node) { @@ -1119,7 +1480,7 @@ func nodeRecordToNodeWithThreshold(id int64, nodeDID string, rec *NodeRecord, st if rec.Info != nil { node.Name = rec.Info.Hostname node.OS = rec.Info.OS - node.Capabilities = rec.Info.Capabilities + node.Capabilities = append([]string(nil), rec.Info.Capabilities...) } // Fall back to node label if no hostname from nodeInfo. @@ -1275,7 +1636,7 @@ func (c *DWNClient) buildDNSConfig() *DNSConfig { } return &DNSConfig{ MagicDNSSuffix: suffix, - Resolvers: c.network.DNSServers, + Resolvers: append([]string(nil), c.network.DNSServers...), } } @@ -1684,46 +2045,66 @@ func (c *DWNClient) decryptRoleAudience(ctx context.Context, ciphertext []byte, var errs []error for _, info := range infos { - if c.audienceSource != nil { - audiencePriv, err := c.audienceSource.AudiencePrivateKeyByKeyID(ctx, info.Protocol, info.RolePath, info.KeyID) - if err == nil { - dec, err := dwncrypto.NewRoleAudienceDecrypter(audiencePriv) - clear(audiencePriv) - if err != nil { - return nil, err - } - plaintext, err := dec.Decrypt(ciphertext, enc) - dec.Close() - if err == nil { - return plaintext, nil - } - errs = append(errs, fmt.Errorf("%s seal decrypt: %w", info.RolePath, err)) - } else { - errs = append(errs, fmt.Errorf("%s seal: %w", info.RolePath, err)) - } + privateKey, err := c.roleAudiencePrivateKey(ctx, info) + if err != nil { + errs = append(errs, fmt.Errorf("%s audience key: %w", info.RolePath, err)) + continue } - - plaintext, err := c.decryptViaDelivery(ctx, ciphertext, enc, info) + dec, err := dwncrypto.NewRoleAudienceDecrypter(privateKey) + clear(privateKey) + if err != nil { + return nil, err + } + plaintext, err := dec.Decrypt(ciphertext, enc) + dec.Close() if err == nil { return plaintext, nil } - errs = append(errs, fmt.Errorf("%s delivery: %w", info.RolePath, err)) + errs = append(errs, fmt.Errorf("%s audience decrypt: %w", info.RolePath, err)) } return nil, fmt.Errorf("role audience key unavailable: %w", errors.Join(errs...)) } +func stableRoleAudienceRouteFailure(err error) bool { + return errors.Is(err, errAudienceRecordAbsent) || + errors.Is(err, errAudienceSealUnavailable) || + errors.Is(err, errAudienceDeliveryUnavailable) || + errors.Is(err, errAudienceKeyDeliveryAbsent) +} + +func (c *DWNClient) roleAudiencePrivateKey(ctx context.Context, info *dwncrypto.RoleAudienceInfo) ([]byte, error) { + key := audienceKeyCacheKey{protocol: info.Protocol, rolePath: info.RolePath, keyID: info.KeyID} + return c.roleAudienceKeys.get(ctx, key, func(ctx context.Context) ([]byte, error) { + var errs []error + allRoutesStable := true + if c.audienceSource != nil { + privateKey, err := c.audienceSource.AudiencePrivateKeyByKeyID(ctx, info.Protocol, info.RolePath, info.KeyID) + if err == nil { + return privateKey, nil + } + errs = append(errs, fmt.Errorf("seal: %w", err)) + allRoutesStable = allRoutesStable && stableRoleAudienceRouteFailure(err) + } + privateKey, err := c.deliveryAudiencePrivateKey(ctx, info) + if err == nil { + return privateKey, nil + } + errs = append(errs, fmt.Errorf("delivery: %w", err)) + allRoutesStable = allRoutesStable && stableRoleAudienceRouteFailure(err) + joined := errors.Join(errs...) + if allRoutesStable { + return nil, fmt.Errorf("%w: role audience key unavailable: %w", errAudienceKeyDeliveryAbsent, joined) + } + return nil, fmt.Errorf("role audience key unavailable: %w", joined) + }) +} + // decryptViaDelivery recovers the audience key from a `$encryption/delivery` // record addressed to this node and unwraps the record's roleAudience entry. // The delivery record is encrypted to this node's OWN role-path key, derived // from its encryption root. func (c *DWNClient) decryptViaDelivery(ctx context.Context, ciphertext []byte, enc *dwncrypto.Encryption, info *dwncrypto.RoleAudienceInfo) ([]byte, error) { - privateKey, err := c.deliveredAudienceKeys.get(ctx, audienceKeyCacheKey{ - protocol: info.Protocol, - rolePath: info.RolePath, - keyID: info.KeyID, - }, func(ctx context.Context) ([]byte, error) { - return c.queryDeliveryAudiencePrivateKey(ctx, info) - }) + privateKey, err := c.deliveryAudiencePrivateKey(ctx, info) if err != nil { return nil, err } @@ -1737,9 +2118,19 @@ func (c *DWNClient) decryptViaDelivery(ctx context.Context, ciphertext []byte, e return dec.Decrypt(ciphertext, enc) } +func (c *DWNClient) deliveryAudiencePrivateKey(ctx context.Context, info *dwncrypto.RoleAudienceInfo) ([]byte, error) { + return c.deliveredAudienceKeys.get(ctx, audienceKeyCacheKey{ + protocol: info.Protocol, + rolePath: info.RolePath, + keyID: info.KeyID, + }, func(ctx context.Context) ([]byte, error) { + return c.queryDeliveryAudiencePrivateKey(ctx, info) + }) +} + func (c *DWNClient) queryDeliveryAudiencePrivateKey(ctx context.Context, info *dwncrypto.RoleAudienceInfo) ([]byte, error) { if c.encManager == nil { - return nil, fmt.Errorf("no encryption root available for delivery records") + return nil, fmt.Errorf("%w: no encryption root available for delivery records", errAudienceDeliveryUnavailable) } reply, err := c.queryDeliveryRecordsWithRetry(ctx, info) if err != nil { @@ -1747,6 +2138,9 @@ func (c *DWNClient) queryDeliveryAudiencePrivateKey(ctx context.Context, info *d } entries, err := dwn.QueryEntries(reply) if err != nil { + if !errors.Is(err, dwn.ErrRateLimited) { + err = errors.Join(dwn.ErrTransport, err) + } return nil, fmt.Errorf("parsing delivery query: %w", err) } for _, entry := range entries { @@ -1768,7 +2162,7 @@ func (c *DWNClient) queryDeliveryAudiencePrivateKey(ctx context.Context, info *d } return privateKey, nil } - return nil, fmt.Errorf("no delivery record for keyId %s at %s", info.KeyID, info.RolePath) + return nil, fmt.Errorf("%w: no delivery record for keyId %s at %s", errAudienceKeyDeliveryAbsent, info.KeyID, info.RolePath) } // queryDeliveryRecordsWithRetry makes at most one bounded retry of this diff --git a/internal/control/dwnclient_raw_capture_test.go b/internal/control/dwnclient_raw_capture_test.go new file mode 100644 index 0000000..978d92e --- /dev/null +++ b/internal/control/dwnclient_raw_capture_test.go @@ -0,0 +1,1087 @@ +package control + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/enboxorg/meshd/internal/dwn" + "github.com/enboxorg/meshd/protocols" +) + +type rawCaptureServerControl struct { + failNetwork atomic.Bool + networkStarted chan struct{} + releaseNetwork chan struct{} + startOnce sync.Once + requests atomic.Int64 + targetedReads atomic.Int64 + targetedRateLimit atomic.Bool + queryRateLimitPath string + queryRateLimitDetail string + queryDelay time.Duration + activeQueries atomic.Int32 + maxConcurrentQueries atomic.Int32 + targetedReadEntry map[string]json.RawMessage + targetedReadBodies map[string][]byte +} + +func newRawCaptureLoadClient( + t *testing.T, + network json.RawMessage, + entriesByPath map[string][]json.RawMessage, + control *rawCaptureServerControl, +) *DWNClient { + t.Helper() + if control == nil { + control = &rawCaptureServerControl{} + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + control.requests.Add(1) + var request dwn.JsonRpcRequest + if err := json.Unmarshal([]byte(r.Header.Get("dwn-request")), &request); err != nil || request.Params == nil || request.Params.Message == nil { + http.Error(w, "invalid DWN request", http.StatusBadRequest) + return + } + message := request.Params.Message + method, _ := message.Descriptor["method"].(string) + reply := &dwn.DwnReply{Status: dwn.Status{Code: http.StatusOK, Detail: "OK"}} + if method == "Read" { + filter, _ := message.Descriptor["filter"].(map[string]any) + recordID, _ := filter["recordId"].(string) + if recordID != materializerNetworkID { + control.targetedReads.Add(1) + if control.targetedRateLimit.Load() { + reply.Status = dwn.Status{Code: http.StatusTooManyRequests, Detail: "RateLimitExceeded: retry after 2s"} + } else if entry, ok := control.targetedReadEntry[recordID]; ok { + reply.Entry = entry + if body, ok := control.targetedReadBodies[recordID]; ok { + response, err := json.Marshal(dwn.JsonRpcResponse{ + JSONRPC: "2.0", ID: request.ID, + Result: &dwn.JsonRpcResult{Reply: reply}, + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("dwn-response", string(response)) + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(body) + return + } + } else { + reply.Status = dwn.Status{Code: http.StatusNotFound, Detail: "targeted fixture missing"} + } + } else { + if control.networkStarted != nil { + control.startOnce.Do(func() { close(control.networkStarted) }) + } + if control.releaseNetwork != nil { + <-control.releaseNetwork + } + if control.failNetwork.Load() { + reply.Status = dwn.Status{Code: http.StatusInternalServerError, Detail: "forced failure"} + } else { + reply.Entry = network + } + } + } else { + filter, _ := message.Descriptor["filter"].(map[string]any) + path, _ := filter["protocolPath"].(string) + contextID, _ := filter["contextId"].(string) + active := control.activeQueries.Add(1) + for { + maximum := control.maxConcurrentQueries.Load() + if active <= maximum || control.maxConcurrentQueries.CompareAndSwap(maximum, active) { + break + } + } + defer control.activeQueries.Add(-1) + if control.queryDelay > 0 { + select { + case <-time.After(control.queryDelay): + case <-r.Context().Done(): + return + } + } + if path == control.queryRateLimitPath { + detail := control.queryRateLimitDetail + if detail == "" { + detail = "RateLimitExceeded: retry after 1s" + } + reply.Status = dwn.Status{Code: http.StatusTooManyRequests, Detail: detail} + } else { + var selected []json.RawMessage + for _, entry := range entriesByPath[path] { + identity, identityErr := rawCaptureIdentity(entry) + if identityErr != nil || identity.parentContextID == contextID { + selected = append(selected, entry) + } + } + reply.Entries, _ = json.Marshal(selected) + } + } + _ = json.NewEncoder(w).Encode(dwn.JsonRpcResponse{ + JSONRPC: "2.0", + ID: request.ID, + Result: &dwn.JsonRpcResult{Reply: reply}, + }) + })) + t.Cleanup(server.Close) + _, signer, _, _ := sealedTestOwner(t) + return NewDWNClient(server.URL, materializerSelfDID, materializerNetworkID, materializerSelfDID, signer) +} + +func rawCaptureLoadFixture(t *testing.T) (json.RawMessage, map[string][]json.RawMessage) { + t.Helper() + record := func(id, path, parent, recipient, timestamp string, data any) json.RawMessage { + raw := materializerRecord(t, materializerRecordSpec{id: id, path: path, parentContext: parent, recipient: recipient, data: data, timestamp: timestamp}) + var message map[string]any + if err := json.Unmarshal(raw, &message); err != nil { + t.Fatal(err) + } + message["descriptor"].(map[string]any)["dateCreated"] = timestamp + raw, err := json.Marshal(message) + if err != nil { + t.Fatal(err) + } + return raw + } + network := record(materializerNetworkID, "network", "", "", "2026-07-11T12:00:00Z", NetworkConfig{Name: "capture", MeshCIDR: "10.200.0.0/16"}) + entries := map[string][]json.RawMessage{ + "network/node": {record("self-node", "network/node", materializerNetworkID, materializerSelfDID, "2026-07-11T12:00:01Z", NodeRecord{MeshIP: "10.200.1.1", Label: "self"})}, + "network/member": {record("member-record", "network/member", materializerNetworkID, "did:jwk:member", "2026-07-11T12:00:02Z", MemberRecord{Label: "member", AddedAt: "2026-07-11T12:00:02Z"})}, + "network/member/node": {record("peer-node", "network/member/node", materializerNetworkID+"/member-record", materializerPeerDID, "2026-07-11T12:00:03Z", NodeRecord{MeshIP: "10.200.1.2", Label: "peer"})}, + "network/relay": {record("relay-record", "network/relay", materializerNetworkID, "", "2026-07-11T12:00:04Z", RelayData{URL: "relay.example.com", Region: "test", STUNPort: 3478})}, + "network/aclPolicy": {record("acl-record", "network/aclPolicy", materializerNetworkID, "", "2026-07-11T12:00:05Z", ACLPolicyData{Version: 1, DefaultAction: "accept"})}, + "network/node/nodeInfo": {record("self-info", "network/node/nodeInfo", materializerNetworkID+"/self-node", "", "2026-07-11T12:00:06Z", NodeInfoData{Hostname: "self-host"})}, + "network/member/node/nodeInfo": {record("peer-info", "network/member/node/nodeInfo", materializerNetworkID+"/member-record/peer-node", "", "2026-07-11T12:00:07Z", NodeInfoData{Hostname: "peer-host"})}, + "network/node/endpoint": {record("self-endpoint", "network/node/endpoint", materializerNetworkID+"/self-node", "", "2026-07-11T12:00:08Z", EndpointData{LocalEndpoints: []string{"192.0.2.1:1111"}})}, + "network/member/node/endpoint": {record("peer-endpoint", "network/member/node/endpoint", materializerNetworkID+"/member-record/peer-node", "", "2026-07-11T12:00:09Z", EndpointData{LocalEndpoints: []string{"192.0.2.2:2222"}})}, + } + return network, entries +} + +func TestRawCaptureReadStatusAndEndpointFailureClassification(t *testing.T) { + if err := rawCaptureReadStatusError(nil); !errors.Is(err, dwn.ErrTransport) { + t.Fatalf("nil read error = %v, want ErrTransport", err) + } + if err := rawCaptureReadStatusError(&dwn.RecordsReadResult{Reply: &dwn.DwnReply{ + Status: dwn.Status{Code: http.StatusInternalServerError, Detail: "failed"}, + }}); !errors.Is(err, dwn.ErrTransport) { + t.Fatalf("500 read error = %v, want ErrTransport", err) + } + if err := rawCaptureReadStatusError(&dwn.RecordsReadResult{Reply: &dwn.DwnReply{ + Status: dwn.Status{Code: http.StatusNotFound, Detail: "missing"}, + }}); !errors.Is(err, errRawCaptureReadNotFound) || errors.Is(err, dwn.ErrTransport) { + t.Fatalf("404 read error = %v, want distinct not-found", err) + } + if got := endpointFailureClass(fmt.Errorf("wrapped: %w", errAudienceKeyDeliveryAbsent)); got != "key-unavailable" { + t.Fatalf("audience absence failure class = %q", got) + } +} + +func TestQueryAllRawMeshRecordsPaginationBoundsAndOpaqueCursor(t *testing.T) { + client := newMaterializerTestClient() + entry := func(id string) json.RawMessage { + return materializerRecord(t, materializerRecordSpec{ + id: id, path: "network/node", parentContext: materializerNetworkID, recipient: materializerSelfDID, + data: NodeRecord{MeshIP: "10.200.1.1"}, timestamp: "2026-07-11T12:00:00Z", + }) + } + reply := func(entries []json.RawMessage, cursor string) *dwn.DwnReply { + rawEntries, err := json.Marshal(entries) + if err != nil { + t.Fatal(err) + } + return &dwn.DwnReply{Status: dwn.Status{Code: http.StatusOK}, Entries: rawEntries, Cursor: json.RawMessage(cursor)} + } + + t.Run("multi-page cursor preserves large numeric lexeme", func(t *testing.T) { + const cursor = ` { "position" : 9007199254740993123456789 } ` + const compact = `{"position":9007199254740993123456789}` + pages := []*dwn.DwnReply{reply([]json.RawMessage{entry("node-a")}, cursor), reply([]json.RawMessage{entry("node-b")}, "")} + calls := 0 + got, err := client.queryAllRawMeshRecordsWith(context.Background(), dwn.RecordsFilter{ + Protocol: protocols.MeshProtocolURI, ProtocolPath: "network/node", ContextID: materializerNetworkID, + }, "", &fullStateFetchBudget{}, func(_ context.Context, pagination *dwn.Pagination) (*dwn.DwnReply, error) { + if calls == 0 && len(pagination.Cursor) != 0 { + t.Fatalf("first cursor = %s", pagination.Cursor) + } + if calls == 1 && string(pagination.Cursor) != compact { + t.Fatalf("second cursor = %s, want %s", pagination.Cursor, compact) + } + page := pages[calls] + calls++ + return page, nil + }) + if err != nil || len(got) != 2 || calls != 2 { + t.Fatalf("queryAll = (%d entries, %v), calls=%d", len(got), err, calls) + } + }) + + t.Run("duplicate compact cursor fails", func(t *testing.T) { + calls := 0 + _, err := client.queryAllRawMeshRecordsWith(context.Background(), dwn.RecordsFilter{ + Protocol: protocols.MeshProtocolURI, ProtocolPath: "network/node", ContextID: materializerNetworkID, + }, "", &fullStateFetchBudget{}, func(context.Context, *dwn.Pagination) (*dwn.DwnReply, error) { + calls++ + return reply(nil, ` { "messageCid" : "same" } `), nil + }) + if err == nil || !strings.Contains(err.Error(), "repeated pagination cursor") || calls != 2 { + t.Fatalf("duplicate cursor error=%v calls=%d", err, calls) + } + }) + + t.Run("page limit fails", func(t *testing.T) { + calls := 0 + _, err := client.queryAllRawMeshRecordsWith(context.Background(), dwn.RecordsFilter{ + Protocol: protocols.MeshProtocolURI, ProtocolPath: "network/node", ContextID: materializerNetworkID, + }, "", &fullStateFetchBudget{}, func(context.Context, *dwn.Pagination) (*dwn.DwnReply, error) { + calls++ + return reply(nil, fmt.Sprintf(`{"page":%d}`, calls)), nil + }) + if err == nil || !strings.Contains(err.Error(), "page limit") || calls != fullStateMaxQueryPages { + t.Fatalf("page limit error=%v calls=%d", err, calls) + } + }) + + t.Run("global limits fail closed", func(t *testing.T) { + budget := &fullStateFetchBudget{requests: fullStateMaxRequests} + if err := budget.takeRequest(); err == nil { + t.Fatal("request limit accepted another request") + } + budget = &fullStateFetchBudget{records: fullStateMaxRecords} + if err := budget.retain([]json.RawMessage{json.RawMessage(`{}`)}, nil); err == nil { + t.Fatal("record limit accepted another record") + } + budget = &fullStateFetchBudget{bytes: fullStateMaxBytes} + if err := budget.retain(nil, json.RawMessage(`{}`)); err == nil { + t.Fatal("byte limit accepted more data") + } + }) +} + +func TestLoadStateUsesBoundedConcurrentParentQueries(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + const peers = 12 + entries["network/member"] = nil + entries["network/member/node"] = nil + entries["network/member/node/nodeInfo"] = nil + entries["network/member/node/endpoint"] = nil + for i := 0; i < peers; i++ { + memberID := fmt.Sprintf("member-%02d", i) + nodeID := fmt.Sprintf("peer-node-%02d", i) + memberDID := fmt.Sprintf("did:jwk:member-%02d", i) + nodeDID := fmt.Sprintf("did:jwk:peer-%02d", i) + timestamp := fmt.Sprintf("2026-07-11T12:00:%02dZ", 10+i) + entries["network/member"] = append(entries["network/member"], materializerRecord(t, materializerRecordSpec{ + id: memberID, path: "network/member", parentContext: materializerNetworkID, recipient: memberDID, + data: MemberRecord{Label: memberID}, timestamp: timestamp, + })) + nodeContext := materializerNetworkID + "/" + memberID + entries["network/member/node"] = append(entries["network/member/node"], materializerRecord(t, materializerRecordSpec{ + id: nodeID, path: "network/member/node", parentContext: nodeContext, recipient: nodeDID, + data: NodeRecord{MeshIP: fmt.Sprintf("10.200.2.%d", i+1), Label: nodeID}, timestamp: timestamp, + })) + nodeContext += "/" + nodeID + entries["network/member/node/nodeInfo"] = append(entries["network/member/node/nodeInfo"], materializerRecord(t, materializerRecordSpec{ + id: "info-" + nodeID, path: "network/member/node/nodeInfo", parentContext: nodeContext, + data: NodeInfoData{Hostname: nodeID}, timestamp: timestamp, + })) + entries["network/member/node/endpoint"] = append(entries["network/member/node/endpoint"], materializerRecord(t, materializerRecordSpec{ + id: "endpoint-" + nodeID, path: "network/member/node/endpoint", parentContext: nodeContext, + data: EndpointData{LocalEndpoints: []string{fmt.Sprintf("192.0.2.%d:4242", i+1)}}, timestamp: timestamp, + })) + } + control := &rawCaptureServerControl{queryDelay: 10 * time.Millisecond} + client := newRawCaptureLoadClient(t, network, entries, control) + response, err := client.LoadState(context.Background()) + if err != nil { + t.Fatalf("LoadState: %v", err) + } + if len(response.Peers) != peers { + t.Fatalf("peers = %d, want %d", len(response.Peers), peers) + } + if maximum := control.maxConcurrentQueries.Load(); maximum <= 1 || maximum > fullStateQueryWorkers { + t.Fatalf("maximum concurrent queries = %d, want 2..%d", maximum, fullStateQueryWorkers) + } + wantRequests := int64(7 + 3*peers) + if got := control.requests.Load(); got != wantRequests { + t.Fatalf("requests = %d, want %d", got, wantRequests) + } +} + +func TestLoadStateInstallsCompleteRawBaseline(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + client := newRawCaptureLoadClient(t, network, entries, nil) + response, err := client.LoadState(context.Background()) + if err != nil { + t.Fatalf("LoadState: %v", err) + } + if response.Node == nil || len(response.Peers) != 1 { + t.Fatalf("map response = %#v", response) + } + client.deltaMu.Lock() + baseline := client.rawBaseline.clone() + repair := client.fullReconciliation + client.deltaMu.Unlock() + if baseline == nil || repair { + t.Fatalf("raw baseline = %v, repair = %v", baseline, repair) + } + wantPaths := []string{"network", "network/node", "network/member", "network/member/node", "network/relay", "network/aclPolicy", "network/node/nodeInfo", "network/member/node/nodeInfo", "network/node/endpoint", "network/member/node/endpoint"} + gotPaths := make(map[string]int) + for _, record := range baseline.all() { + gotPaths[record.protocolPath]++ + } + for _, path := range wantPaths { + if got := gotPaths[path]; got != 1 { + t.Fatalf("captured %s records = %d, want 1; all paths %#v", path, got, gotPaths) + } + } + if len(gotPaths) != len(wantPaths) { + t.Fatalf("captured paths = %#v", gotPaths) + } +} + +func TestLoadStateThenInlineSubscriptionDeltaUsesNoRemoteRequest(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + control := &rawCaptureServerControl{} + client := newRawCaptureLoadClient(t, network, entries, control) + initial, err := client.LoadState(context.Background()) + if err != nil { + t.Fatalf("LoadState: %v", err) + } + if initial.Node == nil || len(initial.Peers) != 1 { + t.Fatalf("initial map = %#v, want self and one peer", initial) + } + + var update map[string]any + if err := json.Unmarshal(entries["network/member/node"][0], &update); err != nil { + t.Fatal(err) + } + update["descriptor"].(map[string]any)["messageTimestamp"] = "2026-07-11T12:02:00Z" + data, err := json.Marshal(NodeRecord{MeshIP: "10.200.1.99", Label: "peer-updated"}) + if err != nil { + t.Fatal(err) + } + update["encodedData"] = base64.RawURLEncoding.EncodeToString(data) + updateRaw, err := json.Marshal(update) + if err != nil { + t.Fatal(err) + } + messageCID, err := computeRawRecordMessageCID(updateRaw) + if err != nil { + t.Fatal(err) + } + event := rawRecordTestSubscription(updateRaw, "") + event.MessageCID = messageCID + event.Cursor = &dwn.ProgressToken{ + StreamID: "topology", Epoch: "epoch", Position: "1", MessageCID: messageCID, + } + beforeApply := control.requests.Load() + if err := client.StageTopologyEvent(event); err != nil { + t.Fatalf("StageTopologyEvent: %v", err) + } + response, err := client.ApplyPendingState(context.Background()) + if err != nil { + t.Fatalf("ApplyPendingState: %v", err) + } + if got := control.requests.Load(); got != beforeApply { + t.Fatalf("delta apply made %d remote requests, want 0", got-beforeApply) + } + var updated *Node + for _, peer := range response.Peers { + if peer.DID == materializerPeerDID { + updated = peer + break + } + } + if updated == nil || updated.MeshIP.String() != "10.200.1.99" || updated.Label != "peer-updated" { + t.Fatalf("updated peer = %#v", updated) + } +} + +func TestLoadStateFailurePreservesRawBaseline(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + control := &rawCaptureServerControl{} + client := newRawCaptureLoadClient(t, network, entries, control) + if _, err := client.LoadState(context.Background()); err != nil { + t.Fatalf("initial LoadState: %v", err) + } + client.deltaMu.Lock() + before := client.rawBaseline + beforeCount := before.len() + client.deltaMu.Unlock() + + control.failNetwork.Store(true) + if _, err := client.LoadState(context.Background()); err == nil { + t.Fatal("failed LoadState unexpectedly succeeded") + } + client.deltaMu.Lock() + after := client.rawBaseline + repair := client.fullReconciliation + client.deltaMu.Unlock() + if after != before { + t.Fatal("failed LoadState replaced the last-good raw baseline") + } + if repair || after.len() != beforeCount { + t.Fatalf("failed LoadState changed raw readiness: repair=%v records=%d want=%d", repair, after.len(), beforeCount) + } +} + +func TestLoadStateValidatedFailurePreservesParsedRawAndPendingState(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + client := newRawCaptureLoadClient(t, network, entries, nil) + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + + client.mu.RLock() + oldNetwork := client.network + oldNode := client.nodes[materializerSelfDID] + client.mu.RUnlock() + client.deltaMu.Lock() + oldRaw := client.rawBaseline + client.deltaMu.Unlock() + if err := client.StageTopologyEvent(rawRecordTestSubscription( + rawRecordTestDelete(t, "covered-by-full", "2026-07-11T12:10:00Z"), "", + )); err != nil { + t.Fatal(err) + } + client.deltaMu.Lock() + oldSequence := client.pendingTopology[0].sequence + oldMessage := client.pendingTopology[0].message + client.deltaMu.Unlock() + + validationErr := errors.New("converter rejected full candidate") + var validated *MapResponse + response, err := client.LoadStateValidated(context.Background(), func(candidate *MapResponse) error { + validated = candidate + if client.loadMu.TryLock() { + client.loadMu.Unlock() + t.Error("validator ran without loadMu held") + } + return validationErr + }) + if !errors.Is(err, validationErr) || response == nil || response != validated { + t.Fatalf("LoadStateValidated = (%p, %v), validated %p", response, err, validated) + } + if response.Node == nil || len(response.Peers) != 1 { + t.Fatalf("validated full candidate = %#v", response) + } + + client.mu.RLock() + afterNetwork := client.network + afterNode := client.nodes[materializerSelfDID] + client.mu.RUnlock() + if afterNetwork != oldNetwork || afterNode != oldNode { + t.Fatalf("validation failure advanced parsed state: network %p/%p node %p/%p", oldNetwork, afterNetwork, oldNode, afterNode) + } + client.deltaMu.Lock() + defer client.deltaMu.Unlock() + if client.rawBaseline != oldRaw || client.fullReconciliation { + t.Fatalf("validation failure advanced raw state: baseline %p/%p repair=%v", oldRaw, client.rawBaseline, client.fullReconciliation) + } + if len(client.pendingTopology) != 1 || client.pendingTopology[0].sequence != oldSequence || + client.pendingTopology[0].message != oldMessage { + t.Fatalf("validation failure trimmed pending prefix: %#v", client.pendingTopology) + } +} + +func TestLoadStateValidatedSuccessCommitsParsedRawAndPrefixAtomically(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + client := newRawCaptureLoadClient(t, network, entries, nil) + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + client.deltaMu.Lock() + oldRaw := client.rawBaseline + client.deltaMu.Unlock() + if err := client.StageTopologyEvent(rawRecordTestSubscription( + rawRecordTestDelete(t, "covered-by-full", "2026-07-11T12:10:00Z"), "", + )); err != nil { + t.Fatal(err) + } + + entered := make(chan struct{}) + release := make(chan struct{}) + type loadResult struct { + response *MapResponse + err error + } + done := make(chan loadResult, 1) + var validated *MapResponse + go func() { + response, err := client.LoadStateValidated(context.Background(), func(candidate *MapResponse) error { + validated = candidate + if client.loadMu.TryLock() { + client.loadMu.Unlock() + return errors.New("validator ran without loadMu held") + } + close(entered) + <-release + return nil + }) + done <- loadResult{response: response, err: err} + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("full load did not reach validator") + } + + client.deltaMu.Lock() + if client.rawBaseline != oldRaw || len(client.pendingTopology) != 1 { + client.deltaMu.Unlock() + t.Fatal("raw baseline or pending prefix committed before validation") + } + client.deltaMu.Unlock() + readerStarted := make(chan struct{}) + readNodes := make(chan map[string]*NodeRecord, 1) + go func() { + close(readerStarted) + readNodes <- client.Nodes() + }() + <-readerStarted + select { + case <-readNodes: + t.Fatal("snapshot reader escaped while full-load validation was pending") + case <-time.After(25 * time.Millisecond): + } + + close(release) + var result loadResult + select { + case result = <-done: + case <-time.After(5 * time.Second): + t.Fatal("validated full load did not finish") + } + if result.err != nil || result.response == nil || result.response != validated { + t.Fatalf("LoadStateValidated = (%p, %v), validated %p", result.response, result.err, validated) + } + var nodes map[string]*NodeRecord + select { + case nodes = <-readNodes: + case <-time.After(5 * time.Second): + t.Fatal("snapshot reader did not resume after commit") + } + if nodes[materializerSelfDID] == nil || nodes[materializerPeerDID] == nil { + t.Fatalf("snapshot reader saw non-candidate nodes: %#v", nodes) + } + + client.deltaMu.Lock() + defer client.deltaMu.Unlock() + if client.rawBaseline == nil || client.rawBaseline == oldRaw || client.fullReconciliation { + t.Fatalf("successful validation did not commit raw state: old=%p current=%p repair=%v", oldRaw, client.rawBaseline, client.fullReconciliation) + } + if len(client.pendingTopology) != 0 { + t.Fatalf("successful validation retained covered prefix: %#v", client.pendingTopology) + } +} + +func TestLoadStateValidatedRejectsNewerRepairMarkers(t *testing.T) { + repairs := []struct { + name string + run func(*testing.T, *DWNClient) + }{ + {name: "lifecycle", run: func(_ *testing.T, client *DWNClient) { client.RequireFullReconciliation() }}, + {name: "poison", run: func(t *testing.T, client *DWNClient) { + if err := client.StageTopologyEvent(&dwn.SubscriptionMessage{Type: dwn.SubscriptionEventType}); err != nil { + t.Errorf("stage poison repair: %v", err) + } + }}, + {name: "overflow", run: func(t *testing.T, client *DWNClient) { + event := rawRecordTestSubscription(rawRecordTestDelete(t, "overflow-repair", "2026-07-11T12:20:00Z"), "") + for i := 0; i <= maxPendingTopologyEvents; i++ { + if err := client.StageTopologyEvent(event); err != nil { + t.Errorf("stage overflow repair: %v", err) + } + } + }}, + } + for _, phase := range []string{"remote-fetch", "validator"} { + for _, repair := range repairs { + t.Run(phase+"/"+repair.name, func(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + control := &rawCaptureServerControl{} + if phase == "remote-fetch" { + control.networkStarted = make(chan struct{}) + control.releaseNetwork = make(chan struct{}) + } + client := newRawCaptureLoadClient(t, network, entries, control) + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + client.mu.RLock() + oldNetwork := client.network + oldNode := client.nodes[materializerSelfDID] + client.mu.RUnlock() + client.deltaMu.Lock() + oldRaw := client.rawBaseline + client.deltaMu.Unlock() + + type result struct { + response *MapResponse + err error + } + done := make(chan result, 1) + go func() { + response, err := client.LoadStateValidated(context.Background(), func(*MapResponse) error { + if phase == "validator" { + repair.run(t, client) + } + return nil + }) + done <- result{response: response, err: err} + }() + if phase == "remote-fetch" { + select { + case <-control.networkStarted: + case <-time.After(5 * time.Second): + t.Fatal("load did not reach remote fetch") + } + repair.run(t, client) + close(control.releaseNetwork) + } + var got result + select { + case got = <-done: + case <-time.After(5 * time.Second): + t.Fatal("load did not finish") + } + if got.response == nil || !errors.Is(got.err, ErrFullReconciliationRequired) { + t.Fatalf("LoadStateValidated = (%#v, %v), want fenced candidate", got.response, got.err) + } + client.mu.RLock() + parsedUnchanged := client.network == oldNetwork && client.nodes[materializerSelfDID] == oldNode + client.mu.RUnlock() + client.deltaMu.Lock() + rawUnchanged := client.rawBaseline == oldRaw + repairPending := client.fullReconciliation && client.repairSequence > 0 + client.deltaMu.Unlock() + if !parsedUnchanged || !rawUnchanged || !repairPending { + t.Fatalf("newer repair published state: parsed=%v raw=%v repair=%v", parsedUnchanged, rawUnchanged, repairPending) + } + }) + } + } +} + +func TestLoadStateMalformedRawRollsBackAuthoritativeLoad(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + var malformed map[string]any + if err := json.Unmarshal(entries["network/node"][0], &malformed); err != nil { + t.Fatal(err) + } + descriptor := malformed["descriptor"].(map[string]any) + delete(descriptor, "protocol") + delete(descriptor, "messageTimestamp") + entries["network/node"][0], _ = json.Marshal(malformed) + client := newRawCaptureLoadClient(t, network, entries, nil) + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + client.mu.RLock() + oldNetwork := client.network + oldNode := client.nodes[materializerSelfDID] + client.mu.RUnlock() + client.deltaMu.Lock() + oldRaw := client.rawBaseline + client.deltaMu.Unlock() + + if response, err := client.LoadState(context.Background()); err == nil || response != nil { + t.Fatalf("malformed authoritative load = (%#v, %v), want failure", response, err) + } + client.mu.RLock() + parsedUnchanged := client.network == oldNetwork && client.nodes[materializerSelfDID] == oldNode + client.mu.RUnlock() + if !parsedUnchanged { + t.Fatal("malformed authoritative load advanced parsed state") + } + client.deltaMu.Lock() + rawUnchanged := client.rawBaseline == oldRaw + client.deltaMu.Unlock() + if !rawUnchanged { + t.Fatal("malformed authoritative load replaced raw baseline") + } +} + +func TestLoadStatePreservesEventStagedAfterReconciliationCut(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + control := &rawCaptureServerControl{networkStarted: make(chan struct{}), releaseNetwork: make(chan struct{})} + client := newRawCaptureLoadClient(t, network, entries, control) + loadDone := make(chan error, 1) + go func() { + _, err := client.LoadState(context.Background()) + loadDone <- err + }() + select { + case <-control.networkStarted: + case <-time.After(2 * time.Second): + t.Fatal("full load did not reach blocked network read") + } + + latest := true + eventRaw := materializerRecord(t, materializerRecordSpec{ + id: "queued-endpoint", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", + data: EndpointData{LocalEndpoints: []string{"192.0.2.1:3333"}}, timestamp: "2026-07-11T12:01:00Z", + }) + if err := client.StageTopologyEvent(&dwn.SubscriptionMessage{ + Type: dwn.SubscriptionEventType, IsLatestBaseState: &latest, Event: &dwn.RecordEvent{Message: eventRaw}, + }); err != nil { + t.Fatalf("StageTopologyEvent: %v", err) + } + close(control.releaseNetwork) + select { + case err := <-loadDone: + if err != nil { + t.Fatalf("LoadState: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("full load did not finish") + } + + client.deltaMu.Lock() + defer client.deltaMu.Unlock() + if client.rawBaseline == nil || client.fullReconciliation { + t.Fatalf("baseline = %v, repair = %v", client.rawBaseline, client.fullReconciliation) + } + if len(client.pendingTopology) != 1 { + t.Fatalf("pending events = %d, want event staged after full-load cut", len(client.pendingTopology)) + } + recordID, err := topologyWriteRecordID(client.pendingTopology[0].message) + if err != nil || recordID != "queued-endpoint" { + t.Fatalf("pending event record = %q, err = %v", recordID, err) + } +} + +func TestLoadStateHydratesMissingQueryDataIntoBaselineAndLegacyState(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + fullEntry := entries["network/aclPolicy"][0] + withoutData, body := rawCaptureEntryWithoutInlineData(t, fullEntry) + entries["network/aclPolicy"] = []json.RawMessage{withoutData} + control := &rawCaptureServerControl{ + targetedReadEntry: map[string]json.RawMessage{"acl-record": withoutData}, + targetedReadBodies: map[string][]byte{"acl-record": body}, + } + client := newRawCaptureLoadClient(t, network, entries, control) + + if _, err := client.LoadState(context.Background()); err != nil { + t.Fatalf("LoadState: %v", err) + } + if got := control.targetedReads.Load(); got != 1 { + t.Fatalf("targeted RecordsRead calls = %d, want 1", got) + } + client.mu.RLock() + acl := client.acl + client.mu.RUnlock() + if acl == nil || acl.Version != 1 { + t.Fatalf("legacy ACL state = %#v, want hydrated version 1", acl) + } + client.deltaMu.Lock() + baseline := client.rawBaseline + repair := client.fullReconciliation + client.deltaMu.Unlock() + if baseline == nil || repair { + t.Fatalf("raw baseline = %v, repair = %v", baseline, repair) + } + record, ok := baseline.get("acl-record") + if !ok || !rawRecordHasEncodedData(record.raw) { + t.Fatalf("hydrated ACL record missing from raw baseline: ok=%v record=%#v", ok, record) + } +} + +func TestLoadStateInlineQueryDataDoesNotReadRecords(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + control := &rawCaptureServerControl{} + client := newRawCaptureLoadClient(t, network, entries, control) + if _, err := client.LoadState(context.Background()); err != nil { + t.Fatalf("LoadState: %v", err) + } + if got := control.targetedReads.Load(); got != 0 { + t.Fatalf("targeted RecordsRead calls = %d, want 0", got) + } +} + +func TestLoadStateStaleHydrationRollsBackAuthoritativeLoad(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + fullEntry := entries["network/aclPolicy"][0] + withoutData, _ := rawCaptureEntryWithoutInlineData(t, fullEntry) + var stale map[string]any + if err := json.Unmarshal(fullEntry, &stale); err != nil { + t.Fatal(err) + } + stale["descriptor"].(map[string]any)["messageTimestamp"] = "2026-07-11T11:59:00Z" + staleRaw, err := json.Marshal(stale) + if err != nil { + t.Fatal(err) + } + staleEntry, body := rawCaptureEntryWithoutInlineData(t, staleRaw) + entries["network/aclPolicy"] = []json.RawMessage{withoutData} + control := &rawCaptureServerControl{ + targetedReadEntry: map[string]json.RawMessage{"acl-record": staleEntry}, targetedReadBodies: map[string][]byte{"acl-record": body}, + } + client := newRawCaptureLoadClient(t, network, entries, control) + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + client.mu.RLock() + oldNetwork := client.network + client.mu.RUnlock() + client.deltaMu.Lock() + oldRaw := client.rawBaseline + client.deltaMu.Unlock() + + if response, err := client.LoadState(context.Background()); err == nil || response != nil { + t.Fatalf("stale hydration load = (%#v, %v), want failure", response, err) + } + if got := control.targetedReads.Load(); got != 1 { + t.Fatalf("targeted RecordsRead calls = %d, want 1", got) + } + client.mu.RLock() + afterNetwork := client.network + client.mu.RUnlock() + client.deltaMu.Lock() + afterRaw := client.rawBaseline + client.deltaMu.Unlock() + if afterNetwork != oldNetwork || afterRaw != oldRaw { + t.Fatal("stale hydration advanced parsed or raw state") + } +} + +func TestLoadStateNewerHydrationHeadWins(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + queryEntry, _ := rawCaptureEntryWithoutInlineData(t, entries["network/aclPolicy"][0]) + var newerMessage map[string]any + if err := json.Unmarshal(entries["network/aclPolicy"][0], &newerMessage); err != nil { + t.Fatal(err) + } + newerMessage["descriptor"].(map[string]any)["messageTimestamp"] = "2026-07-11T12:01:00Z" + newerData, err := json.Marshal(ACLPolicyData{Version: 2, DefaultAction: "accept"}) + if err != nil { + t.Fatal(err) + } + newerMessage["encodedData"] = base64.RawURLEncoding.EncodeToString(newerData) + newerFull, err := json.Marshal(newerMessage) + if err != nil { + t.Fatal(err) + } + newerEntry, newerBody := rawCaptureEntryWithoutInlineData(t, newerFull) + entries["network/aclPolicy"] = []json.RawMessage{queryEntry} + control := &rawCaptureServerControl{ + targetedReadEntry: map[string]json.RawMessage{"acl-record": newerEntry}, + targetedReadBodies: map[string][]byte{"acl-record": newerBody}, + } + client := newRawCaptureLoadClient(t, network, entries, control) + + if _, err := client.LoadState(context.Background()); err != nil { + t.Fatalf("LoadState: %v", err) + } + client.mu.RLock() + acl := client.acl + client.mu.RUnlock() + if acl == nil || acl.Version != 2 { + t.Fatalf("legacy ACL state = %#v, want newer hydrated version 2", acl) + } + client.deltaMu.Lock() + baseline := client.rawBaseline + repair := client.fullReconciliation + client.deltaMu.Unlock() + if baseline == nil || repair { + t.Fatalf("raw baseline = %v, repair = %v", baseline, repair) + } + record, ok := baseline.get("acl-record") + if !ok { + t.Fatal("newer ACL head missing from raw baseline") + } + identity, err := rawCaptureIdentity(record.raw) + if err != nil { + t.Fatal(err) + } + if identity.messageTimestamp != "2026-07-11T12:01:00Z" { + t.Fatalf("baseline ACL revision = %q, want newer read head", identity.messageTimestamp) + } +} + +func TestLoadStateStopsBeforeHydrationAfterMalformedEarlierPath(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + var malformedNode map[string]any + if err := json.Unmarshal(entries["network/node"][0], &malformedNode); err != nil { + t.Fatal(err) + } + delete(malformedNode["descriptor"].(map[string]any), "protocol") + entries["network/node"][0], _ = json.Marshal(malformedNode) + + fullACL := entries["network/aclPolicy"][0] + withoutACL, body := rawCaptureEntryWithoutInlineData(t, fullACL) + entries["network/aclPolicy"] = []json.RawMessage{withoutACL} + control := &rawCaptureServerControl{ + targetedReadEntry: map[string]json.RawMessage{"acl-record": withoutACL}, targetedReadBodies: map[string][]byte{"acl-record": body}, + } + client := newRawCaptureLoadClient(t, network, entries, control) + installDeltaTestBaseline(t, client, materializerBaseRecords(t)) + client.deltaMu.Lock() + oldRaw := client.rawBaseline + client.deltaMu.Unlock() + + if response, err := client.LoadState(context.Background()); err == nil || response != nil { + t.Fatalf("malformed authoritative load = (%#v, %v), want failure", response, err) + } + if got := control.targetedReads.Load(); got != 0 { + t.Fatalf("targeted RecordsRead calls after fatal earlier path = %d, want 0", got) + } + client.deltaMu.Lock() + defer client.deltaMu.Unlock() + if client.rawBaseline != oldRaw { + t.Fatal("fatal earlier path replaced raw baseline") + } +} + +func TestLoadStateRateLimitedHydrationPreservesLastGoodState(t *testing.T) { + network, entries := rawCaptureLoadFixture(t) + fullEntry := entries["network/aclPolicy"][0] + withoutData, body := rawCaptureEntryWithoutInlineData(t, fullEntry) + entries["network/aclPolicy"] = []json.RawMessage{withoutData} + control := &rawCaptureServerControl{ + targetedReadEntry: map[string]json.RawMessage{"acl-record": withoutData}, + targetedReadBodies: map[string][]byte{"acl-record": body}, + } + client := newRawCaptureLoadClient(t, network, entries, control) + if _, err := client.LoadState(context.Background()); err != nil { + t.Fatalf("initial LoadState: %v", err) + } + client.deltaMu.Lock() + beforeRaw := client.rawBaseline + client.deltaMu.Unlock() + client.mu.RLock() + beforeNetwork := client.network + beforeACL := client.acl + client.mu.RUnlock() + + control.targetedRateLimit.Store(true) + _, loadErr := client.LoadState(context.Background()) + if !errors.Is(loadErr, dwn.ErrRateLimited) { + t.Fatalf("rate-limited LoadState error = %v, want ErrRateLimited", loadErr) + } + var rateErr *dwn.RateLimitError + if !errors.As(loadErr, &rateErr) || rateErr.RetryAfter != 2*time.Second { + t.Fatalf("rate-limited LoadState error = %#v, want 2s RetryAfter", loadErr) + } + client.deltaMu.Lock() + afterRaw := client.rawBaseline + repair := client.fullReconciliation + client.deltaMu.Unlock() + client.mu.RLock() + afterNetwork := client.network + afterACL := client.acl + client.mu.RUnlock() + if afterRaw != beforeRaw || repair { + t.Fatalf("rate limit changed raw state: before=%p after=%p repair=%v", beforeRaw, afterRaw, repair) + } + if afterNetwork != beforeNetwork || afterACL != beforeACL { + t.Fatalf("rate limit changed parsed state: network %p/%p ACL %p/%p", beforeNetwork, afterNetwork, beforeACL, afterACL) + } +} + +func TestShouldAbortStateLoadTransientFailures(t *testing.T) { + if !shouldAbortStateLoad(context.Background(), fmt.Errorf("fetching audience material: %w", dwn.ErrTransport)) { + t.Fatal("transport failure must abort and preserve last-good state") + } + if !shouldAbortStateLoad(context.Background(), &dwn.RateLimitError{RetryAfter: time.Second}) { + t.Fatal("rate limit must abort and preserve last-good state") + } + if shouldAbortStateLoad(context.Background(), errors.New("permanently unreadable record")) { + t.Fatal("permanent record error should not abort a usable legacy load") + } +} + +func rawCaptureEntryWithoutInlineData(t *testing.T, entry json.RawMessage) (json.RawMessage, []byte) { + t.Helper() + var object map[string]json.RawMessage + if err := json.Unmarshal(entry, &object); err != nil { + t.Fatal(err) + } + write := object + if wrapped, ok := object["recordsWrite"]; ok { + write = make(map[string]json.RawMessage) + if err := json.Unmarshal(wrapped, &write); err != nil { + t.Fatal(err) + } + } + var encoded string + if err := json.Unmarshal(write["encodedData"], &encoded); err != nil { + t.Fatalf("fixture has no encodedData: %v", err) + } + body, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + t.Fatal(err) + } + delete(write, "encodedData") + if _, wrapped := object["recordsWrite"]; wrapped { + object["recordsWrite"], err = json.Marshal(write) + if err != nil { + t.Fatal(err) + } + } + withoutData, err := json.Marshal(object) + if err != nil { + t.Fatal(err) + } + return withoutData, body +} + +func TestBuildMapResponseOwnsExposedSlices(t *testing.T) { + const selfDID = "did:jwk:self" + client := NewDWNClient("https://dwn.example", selfDID, "network-record", selfDID, nil) + client.network = &NetworkConfig{ + Name: "ownership", + MeshCIDR: "10.200.0.0/16", + DNSServers: []string{"10.200.0.53"}, + } + client.nodes[selfDID] = &NodeRecord{ + DID: selfDID, + MeshIP: "10.200.0.1", + RecordID: "self-node", + Info: &NodeInfoData{ + Hostname: "self", + Capabilities: []string{"ssh"}, + }, + } + + first := client.buildMapResponse() + if first == nil || first.Node == nil || first.DNSConfig == nil { + t.Fatalf("map response = %#v", first) + } + first.Node.Capabilities[0] = "mutated" + first.DNSConfig.Resolvers[0] = "203.0.113.53" + + second := client.buildMapResponse() + if got := second.Node.Capabilities; len(got) != 1 || got[0] != "ssh" { + t.Fatalf("capabilities alias cached NodeInfo: %v", got) + } + if got := second.DNSConfig.Resolvers; len(got) != 1 || got[0] != "10.200.0.53" { + t.Fatalf("DNS resolvers alias cached NetworkConfig: %v", got) + } + + // Under -race this also proves a consumer can mutate its response while a + // subsequent response is built from the immutable control-plane snapshot. + consumer := client.buildMapResponse() + started := make(chan struct{}) + done := make(chan struct{}) + go func() { + close(started) + for i := 0; i < 1000; i++ { + consumer.Node.Capabilities[0] = fmt.Sprintf("consumer-%d", i) + consumer.DNSConfig.Resolvers[0] = fmt.Sprintf("192.0.2.%d", i%255) + } + close(done) + }() + <-started + for i := 0; i < 1000; i++ { + response := client.buildMapResponse() + if response.Node.Capabilities[0] != "ssh" || response.DNSConfig.Resolvers[0] != "10.200.0.53" { + t.Fatalf("cached state changed during consumer mutation: node=%v DNS=%v", response.Node.Capabilities, response.DNSConfig.Resolvers) + } + } + <-done +} diff --git a/internal/control/observability_test.go b/internal/control/observability_test.go index 9a698f4..967b257 100644 --- a/internal/control/observability_test.go +++ b/internal/control/observability_test.go @@ -96,10 +96,48 @@ func TestLoadNodeEntryCountsUndecryptablePeer(t *testing.T) { t.Fatalf("tracked record = %+v, want recordId/memberRecordId set", rec) } - // The counter is cumulative across loads. + // Reprojecting the same opaque record must not create a new episode. c.loadNodeEntry(context.Background(), entry, failing, "member-1") + if got := c.UndecryptablePeerCount(); got != 1 { + t.Fatalf("UndecryptablePeerCount after repeat = %d, want 1", got) + } + const warning = "node record could not be loaded; peer will be invisible until key delivery or record recovery" + if got := handler.count(slog.LevelWarn, warning); got != 1 { + t.Fatalf("repeat warning count = %d, want 1", got) + } + + // A new failure class is a materially new diagnostic episode. + parseFailure := func([]byte, *dwncrypto.Encryption) ([]byte, error) { + return nil, errors.New("malformed ciphertext") + } + c.loadNodeEntry(context.Background(), entry, parseFailure, "member-1") if got := c.UndecryptablePeerCount(); got != 2 { - t.Fatalf("UndecryptablePeerCount after second load = %d, want 2", got) + t.Fatalf("UndecryptablePeerCount after class change = %d, want 2", got) + } + if got := handler.count(slog.LevelWarn, warning); got != 2 { + t.Fatalf("class-change warning count = %d, want 2", got) + } + + // A successful parse closes the episode; the next failure is counted again. + recovering := func([]byte, *dwncrypto.Encryption) ([]byte, error) { + return []byte(`{"meshIP":"10.200.1.9"}`), nil + } + if err := c.loadNodeEntry(context.Background(), entry, recovering, "member-1"); err != nil { + t.Fatalf("recovering loadNodeEntry: %v", err) + } + if _, failed := c.nodeFailures["peer-record"]; failed { + t.Fatal("successful parse retained node failure episode") + } + if got := handler.count(slog.LevelInfo, "node record is readable again"); got != 1 { + t.Fatalf("recovery log count = %d, want 1", got) + } + + c.loadNodeEntry(context.Background(), entry, failing, "member-1") + if got := c.UndecryptablePeerCount(); got != 3 { + t.Fatalf("UndecryptablePeerCount after relapse = %d, want 3", got) + } + if got := handler.count(slog.LevelWarn, warning); got != 3 { + t.Fatalf("relapse warning count = %d, want 3", got) } } @@ -165,14 +203,18 @@ func TestLoadEndpointEntryWarnsOnceAndReportsRecovery(t *testing.T) { }, } - entry := json.RawMessage(`{"recordsWrite":{"recordId":"endpoint-record","descriptor":{"recipient":"` + - peerDID + `","parentId":"` + nodeRecordID + `"},"encodedData":"AAAA","encryption":{}}}`) + entries := []json.RawMessage{ + json.RawMessage(`{"recordsWrite":{"recordId":"endpoint-record-a","descriptor":{"recipient":"` + + peerDID + `","parentId":"` + nodeRecordID + `"},"encodedData":"AAAA","encryption":{}}}`), + json.RawMessage(`{"recordsWrite":{"recordId":"endpoint-record-b","descriptor":{"recipient":"` + + peerDID + `","parentId":"` + nodeRecordID + `"},"encodedData":"AAAA","encryption":{}}}`), + } wantErr := errors.New("delivery query failed") failing := func([]byte, *dwncrypto.Encryption) ([]byte, error) { return nil, wantErr } - for range 2 { + for _, entry := range entries { if err := c.loadEndpointEntry(context.Background(), entry, failing); !errors.Is(err, wantErr) { t.Fatalf("loadEndpointEntry error = %v, want %v", err, wantErr) } @@ -190,7 +232,7 @@ func TestLoadEndpointEntryWarnsOnceAndReportsRecovery(t *testing.T) { recovered := func([]byte, *dwncrypto.Encryption) ([]byte, error) { return []byte(`{"localEndpoints":["192.0.2.1:1234"],"discoKey":"disco","updatedAt":"2026-07-11T00:00:00Z"}`), nil } - if err := c.loadEndpointEntry(context.Background(), entry, recovered); err != nil { + if err := c.loadEndpointEntry(context.Background(), entries[1], recovered); err != nil { t.Fatalf("recovered loadEndpointEntry: %v", err) } if got := len(c.nodes[peerDID].Endpoints); got != 1 { @@ -199,6 +241,15 @@ func TestLoadEndpointEntryWarnsOnceAndReportsRecovery(t *testing.T) { if got := handler.count(slog.LevelInfo, "endpoint record is readable again"); got != 1 { t.Fatalf("endpoint recovery logs = %d, want 1", got) } + if _, failed := c.endpointFailures[nodeRecordID]; failed { + t.Fatal("successful endpoint parse retained parent-slot failure episode") + } + if err := c.loadEndpointEntry(context.Background(), entries[0], failing); !errors.Is(err, wantErr) { + t.Fatalf("post-recovery loadEndpointEntry error = %v, want %v", err, wantErr) + } + if got := handler.count(slog.LevelWarn, "endpoint record could not be loaded; peer connectivity may be degraded"); got != 2 { + t.Fatalf("endpoint warnings after recovery = %d, want a new episode", got) + } } func TestLoadChildRecordsPropagatesRateLimit(t *testing.T) { diff --git a/internal/control/raw_materializer.go b/internal/control/raw_materializer.go new file mode 100644 index 0000000..3d82c7e --- /dev/null +++ b/internal/control/raw_materializer.go @@ -0,0 +1,1082 @@ +package control + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sort" + "strings" + "time" + + "github.com/enboxorg/meshd/internal/dwn" + "github.com/enboxorg/meshd/protocols" +) + +type rawParsedOutcomeKind uint8 + +const ( + rawParsedNetwork rawParsedOutcomeKind = iota + 1 + rawParsedMember + rawParsedNode + rawParsedRelay + rawParsedACL + rawParsedNodeInfo + rawParsedEndpoint +) + +// rawParsedOutcomeKey is the authoritative identity of one parsed contribution. +// The canonical message CID changes for every meaningful RecordsWrite change; +// context/path and the decrypt generation fence parent/key-context changes. +type rawParsedOutcomeKey struct { + kind rawParsedOutcomeKind + recordID string + messageCID string + protocolPath string + contextID string + parentContextID string + recipient string + revision time.Time + generation uint64 +} + +type rawParsedLogicalSlot struct { + kind rawParsedOutcomeKind + recordID string + protocolPath string + parentContextID string + recipient string +} + +type rawParsedSlotOutcome struct { + key rawParsedOutcomeKey + outcome rawParsedOutcome +} + +// rawParsedOutcome owns immutable typed data. opaque means parsing reached a +// stable, non-transient failure and the record contributes only its documented +// ghost/skip/last-good behavior until delivery or full-load invalidation. +type rawParsedOutcome struct { + opaque bool + network *NetworkConfig + member *MemberRecord + node *NodeRecord + relay *RelayData + acl *ACLPolicyData + nodeInfo *NodeInfoData + endpoint *EndpointData +} + +type rawParsedProjection struct { + generation uint64 + previous map[rawParsedOutcomeKey]rawParsedOutcome + previousSlots map[rawParsedLogicalSlot]rawParsedSlotOutcome + currentSlots map[rawParsedLogicalSlot]rawParsedSlotOutcome + next map[rawParsedOutcomeKey]rawParsedOutcome +} + +const revokedSelfExpiry = "1970-01-01T00:00:00Z" + +func (c *DWNClient) beginRawParsedProjection() *rawParsedProjection { + c.mu.RLock() + defer c.mu.RUnlock() + previous := make(map[rawParsedOutcomeKey]rawParsedOutcome, len(c.rawParsedOutcomes)) + for key, outcome := range c.rawParsedOutcomes { + previous[key] = outcome + } + projection := &rawParsedProjection{ + generation: c.rawParsedGeneration, + previous: previous, + previousSlots: make(map[rawParsedLogicalSlot]rawParsedSlotOutcome, len(previous)), + currentSlots: make(map[rawParsedLogicalSlot]rawParsedSlotOutcome, len(previous)), + next: make(map[rawParsedOutcomeKey]rawParsedOutcome, len(previous)), + } + for key, outcome := range previous { + stageRawParsedSlot(projection.previousSlots, key, outcome) + } + return projection +} + +func (c *DWNClient) invalidateRawParsedOutcomes() { + c.mu.Lock() + c.rawParsedGeneration++ + refreshed := make(map[rawParsedOutcomeKey]rawParsedOutcome, len(c.rawParsedOutcomes)) + for key, outcome := range c.rawParsedOutcomes { + if !outcome.opaque { + key.generation = c.rawParsedGeneration + } + refreshed[key] = outcome + } + c.rawParsedOutcomes = refreshed + c.mu.Unlock() +} + +func (p *rawParsedProjection) key(record rawMeshRecord, kind rawParsedOutcomeKind) rawParsedOutcomeKey { + return rawParsedOutcomeKey{ + kind: kind, + recordID: record.recordID, + messageCID: record.messageCID, + protocolPath: record.protocolPath, + contextID: record.contextID, + parentContextID: record.parentContextID, + recipient: record.recipient, + revision: record.revision, + generation: p.generation, + } +} + +func (p *rawParsedProjection) lastGood(record rawMeshRecord, kind rawParsedOutcomeKind) (rawParsedOutcome, bool) { + slot := rawParsedSlot(record, kind) + if staged, ok := p.currentSlots[slot]; ok { + return cloneRawParsedOutcome(staged.outcome), true + } + previous, ok := p.previousSlots[slot] + return cloneRawParsedOutcome(previous.outcome), ok +} + +func (p *rawParsedProjection) lastGoodNodeForRecipient(recipient string) (*NodeRecord, bool) { + var newest rawParsedSlotOutcome + found := false + consider := func(slots map[rawParsedLogicalSlot]rawParsedSlotOutcome) { + for slot, candidate := range slots { + if slot.kind != rawParsedNode || slot.recipient != recipient || candidate.outcome.node == nil { + continue + } + if !found || rawParsedOutcomeKeyIsNewer(candidate.key, newest.key) { + newest = candidate + found = true + } + } + } + consider(p.previousSlots) + consider(p.currentSlots) + if !found { + return nil, false + } + return cloneNodeRecord(newest.outcome.node), true +} + +func rawParsedSlot(record rawMeshRecord, kind rawParsedOutcomeKind) rawParsedLogicalSlot { + return newRawParsedLogicalSlot(kind, record.recordID, record.recipient, record.protocolPath, record.parentContextID) +} + +func rawParsedSlotFromKey(key rawParsedOutcomeKey) rawParsedLogicalSlot { + return newRawParsedLogicalSlot(key.kind, key.recordID, key.recipient, key.protocolPath, key.parentContextID) +} + +func newRawParsedLogicalSlot( + kind rawParsedOutcomeKind, + recordID string, + recipient string, + protocolPath string, + parentContextID string, +) rawParsedLogicalSlot { + slot := rawParsedLogicalSlot{ + kind: kind, recordID: recordID, protocolPath: protocolPath, parentContextID: parentContextID, + } + switch kind { + case rawParsedMember, rawParsedNode: + if recipient != "" { + slot.recordID = "" + slot.recipient = recipient + } + case rawParsedACL, rawParsedNodeInfo, rawParsedEndpoint: + slot.recordID = "" + } + return slot +} + +func stageRawParsedSlot( + slots map[rawParsedLogicalSlot]rawParsedSlotOutcome, + key rawParsedOutcomeKey, + outcome rawParsedOutcome, +) { + if !rawParsedOutcomeHasContribution(key.kind, outcome) { + return + } + slot := rawParsedSlotFromKey(key) + current, ok := slots[slot] + if !ok || rawParsedOutcomeKeyIsNewer(key, current.key) { + slots[slot] = rawParsedSlotOutcome{key: key, outcome: outcome} + } +} + +func rawParsedOutcomeHasContribution(kind rawParsedOutcomeKind, outcome rawParsedOutcome) bool { + switch kind { + case rawParsedNetwork: + return outcome.network != nil + case rawParsedMember: + return outcome.member != nil + case rawParsedNode: + return outcome.node != nil + case rawParsedRelay: + return outcome.relay != nil + case rawParsedACL: + return outcome.acl != nil + case rawParsedNodeInfo: + return outcome.nodeInfo != nil + case rawParsedEndpoint: + return outcome.endpoint != nil + default: + return false + } +} + +func rawParsedOutcomeKeyIsNewer(candidate, current rawParsedOutcomeKey) bool { + if candidate.revision.Before(current.revision) { + return false + } + if candidate.revision.After(current.revision) { + return true + } + return candidate.messageCID > current.messageCID +} + +func (p *rawParsedProjection) lookup(record rawMeshRecord, kind rawParsedOutcomeKind) (rawParsedOutcome, bool) { + key := p.key(record, kind) + outcome, ok := p.previous[key] + if !ok { + return rawParsedOutcome{}, false + } + p.next[key] = outcome + stageRawParsedSlot(p.currentSlots, key, outcome) + return cloneRawParsedOutcome(outcome), true +} + +func (p *rawParsedProjection) store(record rawMeshRecord, kind rawParsedOutcomeKind, outcome rawParsedOutcome) { + key := p.key(record, kind) + owned := cloneRawParsedOutcome(outcome) + p.next[key] = owned + stageRawParsedSlot(p.currentSlots, key, owned) +} + +// materializeRawMeshRecordSet projects a complete raw record snapshot into a +// new parsed control-plane state. The caller must serialize calls with loadMu. +// No state reachable from c is mutated until a usable MapResponse has been +// built, so a failed, canceled, or rate-limited projection leaves the previous +// state (including its pointer identity) untouched. +func (c *DWNClient) materializeRawMeshRecordSet(ctx context.Context, set *rawMeshRecordSet) (*MapResponse, error) { + projection, err := c.projectRawMeshRecordSetWithDecryptors(ctx, set, c.makeDecryptor) + if err != nil { + return nil, err + } + c.commitRawMeshMaterialization(projection) + return projection.response, nil +} + +// materializeRawMeshRecordSetWithDecryptors is split out to make abort and +// rollback behavior deterministic in tests. Production callers use +// materializeRawMeshRecordSet, which supplies the client's normal decryptors. +func (c *DWNClient) materializeRawMeshRecordSetWithDecryptors( + ctx context.Context, + set *rawMeshRecordSet, + decryptorFor func(context.Context, string) EntryDecryptor, +) (*MapResponse, error) { + projection, err := c.projectRawMeshRecordSetWithDecryptors(ctx, set, decryptorFor) + if err != nil { + return nil, err + } + c.commitRawMeshMaterialization(projection) + return projection.response, nil +} + +type rawMeshMaterialization struct { + builder *DWNClient + response *MapResponse + parsedGeneration uint64 + parsedOutcomes map[rawParsedOutcomeKey]rawParsedOutcome +} + +func (c *DWNClient) projectRawMeshRecordSetWithDecryptors( + ctx context.Context, + set *rawMeshRecordSet, + decryptorFor func(context.Context, string) EntryDecryptor, +) (*rawMeshMaterialization, error) { + if c == nil { + return nil, fmt.Errorf("materializing raw mesh state: nil DWN client") + } + if ctx == nil { + return nil, fmt.Errorf("materializing raw mesh state: nil context") + } + if set == nil { + return nil, fmt.Errorf("materializing raw mesh state: nil record set") + } + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("materializing raw mesh state: %w", err) + } + + groups := groupRawMeshMapRecords(set.all(), c.networkRecordID) + if groups.network == nil { + return nil, fmt.Errorf("%w: record %q is absent from local state", ErrNoNetwork, c.networkRecordID) + } + parsed := c.beginRawParsedProjection() + + logger := c.logger + if logger == nil { + logger = slog.Default() + } + endpointFailures := c.cloneEndpointFailures() + nodeFailures := c.cloneNodeFailures() + previousACL := c.cloneACLPolicy() + builder := &DWNClient{ + networkRecordID: c.networkRecordID, + selfDID: c.selfDID, + logger: logger, + members: make(map[string]*MemberRecord), + nodes: make(map[string]*NodeRecord), + endpointFailures: endpointFailures, + nodeFailures: nodeFailures, + } + + if err := requireRawMaterializationData(*groups.network); err != nil { + return nil, err + } + if outcome, ok := parsed.lookup(*groups.network, rawParsedNetwork); ok && !outcome.opaque && outcome.network != nil { + builder.network = outcome.network + } else { + var network NetworkConfig + if err := ParseEntryData(groups.network.raw, &network, nil); err != nil { + return nil, fmt.Errorf("parsing materialized network: %w", err) + } + builder.network = &network + parsed.store(*groups.network, rawParsedNetwork, rawParsedOutcome{network: &network}) + } + + decryptors := make(map[string]EntryDecryptor) + decryptor := func(path string) EntryDecryptor { + if dec, ok := decryptors[path]; ok { + return dec + } + var dec EntryDecryptor + if decryptorFor != nil { + dec = decryptorFor(ctx, path) + } + decryptors[path] = dec + return dec + } + + // Members must be materialized before member-associated nodes because the + // member record ID defines each node's direct parent context. + sortRawRecordsOldestFirst(groups.members) + for _, record := range groups.members { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("materializing members: %w", err) + } + if err := requireRawMaterializationData(record); err != nil { + return nil, err + } + var member MemberRecord + if outcome, ok := parsed.lookup(record, rawParsedMember); ok { + if outcome.member != nil { + member = *outcome.member + } + member.DID = record.recipient + member.RecordID = record.recordID + if member.DID != "" { + builder.members[member.DID] = &member + } + continue + } + if err := ParseEntryData(record.raw, &member, decryptor(record.protocolPath)); err != nil { + logger.DebugContext(ctx, "parsing materialized member entry", + slog.Any("error", err), slog.String("memberDID", record.recipient)) + if shouldAbortRawMaterialization(ctx, err) { + return nil, fmt.Errorf("parsing materialized member %s: %w", record.recordID, err) + } + // Match LoadState: public descriptor metadata keeps an opaque member + // addressable until its audience key arrives. + outcome := rawParsedOutcome{opaque: true} + if errors.Is(err, errAudienceKeyDeliveryAbsent) { + if previous, ok := parsed.lastGood(record, rawParsedMember); ok && previous.member != nil { + member = *previous.member + outcome.member = &member + } + } + member.DID = record.recipient + member.RecordID = record.recordID + if outcome.member != nil { + outcome.member = &member + } + parsed.store(record, rawParsedMember, outcome) + if member.DID != "" { + builder.members[member.DID] = &member + } + continue + } + member.DID = record.recipient + member.RecordID = record.recordID + parsed.store(record, rawParsedMember, rawParsedOutcome{member: &member}) + if member.DID != "" { + builder.members[member.DID] = &member + } + } + + memberRecordIDs := make(map[string]struct{}, len(builder.members)) + for _, member := range builder.members { + if member.RecordID != "" { + memberRecordIDs[member.RecordID] = struct{}{} + } + } + + currentNodeFailures := make(map[string]struct{}, len(groups.ownerNodes)+len(groups.memberNodes)) + + // Preserve full-load precedence: owner nodes are loaded first and a + // member-associated record for the same DID wins afterward. + sortRawRecordsOldestFirst(groups.ownerNodes) + for _, record := range groups.ownerNodes { + currentNodeFailures[record.recordID] = struct{}{} + if err := materializeNodeRecord(ctx, builder, parsed, record, "", decryptor(record.protocolPath)); err != nil { + return nil, err + } + } + sortRawRecordsOldestFirst(groups.memberNodes) + for _, record := range groups.memberNodes { + if _, ok := memberRecordIDs[record.parentID]; !ok { + continue + } + parentContext := c.networkRecordID + "/" + record.parentID + if !rawRecordIsDirectChild(record, parentContext) { + continue + } + currentNodeFailures[record.recordID] = struct{}{} + if err := materializeNodeRecord(ctx, builder, parsed, record, record.parentID, decryptor(record.protocolPath)); err != nil { + return nil, err + } + } + + for failureKey := range builder.nodeFailures { + if _, current := currentNodeFailures[failureKey]; !current { + delete(builder.nodeFailures, failureKey) + } + } + self := builder.nodes[builder.selfDID] + if self == nil { + if builder.selfDID == "" { + return nil, fmt.Errorf("materializing raw mesh state: self DID is empty") + } + installRevokedSelfNode(builder, parsed) + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("materializing raw mesh state: %w", err) + } + return finishRawMeshMaterialization(c, builder, parsed) + } + if self.Opaque { + return nil, fmt.Errorf("materializing self node %s: descriptor-only node is not authorized", self.RecordID) + } + if self.Revoked || nodeRecordExpired(self, time.Now().UTC()) { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("materializing raw mesh state: %w", err) + } + return finishRawMeshMaterialization(c, builder, parsed) + } + + sortRawRelaysOldestFirst(groups.relays) + for _, record := range groups.relays { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("materializing relays: %w", err) + } + if err := requireRawMaterializationData(record); err != nil { + return nil, err + } + if outcome, ok := parsed.lookup(record, rawParsedRelay); ok { + if outcome.relay == nil { + return nil, fmt.Errorf("materialized relay %s is unreadable and no last-good relay is available", record.recordID) + } + builder.relays = append(builder.relays, outcome.relay) + continue + } + var relay RelayData + if err := ParseEntryData(record.raw, &relay, decryptor(record.protocolPath)); err != nil { + logger.DebugContext(ctx, "parsing materialized relay entry", slog.Any("error", err)) + if shouldAbortRawMaterialization(ctx, err) { + return nil, fmt.Errorf("parsing materialized relay %s: %w", record.recordID, err) + } + previous, ok := parsed.lastGood(record, rawParsedRelay) + if !ok || previous.relay == nil { + return nil, fmt.Errorf("parsing materialized relay %s without a last-good relay: %w", record.recordID, err) + } + parsed.store(record, rawParsedRelay, rawParsedOutcome{opaque: true, relay: previous.relay}) + builder.relays = append(builder.relays, previous.relay) + continue + } + parsed.store(record, rawParsedRelay, rawParsedOutcome{relay: &relay}) + builder.relays = append(builder.relays, &relay) + } + + if groups.acl != nil { + if err := requireRawMaterializationData(*groups.acl); err != nil { + return nil, err + } + if outcome, ok := parsed.lookup(*groups.acl, rawParsedACL); ok { + if outcome.opaque { + if previousACL == nil { + return nil, fmt.Errorf("materialized ACL %s is unreadable and no last-good policy is available", groups.acl.recordID) + } + builder.acl = previousACL + } else { + builder.acl = outcome.acl + } + } else { + var policy ACLPolicyData + if err := ParseEntryData(groups.acl.raw, &policy, decryptor(groups.acl.protocolPath)); err != nil { + logger.DebugContext(ctx, "parsing materialized ACL policy", slog.Any("error", err)) + if shouldAbortRawMaterialization(ctx, err) { + return nil, fmt.Errorf("parsing materialized ACL policy: %w", err) + } + if previousACL == nil { + return nil, fmt.Errorf("parsing materialized ACL policy without a last-good policy: %w", err) + } + // Keep the prior parsed ACL until the opaque replacement becomes + // decryptable, matching legacy LoadState's last-good policy behavior. + builder.acl = previousACL + parsed.store(*groups.acl, rawParsedACL, rawParsedOutcome{opaque: true}) + } else { + builder.acl = &policy + parsed.store(*groups.acl, rawParsedACL, rawParsedOutcome{acl: &policy}) + } + } + } + + nodeParents := materializedNodeParents(c.networkRecordID, builder.nodes) + sortRawRecordsNewestFirst(groups.nodeInfo) + seenNodeInfo := make(map[string]struct{}, len(nodeParents)) + for _, record := range groups.nodeInfo { + parent, ok := nodeParents[record.parentID] + if !ok || record.protocolPath != parent.infoPath || !rawRecordIsDirectChild(record, parent.contextID) { + continue + } + if _, seen := seenNodeInfo[record.parentID]; seen { + continue + } + seenNodeInfo[record.parentID] = struct{}{} + if err := requireRawMaterializationData(record); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("materializing node info: %w", err) + } + if outcome, ok := parsed.lookup(record, rawParsedNodeInfo); ok { + if outcome.nodeInfo != nil { + parent.node.Info = outcome.nodeInfo + } + continue + } + var info NodeInfoData + if err := ParseEntryData(record.raw, &info, decryptor(record.protocolPath)); err != nil { + logger.DebugContext(ctx, "parsing materialized nodeInfo entry", slog.Any("error", err)) + if shouldAbortRawMaterialization(ctx, err) { + return nil, fmt.Errorf("decrypting materialized node info %s: %w", record.recordID, err) + } + outcome := rawParsedOutcome{opaque: true} + if errors.Is(err, errAudienceKeyDeliveryAbsent) { + if previous, ok := parsed.lastGood(record, rawParsedNodeInfo); ok { + outcome.nodeInfo = previous.nodeInfo + if previous.nodeInfo != nil { + parent.node.Info = previous.nodeInfo + } + } + } + parsed.store(record, rawParsedNodeInfo, outcome) + continue + } + parent.node.Info = &info + parsed.store(record, rawParsedNodeInfo, rawParsedOutcome{nodeInfo: &info}) + } + + currentEndpointFailures := make(map[string]struct{}, len(groups.endpoints)) + sortRawRecordsNewestFirst(groups.endpoints) + seenEndpoints := make(map[string]struct{}, len(nodeParents)) + for _, record := range groups.endpoints { + parent, ok := nodeParents[record.parentID] + if !ok || record.protocolPath != parent.endpointPath || !rawRecordIsDirectChild(record, parent.contextID) { + continue + } + if _, seen := seenEndpoints[record.parentID]; seen { + continue + } + seenEndpoints[record.parentID] = struct{}{} + failureKey := record.parentID + if failureKey == "" { + failureKey = record.recordID + } + currentEndpointFailures[failureKey] = struct{}{} + if err := requireRawMaterializationData(record); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("materializing endpoints: %w", err) + } + if outcome, ok := parsed.lookup(record, rawParsedEndpoint); ok { + if outcome.endpoint != nil { + parent.node.Endpoints = append(parent.node.Endpoints, *outcome.endpoint) + } + continue + } + endpoint, err := materializeEndpointRecord(ctx, builder, parent.node, record, decryptor(record.protocolPath)) + if err != nil { + if shouldAbortRawMaterialization(ctx, err) { + return nil, fmt.Errorf("decrypting materialized endpoint %s: %w", record.recordID, err) + } + outcome := rawParsedOutcome{opaque: true} + if errors.Is(err, errAudienceKeyDeliveryAbsent) { + if previous, ok := parsed.lastGood(record, rawParsedEndpoint); ok { + outcome.endpoint = previous.endpoint + if previous.endpoint != nil { + parent.node.Endpoints = append(parent.node.Endpoints, *previous.endpoint) + } + } + } + parsed.store(record, rawParsedEndpoint, outcome) + continue + } + parsed.store(record, rawParsedEndpoint, rawParsedOutcome{endpoint: endpoint}) + } + for failureKey := range builder.endpointFailures { + if _, current := currentEndpointFailures[failureKey]; !current { + delete(builder.endpointFailures, failureKey) + } + } + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("materializing raw mesh state: %w", err) + } + return finishRawMeshMaterialization(c, builder, parsed) +} + +func installRevokedSelfNode(builder *DWNClient, parsed *rawParsedProjection) { + revoked, ok := parsed.lastGoodNodeForRecipient(builder.selfDID) + if !ok { + revoked = &NodeRecord{DID: builder.selfDID} + } + revoked.DID = builder.selfDID + revoked.Opaque = false + revoked.Revoked = true + revoked.ExpiresAt = revokedSelfExpiry + builder.nodes[builder.selfDID] = revoked +} + +func finishRawMeshMaterialization( + c *DWNClient, + builder *DWNClient, + parsed *rawParsedProjection, +) (*rawMeshMaterialization, error) { + response := builder.buildMapResponse() + if response == nil { + return nil, fmt.Errorf("self DID %q not found in materialized network node records", c.selfDID) + } + + return &rawMeshMaterialization{ + builder: builder, + response: response, + parsedGeneration: parsed.generation, + parsedOutcomes: parsed.next, + }, nil +} + +func (c *DWNClient) commitRawMeshMaterialization(projection *rawMeshMaterialization) { + c.mu.Lock() + c.commitRawMeshMaterializationLocked(projection) + c.mu.Unlock() + c.commitRawMeshMaterializationCounters(projection) +} + +func (c *DWNClient) commitRawMeshMaterializationLocked(projection *rawMeshMaterialization) { + builder := projection.builder + c.network = builder.network + c.members = builder.members + c.nodes = builder.nodes + c.relays = builder.relays + c.acl = builder.acl + c.nodeFailures = builder.nodeFailures + c.endpointFailures = builder.endpointFailures + if c.rawParsedGeneration == projection.parsedGeneration { + c.rawParsedOutcomes = projection.parsedOutcomes + } +} + +func (c *DWNClient) commitRawMeshMaterializationCounters(projection *rawMeshMaterialization) { + builder := projection.builder + c.undecryptablePeers.Add(builder.undecryptablePeers.Load()) + c.unreadableEndpoints.Add(builder.unreadableEndpoints.Load()) + c.droppedPeers.Add(builder.droppedPeers.Load()) +} + +type rawMeshMapRecordGroups struct { + network *rawMeshRecord + members []rawMeshRecord + ownerNodes []rawMeshRecord + memberNodes []rawMeshRecord + relays []rawMeshRecord + acl *rawMeshRecord + nodeInfo []rawMeshRecord + endpoints []rawMeshRecord +} + +func groupRawMeshMapRecords(records []rawMeshRecord, networkRecordID string) rawMeshMapRecordGroups { + var groups rawMeshMapRecordGroups + for i := range records { + record := records[i] + if record.protocol != protocols.MeshProtocolURI { + continue + } + switch record.protocolPath { + case "network": + if record.recordID == networkRecordID && (record.contextID == "" || record.contextID == record.recordID) { + copy := record + groups.network = © + } + case "network/member": + if rawRecordIsDirectChild(record, networkRecordID) { + groups.members = append(groups.members, record) + } + case "network/node": + if rawRecordIsDirectChild(record, networkRecordID) { + groups.ownerNodes = append(groups.ownerNodes, record) + } + case "network/member/node": + groups.memberNodes = append(groups.memberNodes, record) + case "network/relay": + if rawRecordIsDirectChild(record, networkRecordID) { + groups.relays = append(groups.relays, record) + } + case "network/aclPolicy": + if rawRecordIsDirectChild(record, networkRecordID) && rawRecordIsNewer(record, groups.acl) { + copy := record + groups.acl = © + } + case "network/node/nodeInfo", "network/member/node/nodeInfo": + groups.nodeInfo = append(groups.nodeInfo, record) + case "network/node/endpoint", "network/member/node/endpoint": + groups.endpoints = append(groups.endpoints, record) + } + } + return groups +} + +func rawRecordIsNewer(candidate rawMeshRecord, current *rawMeshRecord) bool { + return current == nil || compareRawMeshRecordRevision(candidate.head(), current.head()) > 0 +} + +func rawRecordIsDirectChild(record rawMeshRecord, parentContextID string) bool { + parentID := parentContextID + if separator := strings.LastIndexByte(parentID, '/'); separator >= 0 { + parentID = parentID[separator+1:] + } + if record.parentID != parentID { + return false + } + return record.contextID == parentContextID+"/"+record.recordID +} + +func sortRawRecordsOldestFirst(records []rawMeshRecord) { + sort.Slice(records, func(i, j int) bool { + if comparison := compareRawMeshRecordRevision(records[i].head(), records[j].head()); comparison != 0 { + return comparison < 0 + } + return records[i].recordID < records[j].recordID + }) +} + +func sortRawRecordsNewestFirst(records []rawMeshRecord) { + sort.Slice(records, func(i, j int) bool { + if comparison := compareRawMeshRecordRevision(records[i].head(), records[j].head()); comparison != 0 { + return comparison > 0 + } + return records[i].recordID > records[j].recordID + }) +} + +// Relay region IDs are positional, so updates must preserve RecordsQuery's +// createdAscending order rather than reordering a relay by update timestamp. +func sortRawRelaysOldestFirst(records []rawMeshRecord) { + sort.Slice(records, func(i, j int) bool { + if records[i].dateCreatedTime.Before(records[j].dateCreatedTime) { + return true + } + if records[i].dateCreatedTime.After(records[j].dateCreatedTime) { + return false + } + if comparison := compareRawMeshRecordRevision(records[i].head(), records[j].head()); comparison != 0 { + return comparison < 0 + } + return records[i].recordID < records[j].recordID + }) +} + +func requireRawMaterializationData(record rawMeshRecord) error { + if !rawRecordHasEncodedData(record.raw) { + return fmt.Errorf("materializing %s record %s: %w", record.protocolPath, record.recordID, errRawMeshRecordDataUnavailable) + } + return nil +} + +func materializeNodeRecord( + ctx context.Context, + builder *DWNClient, + parsed *rawParsedProjection, + record rawMeshRecord, + memberRecordID string, + decryptor EntryDecryptor, +) error { + if err := requireRawMaterializationData(record); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("materializing nodes: %w", err) + } + if outcome, ok := parsed.lookup(record, rawParsedNode); ok { + if outcome.node != nil { + outcome.node.DID = record.recipient + outcome.node.RecordID = record.recordID + outcome.node.MemberRecordID = memberRecordID + outcome.node.Opaque = false + outcome.node.Revoked = false + if record.recipient != "" { + builder.nodes[record.recipient] = outcome.node + } + return nil + } + if outcome.opaque { + if record.recipient == builder.selfDID { + return fmt.Errorf("materializing self node %s: unreadable without a recipient-matched last-good node", record.recordID) + } + if record.recipient != "" { + builder.nodes[record.recipient] = &NodeRecord{ + DID: record.recipient, RecordID: record.recordID, MemberRecordID: memberRecordID, Opaque: true, + } + } + return nil + } + return nil + } + if err := builder.loadNodeEntry(ctx, record.raw, decryptor, memberRecordID); err != nil { + if shouldAbortRawMaterialization(ctx, err) { + return fmt.Errorf("decrypting materialized node %s: %w", record.recordID, err) + } + outcome := rawParsedOutcome{opaque: true} + if errors.Is(err, errAudienceKeyDeliveryAbsent) { + if previous, ok := parsed.lastGood(record, rawParsedNode); ok && previous.node != nil { + outcome.node = previous.node + outcome.node.DID = record.recipient + outcome.node.RecordID = record.recordID + outcome.node.MemberRecordID = memberRecordID + outcome.node.Opaque = false + outcome.node.Revoked = false + if record.recipient != "" { + builder.nodes[record.recipient] = outcome.node + } + } + } + if record.recipient == builder.selfDID && outcome.node == nil { + return fmt.Errorf("materializing self node %s: unreadable without a recipient-matched last-good node: %w", record.recordID, err) + } + parsed.store(record, rawParsedNode, outcome) + return nil + } + if node := builder.nodes[record.recipient]; node != nil && node.RecordID == record.recordID { + parsed.store(record, rawParsedNode, rawParsedOutcome{node: node}) + } else { + // A successfully decoded node without a public recipient contributes + // nothing; cache the stable skip so unrelated deltas do not reparse it. + parsed.store(record, rawParsedNode, rawParsedOutcome{opaque: true}) + } + + return nil +} + +func materializeEndpointRecord( + ctx context.Context, + builder *DWNClient, + parentNode *NodeRecord, + record rawMeshRecord, + decryptor EntryDecryptor, +) (*EndpointData, error) { + failureKey := record.parentID + if failureKey == "" { + failureKey = record.recordID + } + var endpoint EndpointData + if err := ParseEntryData(record.raw, &endpoint, decryptor); err != nil { + builder.unreadableEndpoints.Add(1) + if builder.endpointFailures == nil { + builder.endpointFailures = make(map[string]string) + } + failureClass := endpointFailureClass(err) + if previous, warned := builder.endpointFailures[failureKey]; !warned || previous != failureClass { + builder.logger.WarnContext(ctx, "endpoint record could not be loaded; peer connectivity may be degraded", + slog.Any("error", err), + slog.String("failureClass", failureClass), + slog.String("nodeDID", parentNode.DID), + slog.String("recordId", record.recordID), + slog.String("parentId", record.parentID), + ) + } + builder.endpointFailures[failureKey] = failureClass + return nil, err + } + if _, recovering := builder.endpointFailures[failureKey]; recovering { + delete(builder.endpointFailures, failureKey) + builder.logger.InfoContext(ctx, "endpoint record is readable again", + slog.String("recordId", record.recordID), + slog.String("parentId", record.parentID), + ) + } + parentNode.Endpoints = append(parentNode.Endpoints, endpoint) + return &endpoint, nil +} + +type materializedNodeParent struct { + node *NodeRecord + contextID string + infoPath string + endpointPath string +} + +func materializedNodeParents(networkRecordID string, nodes map[string]*NodeRecord) map[string]materializedNodeParent { + parents := make(map[string]materializedNodeParent, len(nodes)) + for _, node := range nodes { + if node == nil || node.RecordID == "" { + continue + } + if node.MemberRecordID != "" { + parents[node.RecordID] = materializedNodeParent{ + node: node, + contextID: networkRecordID + "/" + node.MemberRecordID + "/" + node.RecordID, + infoPath: "network/member/node/nodeInfo", + endpointPath: "network/member/node/endpoint", + } + continue + } + parents[node.RecordID] = materializedNodeParent{ + node: node, + contextID: networkRecordID + "/" + node.RecordID, + infoPath: "network/node/nodeInfo", + endpointPath: "network/node/endpoint", + } + } + return parents +} + +func (c *DWNClient) cloneEndpointFailures() map[string]string { + c.mu.RLock() + defer c.mu.RUnlock() + cloned := make(map[string]string, len(c.endpointFailures)) + for key, class := range c.endpointFailures { + cloned[key] = class + } + return cloned +} + +func (c *DWNClient) cloneNodeFailures() map[string]string { + c.mu.RLock() + defer c.mu.RUnlock() + cloned := make(map[string]string, len(c.nodeFailures)) + for key, class := range c.nodeFailures { + cloned[key] = class + } + return cloned +} + +func shouldAbortRawMaterialization(ctx context.Context, err error) bool { + return shouldAbortStateLoad(ctx, err) || errors.Is(err, dwn.ErrTransport) +} + +func (c *DWNClient) cloneACLPolicy() *ACLPolicyData { + c.mu.RLock() + defer c.mu.RUnlock() + return cloneACLPolicyData(c.acl) +} + +func cloneRawParsedOutcome(outcome rawParsedOutcome) rawParsedOutcome { + outcome.network = cloneNetworkConfig(outcome.network) + outcome.member = cloneMemberRecord(outcome.member) + outcome.node = cloneNodeRecord(outcome.node) + outcome.relay = cloneRelayData(outcome.relay) + outcome.acl = cloneACLPolicyData(outcome.acl) + outcome.nodeInfo = cloneNodeInfoData(outcome.nodeInfo) + outcome.endpoint = cloneEndpointData(outcome.endpoint) + return outcome +} + +func cloneNetworkConfig(source *NetworkConfig) *NetworkConfig { + if source == nil { + return nil + } + cloned := *source + cloned.DNSServers = append([]string(nil), source.DNSServers...) + return &cloned +} + +func cloneMemberRecord(source *MemberRecord) *MemberRecord { + if source == nil { + return nil + } + cloned := *source + return &cloned +} + +func cloneNodeRecord(source *NodeRecord) *NodeRecord { + if source == nil { + return nil + } + cloned := *source + cloned.AllowedIPs = append([]string(nil), source.AllowedIPs...) + cloned.Info = cloneNodeInfoData(source.Info) + if source.Endpoints != nil { + cloned.Endpoints = make([]EndpointData, len(source.Endpoints)) + for i := range source.Endpoints { + cloned.Endpoints[i] = *cloneEndpointData(&source.Endpoints[i]) + } + } + return &cloned +} + +func cloneRelayData(source *RelayData) *RelayData { + if source == nil { + return nil + } + cloned := *source + return &cloned +} + +func cloneACLPolicyData(source *ACLPolicyData) *ACLPolicyData { + if source == nil { + return nil + } + cloned := *source + if source.Groups != nil { + cloned.Groups = make(map[string][]string, len(source.Groups)) + for name, members := range source.Groups { + cloned.Groups[name] = append([]string(nil), members...) + } + } + if source.Rules != nil { + cloned.Rules = make([]ACLRule, len(source.Rules)) + for i, rule := range source.Rules { + cloned.Rules[i] = rule + cloned.Rules[i].Src = append([]string(nil), rule.Src...) + cloned.Rules[i].Dst = append([]string(nil), rule.Dst...) + cloned.Rules[i].SrcPorts = append([]string(nil), rule.SrcPorts...) + cloned.Rules[i].DstPorts = append([]string(nil), rule.DstPorts...) + } + } + return &cloned +} + +func cloneNodeInfoData(source *NodeInfoData) *NodeInfoData { + if source == nil { + return nil + } + cloned := *source + cloned.Capabilities = append([]string(nil), source.Capabilities...) + return &cloned +} + +func cloneEndpointData(source *EndpointData) *EndpointData { + if source == nil { + return nil + } + cloned := *source + cloned.PublicEndpoints = append([]PublicEndpoint(nil), source.PublicEndpoints...) + cloned.LocalEndpoints = append([]string(nil), source.LocalEndpoints...) + return &cloned +} diff --git a/internal/control/raw_materializer_test.go b/internal/control/raw_materializer_test.go new file mode 100644 index 0000000..ddf90e5 --- /dev/null +++ b/internal/control/raw_materializer_test.go @@ -0,0 +1,1646 @@ +package control + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strings" + "testing" + "time" + + "github.com/enboxorg/meshd/internal/dwn" + dwncrypto "github.com/enboxorg/meshd/internal/dwn/crypto" + "github.com/enboxorg/meshd/protocols" +) + +const ( + materializerNetworkID = "network-record" + materializerSelfDID = "did:jwk:self" + materializerPeerDID = "did:jwk:peer" +) + +func TestMaterializeRawMeshRecordSetProjectsCompleteMapState(t *testing.T) { + records := []json.RawMessage{ + materializerRecord(t, materializerRecordSpec{ + id: "unknown-record", path: "network/invite", parentContext: materializerNetworkID, + data: json.RawMessage(`not-json`), timestamp: "2026-07-11T12:00:20Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "foreign-node", protocol: "https://example.com/not-mesh", path: "network/node", + parentContext: materializerNetworkID, recipient: "did:jwk:foreign", + data: json.RawMessage(`not-json`), timestamp: "2026-07-11T12:00:21Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: materializerNetworkID, path: "network", + data: NetworkConfig{Name: "production", MeshCIDR: "10.200.0.0/16", DNSServers: []string{"10.200.0.53"}, MagicDNSSuffix: "prod.mesh"}, + timestamp: "2026-07-11T12:00:00Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "member-record", path: "network/member", parentContext: materializerNetworkID, + recipient: "did:jwk:member", data: MemberRecord{Label: "member", AddedAt: "2026-07-11T12:00:01Z"}, + timestamp: "2026-07-11T12:00:01Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "self-node", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, + data: NodeRecord{MeshIP: "10.200.1.1", Label: "self-label", AddedAt: "2026-07-11T12:00:02Z"}, + timestamp: "2026-07-11T12:00:02Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "peer-node", path: "network/member/node", parentContext: materializerNetworkID + "/member-record", + recipient: materializerPeerDID, + data: NodeRecord{MeshIP: "10.200.1.2", Label: "peer-label", OwnerDID: "did:jwk:member", AddedAt: "2026-07-11T12:00:03Z"}, + timestamp: "2026-07-11T12:00:03Z", + }), + // This orphan is in the mesh protocol but outside the materialized + // member dependency graph and must not enter the node map. + materializerRecord(t, materializerRecordSpec{ + id: "orphan-node", path: "network/member/node", parentContext: materializerNetworkID + "/missing-member", + recipient: "did:jwk:orphan", data: NodeRecord{MeshIP: "10.200.1.9"}, + timestamp: "2026-07-11T12:00:04Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "relay-record", path: "network/relay", parentContext: materializerNetworkID, + data: RelayData{URL: "relay.example.com", Region: "test", STUNPort: 3478}, + timestamp: "2026-07-11T12:00:05Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "acl-old", path: "network/aclPolicy", parentContext: materializerNetworkID, + data: ACLPolicyData{Version: 1, DefaultAction: "deny"}, timestamp: "2026-07-11T12:00:06Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "acl-new", path: "network/aclPolicy", parentContext: materializerNetworkID, + data: ACLPolicyData{Version: 2, DefaultAction: "accept", Groups: map[string][]string{"all": {materializerSelfDID, materializerPeerDID}}}, + timestamp: "2026-07-11T12:00:07Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "self-info-old", path: "network/node/nodeInfo", parentContext: materializerNetworkID + "/self-node", + data: NodeInfoData{Hostname: "old-self"}, timestamp: "2026-07-11T12:00:08Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "self-info-new", path: "network/node/nodeInfo", parentContext: materializerNetworkID + "/self-node", + data: NodeInfoData{Hostname: "self-host", OS: "darwin", Capabilities: []string{"ssh"}}, + timestamp: "2026-07-11T12:00:09Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "peer-info", path: "network/member/node/nodeInfo", parentContext: materializerNetworkID + "/member-record/peer-node", + data: NodeInfoData{Hostname: "peer-host", OS: "linux"}, timestamp: "2026-07-11T12:00:10Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "self-endpoint", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", + data: EndpointData{LocalEndpoints: []string{"192.0.2.1:1111"}, DiscoKey: "self-disco", UpdatedAt: "2026-07-11T12:00:11Z"}, + timestamp: "2026-07-11T12:00:11Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "peer-endpoint", path: "network/member/node/endpoint", parentContext: materializerNetworkID + "/member-record/peer-node", + data: EndpointData{LocalEndpoints: []string{"192.0.2.2:2222"}, PreferredDERP: 7, DiscoKey: "peer-disco", UpdatedAt: "2026-07-11T12:00:12Z"}, + timestamp: "2026-07-11T12:00:12Z", + }), + } + + set, err := newRawMeshRecordSet(records, "") + if err != nil { + t.Fatalf("newRawMeshRecordSet: %v", err) + } + client := newMaterializerTestClient() + response, err := client.materializeRawMeshRecordSet(context.Background(), set) + if err != nil { + t.Fatalf("materializeRawMeshRecordSet: %v", err) + } + + if response.Node == nil || response.Node.DID != materializerSelfDID || response.Node.Name != "self-host" { + t.Fatalf("self node = %#v", response.Node) + } + if got := response.Node.Endpoints; len(got) != 1 || got[0] != "192.0.2.1:1111" { + t.Fatalf("self endpoints = %#v", got) + } + if len(response.Peers) != 1 || response.Peers[0].DID != materializerPeerDID || response.Peers[0].Name != "peer-host" { + t.Fatalf("peers = %#v", response.Peers) + } + if response.Peers[0].MemberRecordID != "member-record" || response.Peers[0].PreferredDERP != 7 { + t.Fatalf("member peer projection = %#v", response.Peers[0]) + } + if len(response.DERPMap.Regions) != 1 || response.DERPMap.Regions[1].Nodes[0].HostName != "relay.example.com" { + t.Fatalf("DERP map = %#v", response.DERPMap) + } + if response.DNSConfig.MagicDNSSuffix != "prod.mesh" || len(response.PacketFilter) != 1 { + t.Fatalf("DNS/filter = %#v / %#v", response.DNSConfig, response.PacketFilter) + } + if client.network == nil || client.network.Name != "production" || client.acl == nil || client.acl.Version != 2 { + t.Fatalf("network/ACL = %#v / %#v", client.network, client.acl) + } + if member := client.members["did:jwk:member"]; member == nil || member.RecordID != "member-record" { + t.Fatalf("member projection = %#v", member) + } + if _, ok := client.nodes["did:jwk:foreign"]; ok { + t.Fatal("foreign-protocol node entered materialized state") + } + if _, ok := client.nodes["did:jwk:orphan"]; ok { + t.Fatal("orphan member node entered materialized state") + } +} + +func TestMaterializeRawMeshRecordSetAppliesEndpointUpdateAndDelete(t *testing.T) { + base := materializerBaseRecords(t) + base = append(base, materializerRecord(t, materializerRecordSpec{ + id: "self-endpoint", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", + data: EndpointData{LocalEndpoints: []string{"192.0.2.1:1111"}, UpdatedAt: "2026-07-11T12:00:02Z"}, + timestamp: "2026-07-11T12:00:02Z", + })) + set, err := newRawMeshRecordSet(base, "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + firstNode := client.nodes[materializerSelfDID] + if got := firstNode.Endpoints[0].LocalEndpoints[0]; got != "192.0.2.1:1111" { + t.Fatalf("initial endpoint = %q", got) + } + + updated := materializerRecord(t, materializerRecordSpec{ + id: "self-endpoint", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", + data: EndpointData{LocalEndpoints: []string{"192.0.2.1:2222"}, UpdatedAt: "2026-07-11T12:00:03Z"}, + timestamp: "2026-07-11T12:00:03Z", + }) + if changed, err := set.addEntries([]json.RawMessage{updated}, ""); err != nil || !changed { + t.Fatalf("update raw set changed=%v err=%v", changed, err) + } + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + secondNode := client.nodes[materializerSelfDID] + if secondNode == firstNode { + t.Fatal("endpoint update reused the prior node pointer") + } + if got := secondNode.Endpoints[0].LocalEndpoints[0]; got != "192.0.2.1:2222" { + t.Fatalf("updated endpoint = %q", got) + } + if got := firstNode.Endpoints[0].LocalEndpoints[0]; got != "192.0.2.1:1111" { + t.Fatalf("new projection mutated prior endpoint = %q", got) + } + + deleted := rawRecordTestSubscription(rawRecordTestDelete(t, "self-endpoint", "2026-07-11T12:00:04Z"), "") + if changed, err := set.applySubscriptionMessage(deleted, ""); err != nil || !changed { + t.Fatalf("delete raw record changed=%v err=%v", changed, err) + } + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + if got := len(client.nodes[materializerSelfDID].Endpoints); got != 0 { + t.Fatalf("endpoints after delete = %d, want 0", got) + } +} + +func TestMaterializeRawMeshRecordSetRollbackAndDeepIsolation(t *testing.T) { + client := newMaterializerTestClient() + oldNetwork := &NetworkConfig{Name: "old", MeshCIDR: "10.199.0.0/16", DNSServers: []string{"old-dns"}} + oldMember := &MemberRecord{DID: "did:jwk:old-member", Label: "old-member", RecordID: "old-member-record"} + oldNode := &NodeRecord{ + DID: materializerSelfDID, MeshIP: "10.199.1.1", RecordID: "old-node", + Endpoints: []EndpointData{{LocalEndpoints: []string{"old-endpoint"}}}, + } + oldRelay := &RelayData{URL: "old-relay", Region: "old"} + oldACL := &ACLPolicyData{Version: 99, Groups: map[string][]string{"old": {materializerSelfDID}}} + client.network = oldNetwork + client.members = map[string]*MemberRecord{oldMember.DID: oldMember} + client.nodes = map[string]*NodeRecord{materializerSelfDID: oldNode} + client.relays = []*RelayData{oldRelay} + client.acl = oldACL + client.endpointFailures = map[string]string{"old-endpoint": "key-unavailable"} + + badSet, err := newRawMeshRecordSet([]json.RawMessage{ + materializerRecord(t, materializerRecordSpec{ + id: materializerNetworkID, path: "network", data: NetworkConfig{Name: "bad", MeshCIDR: "10.200.0.0/16"}, + timestamp: "2026-07-11T12:00:00Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "self-opaque", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, data: NodeRecord{MeshIP: "10.200.1.9"}, encrypted: true, + timestamp: "2026-07-11T12:00:01Z", + }), + }, "") + if err != nil { + t.Fatal(err) + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), badSet, materializerUnavailableDecryptors("network/node"), + ); err == nil { + t.Fatal("cold opaque self materialization succeeded") + } + assertMaterializerOldState(t, client, oldNetwork, oldMember, oldNode, oldRelay, oldACL) + + rateSet, err := newRawMeshRecordSet(append(materializerBaseRecords(t), + materializerRecord(t, materializerRecordSpec{ + id: "encrypted-peer", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.2"}, encrypted: true, + timestamp: "2026-07-11T12:00:02Z", + }), + ), "") + if err != nil { + t.Fatal(err) + } + rateLimitDecryptors := func(_ context.Context, path string) EntryDecryptor { + if path != "network/node" { + return nil + } + return func([]byte, *dwncrypto.Encryption) ([]byte, error) { + return nil, fmt.Errorf("delivery lookup: %w", dwn.ErrRateLimited) + } + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), rateSet, rateLimitDecryptors); !errors.Is(err, dwn.ErrRateLimited) { + t.Fatalf("rate-limited materialization error = %v", err) + } + assertMaterializerOldState(t, client, oldNetwork, oldMember, oldNode, oldRelay, oldACL) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := client.materializeRawMeshRecordSet(ctx, rateSet); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled materialization error = %v", err) + } + assertMaterializerOldState(t, client, oldNetwork, oldMember, oldNode, oldRelay, oldACL) + + goodSet, err := newRawMeshRecordSet(materializerBaseRecords(t), "") + if err != nil { + t.Fatal(err) + } + if _, err := client.materializeRawMeshRecordSet(context.Background(), goodSet); err != nil { + t.Fatal(err) + } + if client.network == oldNetwork || client.nodes[materializerSelfDID] == oldNode { + t.Fatal("successful projection reused prior state pointers") + } + client.network.DNSServers[0] = "new-dns" + client.nodes[materializerSelfDID].Endpoints = append(client.nodes[materializerSelfDID].Endpoints, + EndpointData{LocalEndpoints: []string{"new-endpoint"}}) + if oldNetwork.DNSServers[0] != "old-dns" || oldNode.Endpoints[0].LocalEndpoints[0] != "old-endpoint" { + t.Fatalf("successful projection aliased prior nested state: network=%#v node=%#v", oldNetwork, oldNode) + } +} + +func assertMaterializerOldState( + t *testing.T, + client *DWNClient, + oldNetwork *NetworkConfig, + oldMember *MemberRecord, + oldNode *NodeRecord, + oldRelay *RelayData, + oldACL *ACLPolicyData, +) { + t.Helper() + if client.network != oldNetwork || client.members[oldMember.DID] != oldMember || + client.nodes[materializerSelfDID] != oldNode || len(client.relays) != 1 || + client.relays[0] != oldRelay || client.acl != oldACL { + t.Fatalf("failed materialization replaced live state: network=%p member=%p node=%p relay=%p acl=%p", + client.network, client.members[oldMember.DID], client.nodes[materializerSelfDID], client.relays[0], client.acl) + } + client.members["same-map-member"] = oldMember + client.nodes["same-map-node"] = oldNode + client.endpointFailures["same-map-failure"] = "parse" + if client.members["same-map-member"] != oldMember || client.nodes["same-map-node"] != oldNode || + client.endpointFailures["same-map-failure"] != "parse" { + t.Fatal("failed materialization replaced a live map") + } + delete(client.members, "same-map-member") + delete(client.nodes, "same-map-node") + delete(client.endpointFailures, "same-map-failure") +} + +func materializerBaseRecords(t *testing.T) []json.RawMessage { + t.Helper() + return []json.RawMessage{ + materializerRecord(t, materializerRecordSpec{ + id: materializerNetworkID, path: "network", + data: NetworkConfig{Name: "base", MeshCIDR: "10.200.0.0/16", DNSServers: []string{"10.200.0.53"}}, + timestamp: "2026-07-11T12:00:00Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "self-node", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, + data: NodeRecord{MeshIP: "10.200.1.1", Label: "self", AddedAt: "2026-07-11T12:00:01Z"}, + timestamp: "2026-07-11T12:00:01Z", + }), + } +} + +func TestMaterializeRawMeshRecordSetUsesNewestEndpointSnapshot(t *testing.T) { + records := append(materializerBaseRecords(t), + materializerRecord(t, materializerRecordSpec{ + id: "self-endpoint-old", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", + data: EndpointData{LocalEndpoints: []string{"192.0.2.1:1111"}}, timestamp: "2026-07-11T12:00:02Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "self-endpoint-new", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", + data: EndpointData{LocalEndpoints: []string{"192.0.2.1:2222"}}, timestamp: "2026-07-11T12:00:03Z", + }), + ) + set, err := newRawMeshRecordSet(records, "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + endpoints := client.nodes[materializerSelfDID].Endpoints + if len(endpoints) != 1 || len(endpoints[0].LocalEndpoints) != 1 || endpoints[0].LocalEndpoints[0] != "192.0.2.1:2222" { + t.Fatalf("materialized endpoints = %#v, want only newest snapshot", endpoints) + } +} + +func TestMaterializeRawMeshRecordSetACLDeleteClearsLastPolicy(t *testing.T) { + records := append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "acl-record", path: "network/aclPolicy", parentContext: materializerNetworkID, + data: ACLPolicyData{Version: 1, DefaultAction: "deny"}, timestamp: "2026-07-11T12:00:02Z", + })) + set, err := newRawMeshRecordSet(records, "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + if client.acl == nil || client.acl.Version != 1 { + t.Fatalf("initial ACL = %#v", client.acl) + } + deleted := rawRecordTestSubscription(rawRecordTestDelete(t, "acl-record", "2026-07-11T12:00:03Z"), "") + if changed, err := set.applySubscriptionMessage(deleted, ""); err != nil || !changed { + t.Fatalf("delete ACL changed=%v err=%v", changed, err) + } + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + if client.acl != nil { + t.Fatalf("deleted ACL remained active: %#v", client.acl) + } +} + +func TestMaterializeRawMeshRecordSetRelayLastGoodPolicy(t *testing.T) { + invalidRelay := materializerRecord(t, materializerRecordSpec{ + id: "relay-record", path: "network/relay", parentContext: materializerNetworkID, + data: json.RawMessage("not-json"), dateCreated: "2026-07-11T12:00:02Z", timestamp: "2026-07-11T12:01:00Z", + }) + t.Run("cold unreadable fails", func(t *testing.T) { + set, err := newRawMeshRecordSet(append(materializerBaseRecords(t), invalidRelay), "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err == nil { + t.Fatal("cold unreadable relay unexpectedly projected public defaults") + } + }) + + t.Run("warm unreadable preserves and delete removes", func(t *testing.T) { + validRelay := materializerRecord(t, materializerRecordSpec{ + id: "relay-record", path: "network/relay", parentContext: materializerNetworkID, + data: RelayData{URL: "last-good.example.com", Region: "private"}, + dateCreated: "2026-07-11T12:00:02Z", timestamp: "2026-07-11T12:00:02Z", + }) + set, err := newRawMeshRecordSet(append(materializerBaseRecords(t), validRelay), "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(invalidRelay, ""), ""); err != nil || !changed { + t.Fatalf("relay replacement changed=%v err=%v", changed, err) + } + response, err := client.materializeRawMeshRecordSet(context.Background(), set) + if err != nil { + t.Fatalf("warm unreadable relay: %v", err) + } + if len(client.relays) != 1 || client.relays[0].URL != "last-good.example.com" || len(response.DERPMap.Regions) != 1 { + t.Fatalf("warm relay projection = %#v DERP=%#v", client.relays, response.DERPMap) + } + + deleted := rawRecordTestSubscription(rawRecordTestDelete(t, "relay-record", "2026-07-11T12:02:00Z"), "") + if changed, err := set.applySubscriptionMessage(deleted, ""); err != nil || !changed { + t.Fatalf("delete relay changed=%v err=%v", changed, err) + } + response, err = client.materializeRawMeshRecordSet(context.Background(), set) + if err != nil { + t.Fatal(err) + } + if len(client.relays) != 0 { + t.Fatalf("deleted relay remained active: %#v", client.relays) + } + for _, region := range response.DERPMap.Regions { + for _, node := range region.Nodes { + if node.HostName == "last-good.example.com" { + t.Fatalf("deleted custom relay remained in DERP map: %#v", response.DERPMap) + } + } + } + }) +} + +func TestMaterializeRawMeshRecordSetRelayOrderUsesDateCreated(t *testing.T) { + records := append(materializerBaseRecords(t), + materializerRecord(t, materializerRecordSpec{ + id: "relay-created-first", path: "network/relay", parentContext: materializerNetworkID, + data: RelayData{URL: "first.example.com", Region: "first"}, + dateCreated: "2026-07-11T12:00:00Z", timestamp: "2026-07-11T12:10:00Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "relay-created-second", path: "network/relay", parentContext: materializerNetworkID, + data: RelayData{URL: "second.example.com", Region: "second"}, + dateCreated: "2026-07-11T12:01:00Z", timestamp: "2026-07-11T12:01:00Z", + }), + ) + set, err := newRawMeshRecordSet(records, "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + response, err := client.materializeRawMeshRecordSet(context.Background(), set) + if err != nil { + t.Fatal(err) + } + if len(client.relays) != 2 || client.relays[0].URL != "first.example.com" || client.relays[1].URL != "second.example.com" { + t.Fatalf("relay order = %#v", client.relays) + } + if response.DERPMap.Regions[1].Nodes[0].HostName != "first.example.com" || response.DERPMap.Regions[2].Nodes[0].HostName != "second.example.com" { + t.Fatalf("DERP region order = %#v", response.DERPMap.Regions) + } +} + +func TestMaterializeRawMeshRecordSetUnreadableACLRetainsDeepClone(t *testing.T) { + set, err := newRawMeshRecordSet(append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "acl-replacement", path: "network/aclPolicy", parentContext: materializerNetworkID, encrypted: true, + data: ACLPolicyData{Version: 2, DefaultAction: "accept"}, timestamp: "2026-07-11T12:00:02Z", + })), "") + if err != nil { + t.Fatal(err) + } + old := &ACLPolicyData{ + Version: 1, DefaultAction: "deny", + Groups: map[string][]string{"operators": {"did:jwk:operator"}}, + Rules: []ACLRule{{ + Action: "accept", Src: []string{"did:jwk:operator"}, Dst: []string{materializerSelfDID}, + SrcPorts: []string{"1024-65535"}, DstPorts: []string{"22"}, + }}, + } + client := newMaterializerTestClient() + client.acl = old + decryptors := func(_ context.Context, path string) EntryDecryptor { + if path != "network/aclPolicy" { + return nil + } + return func([]byte, *dwncrypto.Encryption) ([]byte, error) { + return nil, fmt.Errorf("%w: role audience key unavailable", errAudienceKeyDeliveryAbsent) + } + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatal(err) + } + if client.acl == nil || client.acl == old || client.acl.Version != old.Version { + t.Fatalf("retained ACL = %#v, old=%p retained=%p", client.acl, old, client.acl) + } + client.acl.Groups["operators"][0] = "did:jwk:mutated" + client.acl.Rules[0].Src[0] = "did:jwk:mutated" + client.acl.Rules[0].Dst[0] = "did:jwk:mutated" + client.acl.Rules[0].SrcPorts[0] = "1" + client.acl.Rules[0].DstPorts[0] = "1" + if old.Groups["operators"][0] != "did:jwk:operator" || old.Rules[0].Src[0] != "did:jwk:operator" || + old.Rules[0].Dst[0] != materializerSelfDID || old.Rules[0].SrcPorts[0] != "1024-65535" || old.Rules[0].DstPorts[0] != "22" { + t.Fatalf("retained ACL aliases last-good policy: %#v", old) + } +} + +func TestMaterializeRawMeshRecordSetColdUnreadableACLRejectsAtomically(t *testing.T) { + tests := []struct { + name string + data any + encrypted bool + }{ + {name: "missing audience key", data: ACLPolicyData{Version: 2}, encrypted: true}, + {name: "malformed payload", data: json.RawMessage(`not-json`)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + set, err := newRawMeshRecordSet(append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "acl-cold", path: "network/aclPolicy", parentContext: materializerNetworkID, + data: test.data, encrypted: test.encrypted, timestamp: "2026-07-11T12:00:02Z", + })), "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + decryptors := func(_ context.Context, path string) EntryDecryptor { + if !test.encrypted || path != "network/aclPolicy" { + return nil + } + return func([]byte, *dwncrypto.Encryption) ([]byte, error) { + return nil, fmt.Errorf("%w: role audience key unavailable", errAudienceKeyDeliveryAbsent) + } + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err == nil { + t.Fatal("cold unreadable ACL installed a map") + } + if client.network != nil || client.acl != nil || len(client.rawParsedOutcomes) != 0 { + t.Fatalf("failed ACL projection mutated state: network=%#v acl=%#v cache=%d", + client.network, client.acl, len(client.rawParsedOutcomes)) + } + }) + } +} + +func TestMaterializeRawMeshRecordSetKeyUnavailableSnapshotKeepsLastGoodUntilInvalidated(t *testing.T) { + set, err := newRawMeshRecordSet(append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "endpoint-old", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", + data: EndpointData{LocalEndpoints: []string{"192.0.2.1:1111"}}, timestamp: "2026-07-11T12:00:02Z", + })), "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + + replacement := materializerRecord(t, materializerRecordSpec{ + id: "endpoint-new", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", + data: EndpointData{LocalEndpoints: []string{"192.0.2.1:2222"}}, encrypted: true, squash: true, + timestamp: "2026-07-11T12:00:03Z", + }) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(replacement, ""), ""); err != nil || !changed { + t.Fatalf("replacement changed=%v err=%v", changed, err) + } + available := false + decryptCalls := 0 + decryptors := func(_ context.Context, path string) EntryDecryptor { + if path != "network/node/endpoint" { + return nil + } + return func(ciphertext []byte, _ *dwncrypto.Encryption) ([]byte, error) { + decryptCalls++ + if !available { + return nil, fmt.Errorf("%w: delivery not present", errAudienceKeyDeliveryAbsent) + } + return ciphertext, nil + } + } + for range 2 { + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatal(err) + } + endpoints := client.nodes[materializerSelfDID].Endpoints + if len(endpoints) != 1 || endpoints[0].LocalEndpoints[0] != "192.0.2.1:1111" { + t.Fatalf("key-unavailable snapshot replaced last-good: %#v", endpoints) + } + } + if decryptCalls != 1 { + t.Fatalf("cached opaque replacement decrypt calls = %d, want 1", decryptCalls) + } + + cachedCount := len(client.rawParsedOutcomes) + var opaqueKeys []rawParsedOutcomeKey + for key, outcome := range client.rawParsedOutcomes { + if outcome.opaque { + opaqueKeys = append(opaqueKeys, key) + } + } + client.invalidateRawParsedOutcomes() + if len(client.rawParsedOutcomes) != cachedCount { + t.Fatalf("generation invalidation cache size = %d, want %d", len(client.rawParsedOutcomes), cachedCount) + } + for _, key := range opaqueKeys { + if _, ok := client.rawParsedOutcomes[key]; !ok { + t.Fatalf("generation invalidation discarded opaque prior outcome %#v", key) + } + } + failing := set.clone() + if _, err := failing.addEntries([]json.RawMessage{materializerRecord(t, materializerRecordSpec{ + id: "acl-unreadable", path: "network/aclPolicy", parentContext: materializerNetworkID, + data: json.RawMessage(`not-json`), timestamp: "2026-07-11T12:00:04Z", + })}, ""); err != nil { + t.Fatal(err) + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), failing, decryptors); err == nil { + t.Fatal("failed projection with cold unreadable ACL succeeded") + } + for _, key := range opaqueKeys { + if _, ok := client.rawParsedOutcomes[key]; !ok { + t.Fatalf("failed projection discarded prior outcome %#v", key) + } + } + + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatal(err) + } + endpoints := client.nodes[materializerSelfDID].Endpoints + if decryptCalls != 2 || len(endpoints) != 1 || endpoints[0].LocalEndpoints[0] != "192.0.2.1:1111" { + t.Fatalf("opaque retry lost last-good: calls=%d endpoints=%#v", decryptCalls, endpoints) + } + + available = true + client.invalidateRawParsedOutcomes() + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatal(err) + } + endpoints = client.nodes[materializerSelfDID].Endpoints + if decryptCalls != 3 || len(endpoints) != 1 || endpoints[0].LocalEndpoints[0] != "192.0.2.1:2222" { + t.Fatalf("delivery invalidation did not install replacement: calls=%d endpoints=%#v", decryptCalls, endpoints) + } + client.invalidateRawParsedOutcomes() + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatal(err) + } + if decryptCalls != 3 { + t.Fatalf("healthy anti-entropy invalidation re-decrypted successful outcome: calls=%d", decryptCalls) + } +} + +func TestMaterializeRawMeshRecordSetMemberAndNodeKeepLastGoodForNewerCID(t *testing.T) { + records := append(materializerBaseRecords(t), + materializerRecord(t, materializerRecordSpec{ + id: "member-record", path: "network/member", parentContext: materializerNetworkID, + recipient: "did:jwk:member", data: MemberRecord{Label: "last-good-member"}, + timestamp: "2026-07-11T12:00:02Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "peer-node", path: "network/member/node", parentContext: materializerNetworkID + "/member-record", + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.2", Label: "last-good-node"}, + timestamp: "2026-07-11T12:00:03Z", + }), + ) + set, err := newRawMeshRecordSet(records, "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + + memberReplacement := materializerRecord(t, materializerRecordSpec{ + id: "member-record", path: "network/member", parentContext: materializerNetworkID, + recipient: "did:jwk:member", data: MemberRecord{Label: "unreadable-member"}, encrypted: true, + dateCreated: "2026-07-11T12:00:02Z", timestamp: "2026-07-11T12:00:04Z", + }) + nodeReplacement := materializerRecord(t, materializerRecordSpec{ + id: "peer-node", path: "network/member/node", parentContext: materializerNetworkID + "/member-record", + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.3", Label: "unreadable-node"}, encrypted: true, + dateCreated: "2026-07-11T12:00:03Z", timestamp: "2026-07-11T12:00:05Z", + }) + for _, replacement := range []json.RawMessage{memberReplacement, nodeReplacement} { + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(replacement, ""), ""); err != nil || !changed { + t.Fatalf("apply replacement changed=%v err=%v", changed, err) + } + } + decryptors := func(_ context.Context, path string) EntryDecryptor { + if path != "network/member" && path != "network/member/node" { + return nil + } + return func([]byte, *dwncrypto.Encryption) ([]byte, error) { + return nil, fmt.Errorf("%w: delivery not present", errAudienceKeyDeliveryAbsent) + } + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatal(err) + } + if member := client.members["did:jwk:member"]; member == nil || member.Label != "last-good-member" { + t.Fatalf("newer unreadable member replaced last-good: %#v", member) + } + if node := client.nodes[materializerPeerDID]; node == nil || node.MeshIP != "10.200.1.2" || node.Label != "last-good-node" { + t.Fatalf("newer unreadable node replaced last-good: %#v", node) + } + + for _, recordID := range []string{"peer-node", "member-record"} { + deleted := rawRecordTestSubscription(rawRecordTestDelete(t, recordID, "2026-07-11T12:00:06Z"), "") + if changed, err := set.applySubscriptionMessage(deleted, ""); err != nil || !changed { + t.Fatalf("delete %s changed=%v err=%v", recordID, changed, err) + } + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatal(err) + } + if _, ok := client.members["did:jwk:member"]; ok { + t.Fatalf("deleted member remained active: %#v", client.members) + } + if _, ok := client.nodes[materializerPeerDID]; ok { + t.Fatalf("deleted peer node remained active: %#v", client.nodes) + } +} + +func TestMaterializeRawMeshRecordSetNewIDNonSquashUsesStagedRecipientContribution(t *testing.T) { + const ( + memberDID = "did:jwk:replacement-member" + ownerDID = "did:jwk:owner-peer" + ) + records := append(materializerBaseRecords(t), + materializerRecord(t, materializerRecordSpec{ + id: "member-old", path: "network/member", parentContext: materializerNetworkID, + recipient: memberDID, data: MemberRecord{Label: "last-good-member"}, + timestamp: "2026-07-11T12:00:02Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "member-new", path: "network/member", parentContext: materializerNetworkID, + recipient: memberDID, data: MemberRecord{Label: "unreadable-member"}, encrypted: true, + timestamp: "2026-07-11T12:00:03Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "member-child", path: "network/member/node", parentContext: materializerNetworkID + "/member-new", + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.2", Label: "member-child"}, + timestamp: "2026-07-11T12:00:04Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "owner-old", path: "network/node", parentContext: materializerNetworkID, + recipient: ownerDID, data: NodeRecord{MeshIP: "10.200.1.3", Label: "last-good-owner"}, + timestamp: "2026-07-11T12:00:02.100Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "owner-new", path: "network/node", parentContext: materializerNetworkID, + recipient: ownerDID, data: NodeRecord{MeshIP: "10.200.1.30", Label: "unreadable-owner"}, encrypted: true, + timestamp: "2026-07-11T12:00:03.100Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "owner-new-info", path: "network/node/nodeInfo", parentContext: materializerNetworkID + "/owner-new", + data: NodeInfoData{Hostname: "owner-new-host"}, timestamp: "2026-07-11T12:00:04.100Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "owner-new-endpoint", path: "network/node/endpoint", parentContext: materializerNetworkID + "/owner-new", + data: EndpointData{LocalEndpoints: []string{"192.0.2.30:3030"}}, timestamp: "2026-07-11T12:00:04.200Z", + }), + ) + set, err := newRawMeshRecordSet(records, "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/member", "network/node"), + ); err != nil { + t.Fatal(err) + } + + member := client.members[memberDID] + if member == nil || member.Label != "last-good-member" || member.RecordID != "member-new" { + t.Fatalf("staged member replacement = %#v", member) + } + memberChild := client.nodes[materializerPeerDID] + if memberChild == nil || memberChild.MemberRecordID != "member-new" || memberChild.Label != "member-child" { + t.Fatalf("child under replacement member = %#v", memberChild) + } + owner := client.nodes[ownerDID] + if owner == nil || owner.MeshIP != "10.200.1.3" || owner.Label != "last-good-owner" || owner.RecordID != "owner-new" { + t.Fatalf("staged owner-node replacement = %#v", owner) + } + if owner.Info == nil || owner.Info.Hostname != "owner-new-host" || len(owner.Endpoints) != 1 || + owner.Endpoints[0].LocalEndpoints[0] != "192.0.2.30:3030" { + t.Fatalf("children did not attach to replacement owner node: %#v", owner) + } +} + +func TestMaterializeRawMeshRecordSetNewIDSquashUsesPriorRecipientContribution(t *testing.T) { + t.Run("member", func(t *testing.T) { + const memberDID = "did:jwk:squashed-member" + set, err := newRawMeshRecordSet(append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "member-old", path: "network/member", parentContext: materializerNetworkID, + recipient: memberDID, data: MemberRecord{Label: "last-good-member"}, timestamp: "2026-07-11T12:00:02Z", + })), "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + replacement := materializerRecord(t, materializerRecordSpec{ + id: "member-new", path: "network/member", parentContext: materializerNetworkID, + recipient: memberDID, data: MemberRecord{Label: "unreadable"}, encrypted: true, squash: true, + timestamp: "2026-07-11T12:00:03Z", + }) + child := materializerRecord(t, materializerRecordSpec{ + id: "child-new", path: "network/member/node", parentContext: materializerNetworkID + "/member-new", + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.4", Label: "new-parent-child"}, + timestamp: "2026-07-11T12:00:04Z", + }) + for _, raw := range []json.RawMessage{replacement, child} { + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(raw, ""), ""); err != nil || !changed { + t.Fatalf("apply replacement changed=%v err=%v", changed, err) + } + } + if _, ok := set.get("member-old"); ok { + t.Fatal("member squash retained old raw record") + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/member"), + ); err != nil { + t.Fatal(err) + } + member := client.members[memberDID] + if member == nil || member.Label != "last-good-member" || member.RecordID != "member-new" { + t.Fatalf("squashed member replacement = %#v", member) + } + if node := client.nodes[materializerPeerDID]; node == nil || node.MemberRecordID != "member-new" || node.Label != "new-parent-child" { + t.Fatalf("replacement member child = %#v", node) + } + deleted := rawRecordTestSubscription(rawRecordTestDelete(t, "member-new", "2026-07-11T12:00:05Z"), "") + if changed, err := set.applySubscriptionMessage(deleted, ""); err != nil || !changed { + t.Fatalf("delete replacement member changed=%v err=%v", changed, err) + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/member"), + ); err != nil { + t.Fatal(err) + } + if _, ok := client.members[memberDID]; ok { + t.Fatalf("deleted replacement member remained: %#v", client.members) + } + if _, ok := client.nodes[materializerPeerDID]; ok { + t.Fatalf("orphaned replacement child remained: %#v", client.nodes) + } + }) + + t.Run("member node", func(t *testing.T) { + const memberDID = "did:jwk:node-parent" + set, err := newRawMeshRecordSet(append(materializerBaseRecords(t), + materializerRecord(t, materializerRecordSpec{ + id: "member-parent", path: "network/member", parentContext: materializerNetworkID, + recipient: memberDID, data: MemberRecord{Label: "parent"}, timestamp: "2026-07-11T12:00:02Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "node-old", path: "network/member/node", parentContext: materializerNetworkID + "/member-parent", + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.5", Label: "last-good-node"}, + timestamp: "2026-07-11T12:00:03Z", + }), + ), "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + replacement := materializerRecord(t, materializerRecordSpec{ + id: "node-new", path: "network/member/node", parentContext: materializerNetworkID + "/member-parent", + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.50", Label: "unreadable"}, + encrypted: true, squash: true, timestamp: "2026-07-11T12:00:04Z", + }) + info := materializerRecord(t, materializerRecordSpec{ + id: "node-new-info", path: "network/member/node/nodeInfo", + parentContext: materializerNetworkID + "/member-parent/node-new", + data: NodeInfoData{Hostname: "node-new-host"}, timestamp: "2026-07-11T12:00:05Z", + }) + endpoint := materializerRecord(t, materializerRecordSpec{ + id: "node-new-endpoint", path: "network/member/node/endpoint", + parentContext: materializerNetworkID + "/member-parent/node-new", + data: EndpointData{LocalEndpoints: []string{"192.0.2.50:5050"}}, timestamp: "2026-07-11T12:00:05.100Z", + }) + for _, raw := range []json.RawMessage{replacement, info, endpoint} { + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(raw, ""), ""); err != nil || !changed { + t.Fatalf("apply node replacement changed=%v err=%v", changed, err) + } + } + if _, ok := set.get("node-old"); ok { + t.Fatal("node squash retained old raw record") + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/member/node"), + ); err != nil { + t.Fatal(err) + } + node := client.nodes[materializerPeerDID] + if node == nil || node.MeshIP != "10.200.1.5" || node.Label != "last-good-node" || + node.RecordID != "node-new" || node.MemberRecordID != "member-parent" { + t.Fatalf("squashed member-node replacement = %#v", node) + } + if node.Info == nil || node.Info.Hostname != "node-new-host" || len(node.Endpoints) != 1 || + node.Endpoints[0].LocalEndpoints[0] != "192.0.2.50:5050" { + t.Fatalf("children did not attach to replacement member node: %#v", node) + } + deleted := rawRecordTestSubscription(rawRecordTestDelete(t, "node-new", "2026-07-11T12:00:06Z"), "") + if changed, err := set.applySubscriptionMessage(deleted, ""); err != nil || !changed { + t.Fatalf("delete replacement node changed=%v err=%v", changed, err) + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/member/node"), + ); err != nil { + t.Fatal(err) + } + if _, ok := client.nodes[materializerPeerDID]; ok { + t.Fatalf("deleted replacement node remained: %#v", client.nodes) + } + }) +} + +func TestMaterializeRawMeshRecordSetNewIDNeverInheritsAcrossRecipients(t *testing.T) { + t.Run("member squash", func(t *testing.T) { + const oldDID, newDID = "did:jwk:old-member", "did:jwk:new-member" + set, err := newRawMeshRecordSet(append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "member-old", path: "network/member", parentContext: materializerNetworkID, + recipient: oldDID, data: MemberRecord{Label: "must-not-leak"}, timestamp: "2026-07-11T12:00:02Z", + })), "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + replacement := materializerRecord(t, materializerRecordSpec{ + id: "member-new", path: "network/member", parentContext: materializerNetworkID, + recipient: newDID, data: MemberRecord{Label: "unreadable"}, encrypted: true, squash: true, + timestamp: "2026-07-11T12:00:03Z", + }) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(replacement, ""), ""); err != nil || !changed { + t.Fatalf("apply member replacement changed=%v err=%v", changed, err) + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/member"), + ); err != nil { + t.Fatal(err) + } + if _, ok := client.members[oldDID]; ok { + t.Fatalf("squashed old recipient remained: %#v", client.members) + } + member := client.members[newDID] + if member == nil || member.RecordID != "member-new" || member.Label != "" { + t.Fatalf("new recipient inherited old member data: %#v", member) + } + }) + + t.Run("owner node non-squash", func(t *testing.T) { + const oldDID, newDID = "did:jwk:old-owner-node", "did:jwk:new-owner-node" + set, err := newRawMeshRecordSet(append(materializerBaseRecords(t), + materializerRecord(t, materializerRecordSpec{ + id: "owner-old", path: "network/node", parentContext: materializerNetworkID, + recipient: oldDID, data: NodeRecord{MeshIP: "10.200.1.6", Label: "must-not-leak"}, + timestamp: "2026-07-11T12:00:02Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "owner-new", path: "network/node", parentContext: materializerNetworkID, + recipient: newDID, data: NodeRecord{MeshIP: "10.200.1.60", Label: "unreadable"}, encrypted: true, + timestamp: "2026-07-11T12:00:03Z", + }), + ), "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/node"), + ); err != nil { + t.Fatal(err) + } + if old := client.nodes[oldDID]; old == nil || old.MeshIP != "10.200.1.6" || old.Label != "must-not-leak" { + t.Fatalf("old recipient contribution = %#v", old) + } + fresh := client.nodes[newDID] + if fresh == nil || fresh.RecordID != "owner-new" || fresh.MeshIP != "" || fresh.Label != "" { + t.Fatalf("new recipient inherited old owner-node data: %#v", fresh) + } + }) + + t.Run("member node squash", func(t *testing.T) { + const oldDID, newDID = "did:jwk:old-member-node", "did:jwk:new-member-node" + set, err := newRawMeshRecordSet(append(materializerBaseRecords(t), + materializerRecord(t, materializerRecordSpec{ + id: "member-parent", path: "network/member", parentContext: materializerNetworkID, + recipient: "did:jwk:parent", data: MemberRecord{Label: "parent"}, timestamp: "2026-07-11T12:00:02Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "node-old", path: "network/member/node", parentContext: materializerNetworkID + "/member-parent", + recipient: oldDID, data: NodeRecord{MeshIP: "10.200.1.7", Label: "must-not-leak"}, + timestamp: "2026-07-11T12:00:03Z", + }), + ), "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + replacement := materializerRecord(t, materializerRecordSpec{ + id: "node-new", path: "network/member/node", parentContext: materializerNetworkID + "/member-parent", + recipient: newDID, data: NodeRecord{MeshIP: "10.200.1.70", Label: "unreadable"}, + encrypted: true, squash: true, timestamp: "2026-07-11T12:00:04Z", + }) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(replacement, ""), ""); err != nil || !changed { + t.Fatalf("apply member-node replacement changed=%v err=%v", changed, err) + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/member/node"), + ); err != nil { + t.Fatal(err) + } + if _, ok := client.nodes[oldDID]; ok { + t.Fatalf("squashed old node recipient remained: %#v", client.nodes) + } + fresh := client.nodes[newDID] + if fresh == nil || fresh.RecordID != "node-new" || fresh.MemberRecordID != "member-parent" || + fresh.MeshIP != "" || fresh.Label != "" { + t.Fatalf("new recipient inherited old member-node data: %#v", fresh) + } + }) +} + +func TestMaterializeRawMeshRecordSetNodeInfoSquashKeepsLastGoodUntilDelete(t *testing.T) { + records := append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "node-info-old", path: "network/node/nodeInfo", parentContext: materializerNetworkID + "/self-node", + data: NodeInfoData{Hostname: "last-good-host"}, timestamp: "2026-07-11T12:00:02Z", + })) + set, err := newRawMeshRecordSet(records, "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + + replacement := materializerRecord(t, materializerRecordSpec{ + id: "node-info-new", path: "network/node/nodeInfo", parentContext: materializerNetworkID + "/self-node", + data: NodeInfoData{Hostname: "unreadable-host"}, encrypted: true, squash: true, + timestamp: "2026-07-11T12:00:03Z", + }) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(replacement, ""), ""); err != nil || !changed { + t.Fatalf("nodeInfo squash changed=%v err=%v", changed, err) + } + if _, ok := set.get("node-info-old"); ok { + t.Fatal("squashed nodeInfo sibling remained in the raw set") + } + decryptors := func(_ context.Context, path string) EntryDecryptor { + if path != "network/node/nodeInfo" { + return nil + } + return func([]byte, *dwncrypto.Encryption) ([]byte, error) { + return nil, fmt.Errorf("%w: delivery not present", errAudienceKeyDeliveryAbsent) + } + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatal(err) + } + if info := client.nodes[materializerSelfDID].Info; info == nil || info.Hostname != "last-good-host" { + t.Fatalf("new-ID same-parent nodeInfo squash replaced last-good: %#v", info) + } + + deleted := rawRecordTestSubscription(rawRecordTestDelete(t, "node-info-new", "2026-07-11T12:00:04Z"), "") + if changed, err := set.applySubscriptionMessage(deleted, ""); err != nil || !changed { + t.Fatalf("delete nodeInfo changed=%v err=%v", changed, err) + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatal(err) + } + if info := client.nodes[materializerSelfDID].Info; info != nil { + t.Fatalf("deleted nodeInfo remained active: %#v", info) + } +} + +func TestMaterializeRawMeshRecordSetColdOpaqueSelfFailsClosed(t *testing.T) { + network := materializerRecord(t, materializerRecordSpec{ + id: materializerNetworkID, path: "network", + data: NetworkConfig{Name: "security", MeshCIDR: "10.200.0.0/16"}, timestamp: "2026-07-11T12:00:00Z", + }) + tests := []struct { + name string + records []json.RawMessage + }{ + { + name: "no prior node", + records: []json.RawMessage{network, materializerRecord(t, materializerRecordSpec{ + id: "self-opaque", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, data: NodeRecord{MeshIP: "10.200.1.1"}, encrypted: true, + timestamp: "2026-07-11T12:00:02Z", + })}, + }, + { + name: "different recipient is not prior", + records: []json.RawMessage{ + network, + materializerRecord(t, materializerRecordSpec{ + id: "other-readable", path: "network/node", parentContext: materializerNetworkID, + recipient: "did:jwk:different-recipient", data: NodeRecord{MeshIP: "10.200.1.2", ExpiresAt: "2099-01-01T00:00:00Z"}, + timestamp: "2026-07-11T12:00:01Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "self-opaque", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, data: NodeRecord{MeshIP: "10.200.1.1"}, encrypted: true, + timestamp: "2026-07-11T12:00:02Z", + }), + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + set, err := newRawMeshRecordSet(test.records, "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/node"), + ); err == nil || !strings.Contains(err.Error(), "self node") || !strings.Contains(err.Error(), "last-good") { + t.Fatalf("cold opaque self error = %v", err) + } + if client.network != nil || len(client.nodes) != 0 || len(client.rawParsedOutcomes) != 0 { + t.Fatalf("failed self projection mutated state: network=%#v nodes=%#v cache=%d", + client.network, client.nodes, len(client.rawParsedOutcomes)) + } + }) + } +} + +func TestMaterializeRawMeshRecordSetWarmOpaqueSelfPreservesExpiryThenDeleteDownAndRenew(t *testing.T) { + now := time.Now().UTC() + expiresAt := now.Add(time.Hour).Format(time.RFC3339Nano) + renewedExpiry := now.Add(2 * time.Hour).Format(time.RFC3339Nano) + records := []json.RawMessage{ + materializerRecord(t, materializerRecordSpec{ + id: materializerNetworkID, path: "network", + data: NetworkConfig{Name: "security", MeshCIDR: "10.200.0.0/16"}, timestamp: "2026-07-11T12:00:00Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "self-node", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, + data: NodeRecord{MeshIP: "10.200.1.1", Label: "last-good-self", ExpiresAt: expiresAt}, + timestamp: "2026-07-11T12:00:01Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "peer-node", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.2", Label: "peer"}, + timestamp: "2026-07-11T12:00:02Z", + }), + } + set, err := newRawMeshRecordSet(records, "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSet(context.Background(), set); err != nil { + t.Fatal(err) + } + + replacement := materializerRecord(t, materializerRecordSpec{ + id: "self-node", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, + data: NodeRecord{MeshIP: "10.200.99.99", Label: "unreadable", ExpiresAt: ""}, encrypted: true, + dateCreated: "2026-07-11T12:00:01Z", timestamp: "2026-07-11T12:00:03Z", + }) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(replacement, ""), ""); err != nil || !changed { + t.Fatalf("apply opaque self replacement changed=%v err=%v", changed, err) + } + response, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/node"), + ) + if err != nil { + t.Fatal(err) + } + self := client.nodes[materializerSelfDID] + if self == nil || self.Opaque || self.Revoked || self.RecordID != "self-node" || self.MeshIP != "10.200.1.1" || + self.Label != "last-good-self" || self.ExpiresAt != expiresAt { + t.Fatalf("warm opaque self did not preserve typed last-good: %#v", self) + } + if response.Node == nil || response.Node.ExpiresAt != expiresAt || nodeRecordExpired(self, now) || + !nodeRecordExpired(self, now.Add(2*time.Hour)) { + t.Fatalf("preserved self expiry was not enforced: node=%#v response=%#v", self, response.Node) + } + + deleted := rawRecordTestSubscription(rawRecordTestDelete(t, "self-node", "2026-07-11T12:00:04Z"), "") + if changed, err := set.applySubscriptionMessage(deleted, ""); err != nil || !changed { + t.Fatalf("delete self changed=%v err=%v", changed, err) + } + down, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/node"), + ) + if err != nil { + t.Fatal(err) + } + revoked := client.nodes[materializerSelfDID] + if down.Node == nil || len(down.Peers) != 0 || revoked == nil || !revoked.Revoked || revoked.Opaque || + revoked.ExpiresAt != revokedSelfExpiry || !nodeRecordExpired(revoked, now) { + t.Fatalf("self deletion did not commit down map: response=%#v node=%#v", down, revoked) + } + + renewal := materializerRecord(t, materializerRecordSpec{ + id: "self-renewed", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, + data: NodeRecord{MeshIP: "10.200.1.1", Label: "renewed-self", ExpiresAt: renewedExpiry}, + timestamp: "2026-07-11T12:00:05Z", + }) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(renewal, ""), ""); err != nil || !changed { + t.Fatalf("apply renewal changed=%v err=%v", changed, err) + } + active, err := client.materializeRawMeshRecordSet(context.Background(), set) + if err != nil { + t.Fatal(err) + } + renewed := client.nodes[materializerSelfDID] + if active.Node == nil || len(active.Peers) != 1 || renewed == nil || renewed.Revoked || renewed.Opaque || + renewed.RecordID != "self-renewed" || renewed.ExpiresAt != renewedExpiry { + t.Fatalf("renewal did not restore active map: response=%#v node=%#v", active, renewed) + } +} + +func TestMaterializeRawMeshRecordSetColdMissingSelfEmitsDownMapAndRenewal(t *testing.T) { + peerDID, _ := testDIDJWK(t) + set, err := newRawMeshRecordSet([]json.RawMessage{ + materializerRecord(t, materializerRecordSpec{ + id: materializerNetworkID, path: "network", + data: NetworkConfig{Name: "security", MeshCIDR: "10.200.0.0/16"}, timestamp: "2026-07-11T12:00:00Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "peer-node", path: "network/node", parentContext: materializerNetworkID, + recipient: peerDID, data: NodeRecord{MeshIP: "10.200.1.2"}, timestamp: "2026-07-11T12:00:01Z", + }), + }, "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + down, err := client.materializeRawMeshRecordSet(context.Background(), set) + if err != nil { + t.Fatal(err) + } + self := client.nodes[materializerSelfDID] + if down.Node == nil || len(down.Peers) != 0 || self == nil || !self.Revoked || self.ExpiresAt != revokedSelfExpiry { + t.Fatalf("cold missing-self projection = %#v node=%#v", down, self) + } + renewal := materializerRecord(t, materializerRecordSpec{ + id: "self-renewed", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, data: NodeRecord{MeshIP: "10.200.1.1"}, timestamp: "2026-07-11T12:00:02Z", + }) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(renewal, ""), ""); err != nil || !changed { + t.Fatalf("apply cold renewal changed=%v err=%v", changed, err) + } + active, err := client.materializeRawMeshRecordSet(context.Background(), set) + if err != nil { + t.Fatal(err) + } + if active.Node == nil || len(active.Peers) != 1 || client.nodes[materializerSelfDID].Revoked { + t.Fatalf("cold renewal did not restore peers: response=%#v node=%#v", active, client.nodes[materializerSelfDID]) + } +} + +func TestApplyPendingStateSelfDeleteCommitsDownBaselineAndRenewal(t *testing.T) { + peerDID, _ := testDIDJWK(t) + records := append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "peer-node", path: "network/node", parentContext: materializerNetworkID, + recipient: peerDID, data: NodeRecord{MeshIP: "10.200.1.2"}, timestamp: "2026-07-11T12:00:02Z", + })) + client := newMaterializerTestClient() + installDeltaTestBaseline(t, client, records) + + deleted := rawRecordTestSubscription(rawRecordTestDelete(t, "self-node", "2026-07-11T12:00:03Z"), "") + if err := client.StageTopologyEvent(deleted); err != nil { + t.Fatal(err) + } + down, err := client.ApplyPendingState(context.Background()) + if err != nil { + t.Fatal(err) + } + if down.Node == nil || len(down.Peers) != 0 || !client.nodes[materializerSelfDID].Revoked { + t.Fatalf("self delete did not publish down map: response=%#v node=%#v", down, client.nodes[materializerSelfDID]) + } + client.deltaMu.Lock() + baseline := client.rawBaseline + pending := len(client.pendingTopology) + repair := client.fullReconciliation + client.deltaMu.Unlock() + if baseline == nil { + t.Fatal("self delete discarded raw baseline") + } + if _, present := baseline.get("self-node"); present || pending != 0 || repair { + t.Fatalf("self delete not committed: present=%v pending=%d repair=%v", present, pending, repair) + } + + renewal := materializerRecord(t, materializerRecordSpec{ + id: "self-renewed", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, data: NodeRecord{MeshIP: "10.200.1.1"}, timestamp: "2026-07-11T12:00:04Z", + }) + if err := client.StageTopologyEvent(rawRecordTestSubscription(renewal, "")); err != nil { + t.Fatal(err) + } + active, err := client.ApplyPendingState(context.Background()) + if err != nil { + t.Fatal(err) + } + if active.Node == nil || len(active.Peers) != 1 || client.nodes[materializerSelfDID].Revoked { + t.Fatalf("self renewal did not restore committed map: response=%#v node=%#v", active, client.nodes[materializerSelfDID]) + } +} + +func TestApplyPendingStateSelfDeletePreemptsUnreadableOptionalRecords(t *testing.T) { + peerDID, _ := testDIDJWK(t) + records := append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "peer-node", path: "network/node", parentContext: materializerNetworkID, + recipient: peerDID, data: NodeRecord{MeshIP: "10.200.1.2"}, timestamp: "2026-07-11T12:00:02Z", + })) + client := newMaterializerTestClient() + installDeltaTestBaseline(t, client, records) + + events := []json.RawMessage{ + rawRecordTestDelete(t, "self-node", "2026-07-11T12:00:03Z"), + materializerRecord(t, materializerRecordSpec{ + id: "relay-cold", path: "network/relay", parentContext: materializerNetworkID, + data: RelayData{URL: "unreadable-relay.example", Region: "unreadable"}, encrypted: true, + timestamp: "2026-07-11T12:00:03.100Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "acl-cold", path: "network/aclPolicy", parentContext: materializerNetworkID, + data: ACLPolicyData{Version: 1, DefaultAction: "deny"}, encrypted: true, + timestamp: "2026-07-11T12:00:03.200Z", + }), + } + for _, raw := range events { + if err := client.StageTopologyEvent(rawRecordTestSubscription(raw, "")); err != nil { + t.Fatal(err) + } + } + down, err := client.ApplyPendingState(context.Background()) + if err != nil { + t.Fatalf("self revocation was blocked by unreadable optional record: %v", err) + } + if down.Node == nil || len(down.Peers) != 0 || !client.nodes[materializerSelfDID].Revoked { + t.Fatalf("batched self delete did not publish down map: response=%#v node=%#v", down, client.nodes[materializerSelfDID]) + } + client.deltaMu.Lock() + baseline := client.rawBaseline + pending := len(client.pendingTopology) + client.deltaMu.Unlock() + if baseline == nil { + t.Fatal("batched self revocation discarded raw baseline") + } + for _, recordID := range []string{"relay-cold", "acl-cold"} { + if _, present := baseline.get(recordID); !present { + t.Fatalf("committed down baseline lost %s", recordID) + } + } + if _, present := baseline.get("self-node"); present || pending != 0 { + t.Fatalf("batched down commit incomplete: selfPresent=%v pending=%d", present, pending) + } + + renewalEvents := []json.RawMessage{ + materializerRecord(t, materializerRecordSpec{ + id: "self-renewed", path: "network/node", parentContext: materializerNetworkID, + recipient: materializerSelfDID, data: NodeRecord{MeshIP: "10.200.1.1"}, + timestamp: "2026-07-11T12:00:04Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "relay-cold", path: "network/relay", parentContext: materializerNetworkID, + data: RelayData{URL: "relay-restored.example", Region: "restored"}, + dateCreated: "2026-07-11T12:00:03.100Z", timestamp: "2026-07-11T12:00:04.100Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "acl-cold", path: "network/aclPolicy", parentContext: materializerNetworkID, + data: ACLPolicyData{Version: 2, DefaultAction: "accept"}, + dateCreated: "2026-07-11T12:00:03.200Z", timestamp: "2026-07-11T12:00:04.200Z", + }), + } + for _, raw := range renewalEvents { + if err := client.StageTopologyEvent(rawRecordTestSubscription(raw, "")); err != nil { + t.Fatal(err) + } + } + active, err := client.ApplyPendingState(context.Background()) + if err != nil { + t.Fatal(err) + } + if active.Node == nil || len(active.Peers) != 1 || client.nodes[materializerSelfDID].Revoked || + len(client.relays) != 1 || client.relays[0].URL != "relay-restored.example" || + client.acl == nil || client.acl.Version != 2 { + t.Fatalf("renewal did not reparse optional state: response=%#v relays=%#v acl=%#v", active, client.relays, client.acl) + } +} + +func TestMaterializeRawMeshRecordSetOpaquePeerSkippedBeforeFallbackIdentity(t *testing.T) { + peerDID, _ := testDIDJWK(t) + set, err := newRawMeshRecordSet(append(materializerBaseRecords(t), materializerRecord(t, materializerRecordSpec{ + id: "opaque-production-peer", path: "network/node", parentContext: materializerNetworkID, + recipient: peerDID, data: NodeRecord{MeshIP: "10.200.88.88", ExpiresAt: "2099-01-01T00:00:00Z"}, + encrypted: true, timestamp: "2026-07-11T12:00:02Z", + })), "") + if err != nil { + t.Fatal(err) + } + client := newMaterializerTestClient() + response, err := client.materializeRawMeshRecordSetWithDecryptors( + context.Background(), set, materializerUnavailableDecryptors("network/node"), + ) + if err != nil { + t.Fatal(err) + } + ghost := client.nodes[peerDID] + if ghost == nil || !ghost.Opaque || ghost.MeshIP != "" || len(response.Peers) != 0 { + t.Fatalf("opaque production peer entered map: ghost=%#v peers=%#v", ghost, response.Peers) + } + fixture := nodeRecordToNode(1, peerDID, ghost) + client.applyFallbackMeshIP(fixture) + if !fixture.MeshIP.IsValid() || fixture.Key == "" { + t.Fatalf("fixture did not prove fallback/key bypass risk: %#v", fixture) + } +} + +func TestMaterializeRawMeshRecordSetOpaquePeerDoesNotBlockEndpointDelta(t *testing.T) { + records := append(materializerBaseRecords(t), + materializerRecord(t, materializerRecordSpec{ + id: "member-record", path: "network/member", parentContext: materializerNetworkID, + recipient: "did:jwk:member", data: MemberRecord{Label: "member"}, timestamp: "2026-07-11T12:00:02Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "opaque-peer-node", path: "network/member/node", parentContext: materializerNetworkID + "/member-record", + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.2"}, encrypted: true, + timestamp: "2026-07-11T12:00:03Z", + }), + materializerRecord(t, materializerRecordSpec{ + id: "self-endpoint-old", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", + data: EndpointData{LocalEndpoints: []string{"192.0.2.1:1111"}}, timestamp: "2026-07-11T12:00:04Z", + }), + ) + set, err := newRawMeshRecordSet(records, "") + if err != nil { + t.Fatal(err) + } + decryptCalls := 0 + decryptors := func(_ context.Context, path string) EntryDecryptor { + if path != "network/member/node" { + return nil + } + return func([]byte, *dwncrypto.Encryption) ([]byte, error) { + decryptCalls++ + return nil, fmt.Errorf("%w: role audience key unavailable", errAudienceKeyDeliveryAbsent) + } + } + client := newMaterializerTestClient() + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatalf("initial opaque-peer materialization: %v", err) + } + if got := client.undecryptablePeers.Load(); got != 1 { + t.Fatalf("initial undecryptable count = %d, want 1", got) + } + if decryptCalls != 1 { + t.Fatalf("initial decrypt calls = %d, want 1", decryptCalls) + } + if _, ok := client.nodeFailures["opaque-peer-node"]; !ok { + t.Fatalf("node failure episode was not retained: %#v", client.nodeFailures) + } + + newEndpoint := materializerRecord(t, materializerRecordSpec{ + id: "self-endpoint-new", path: "network/node/endpoint", parentContext: materializerNetworkID + "/self-node", squash: true, + data: EndpointData{LocalEndpoints: []string{"192.0.2.1:2222"}}, timestamp: "2026-07-11T12:00:05Z", + }) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(newEndpoint, ""), ""); err != nil || !changed { + t.Fatalf("endpoint delta changed=%v err=%v", changed, err) + } + response, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors) + if err != nil { + t.Fatalf("endpoint delta with opaque peer: %v", err) + } + if response.Node == nil || len(response.Node.Endpoints) != 1 || response.Node.Endpoints[0] != "192.0.2.1:2222" { + t.Fatalf("endpoint delta response = %#v", response.Node) + } + if got := client.undecryptablePeers.Load(); got != 1 { + t.Fatalf("unchanged failure episode incremented count to %d", got) + } + if decryptCalls != 1 { + t.Fatalf("unrelated endpoint delta retried unchanged encrypted peer %d times", decryptCalls-1) + } + + changedPeer := materializerRecord(t, materializerRecordSpec{ + id: "opaque-peer-node", path: "network/member/node", parentContext: materializerNetworkID + "/member-record", + recipient: materializerPeerDID, data: NodeRecord{MeshIP: "10.200.1.3"}, encrypted: true, + timestamp: "2026-07-11T12:00:05.500Z", + }) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(changedPeer, ""), ""); err != nil || !changed { + t.Fatalf("changed peer winner changed=%v err=%v", changed, err) + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatalf("changed opaque peer materialization: %v", err) + } + if decryptCalls != 2 { + t.Fatalf("changed winner decrypt calls = %d, want exactly 2 total", decryptCalls) + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatalf("cached changed opaque peer materialization: %v", err) + } + if decryptCalls != 2 { + t.Fatalf("unchanged changed-winner projection retried decrypt: %d", decryptCalls) + } + + client.invalidateRawParsedOutcomes() + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatalf("materialize after delivery/full invalidation: %v", err) + } + if decryptCalls != 3 { + t.Fatalf("delivery/full invalidation decrypt calls = %d, want 3", decryptCalls) + } + + deleted := rawRecordTestSubscription(rawRecordTestDelete(t, "opaque-peer-node", "2026-07-11T12:00:06Z"), "") + if changed, err := set.applySubscriptionMessage(deleted, ""); err != nil || !changed { + t.Fatalf("delete opaque peer changed=%v err=%v", changed, err) + } + if _, err := client.materializeRawMeshRecordSetWithDecryptors(context.Background(), set, decryptors); err != nil { + t.Fatalf("materialize opaque-peer deletion: %v", err) + } + if _, ok := client.nodeFailures["opaque-peer-node"]; ok { + t.Fatalf("deleted node failure episode was retained: %#v", client.nodeFailures) + } +} + +type materializerRecordSpec struct { + id string + protocol string + path string + parentContext string + recipient string + data any + timestamp string + dateCreated string + squash bool + encrypted bool +} + +func materializerRecord(t *testing.T, spec materializerRecordSpec) json.RawMessage { + t.Helper() + protocol := spec.protocol + if protocol == "" { + protocol = protocols.MeshProtocolURI + } + timestamp := spec.timestamp + if timestamp == "" { + timestamp = time.Date(2026, 7, 11, 12, 0, 0, 0, time.UTC).Format(time.RFC3339Nano) + } + dateCreated := spec.dateCreated + if dateCreated == "" { + dateCreated = timestamp + } + data, ok := spec.data.(json.RawMessage) + if !ok { + var err error + data, err = json.Marshal(spec.data) + if err != nil { + t.Fatalf("marshal record %s data: %v", spec.id, err) + } + } + descriptor := map[string]any{ + "interface": "Records", + "method": "Write", + "protocol": protocol, + "protocolPath": spec.path, + "dateCreated": dateCreated, + "messageTimestamp": timestamp, + } + contextID := spec.id + if spec.parentContext != "" { + segments := strings.Split(spec.parentContext, "/") + descriptor["parentId"] = segments[len(segments)-1] + contextID = spec.parentContext + "/" + spec.id + } + if spec.squash { + descriptor["squash"] = true + } + if spec.recipient != "" { + descriptor["recipient"] = spec.recipient + } + message := map[string]any{ + "recordId": spec.id, + "contextId": contextID, + "descriptor": descriptor, + "encodedData": base64.RawURLEncoding.EncodeToString(data), + } + if spec.encrypted { + message["encryption"] = map[string]any{} + } + raw, err := json.Marshal(message) + if err != nil { + t.Fatalf("marshal record %s: %v", spec.id, err) + } + return raw +} + +func newMaterializerTestClient() *DWNClient { + return &DWNClient{ + networkRecordID: materializerNetworkID, + selfDID: materializerSelfDID, + logger: slog.Default(), + members: make(map[string]*MemberRecord), + nodes: make(map[string]*NodeRecord), + endpointFailures: make(map[string]string), + } +} + +func materializerUnavailableDecryptors(paths ...string) func(context.Context, string) EntryDecryptor { + unavailable := make(map[string]struct{}, len(paths)) + for _, path := range paths { + unavailable[path] = struct{}{} + } + return func(_ context.Context, path string) EntryDecryptor { + if _, ok := unavailable[path]; !ok { + return nil + } + return func([]byte, *dwncrypto.Encryption) ([]byte, error) { + return nil, fmt.Errorf("%w: delivery not present", errAudienceKeyDeliveryAbsent) + } + } +} diff --git a/internal/control/raw_record_set.go b/internal/control/raw_record_set.go new file mode 100644 index 0000000..e86154b --- /dev/null +++ b/internal/control/raw_record_set.go @@ -0,0 +1,807 @@ +package control + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/enboxorg/meshd/internal/dwn" +) + +var ( + errMalformedRawMeshRecord = errors.New("malformed raw mesh record") + errRawMeshRecordDataUnavailable = errors.New("raw mesh record data unavailable") + errUnsupportedRawMeshRecordEvent = errors.New("unsupported raw mesh record event") +) + +// rawMeshRecordMethod and rawMeshRecordRevision mirror the DWN base-state +// lattice. messageCID is the canonical tie-breaker for equal timestamps. +type rawMeshRecordMethod uint8 + +const ( + rawMeshRecordWrite rawMeshRecordMethod = iota + 1 + rawMeshRecordDelete +) + +type rawMeshSquashSlot struct { + protocol string + protocolPath string + parentContextID string +} + +type rawMeshRecordRevision struct { + method rawMeshRecordMethod + messageTimestamp string + revision time.Time + messageCID string + prune bool + slot rawMeshSquashSlot +} + +// rawMeshRecord is the lossless local representation of one RecordsWrite. +// The descriptor fields are duplicated here so callers can route records +// without repeatedly decoding raw. raw and all values returned from this type +// are deep copies; a caller never receives storage owned by the record set. +type rawMeshRecord struct { + raw json.RawMessage + recordID string + protocol string + protocolPath string + contextID string + parentContextID string + parentID string + recipient string + dateCreated string + dateCreatedTime time.Time + messageTimestamp string + revision time.Time + messageCID string + squash bool +} + +func (r rawMeshRecord) clone() rawMeshRecord { + r.raw = cloneRawJSON(r.raw) + return r +} + +func (r rawMeshRecord) slot() rawMeshSquashSlot { + return rawMeshSquashSlot{ + protocol: r.protocol, + protocolPath: r.protocolPath, + parentContextID: r.parentContextID, + } +} + +func (r rawMeshRecord) head() rawMeshRecordRevision { + return rawMeshRecordRevision{ + method: rawMeshRecordWrite, + messageTimestamp: r.messageTimestamp, + revision: r.revision, + messageCID: r.messageCID, + slot: r.slot(), + } +} + +type rawMeshRecordDeleteData struct { + recordID string + rawMeshRecordRevision +} + +// rawMeshRecordSet indexes the latest RecordsWrite by record ID. heads also +// retains delete tombstones, whose delete-wins lattice permanently prevents a +// delayed RecordsWrite from resurrecting the record. Squash indexes mirror the +// DWN server's (protocol, protocolPath, parent-context) scope. +type rawMeshRecordSet struct { + mu sync.RWMutex + records map[string]rawMeshRecord + heads map[string]rawMeshRecordRevision + slotRecords map[rawMeshSquashSlot]map[string]struct{} + squashFloors map[rawMeshSquashSlot]time.Time +} + +func (s *rawMeshRecordSet) initLocked(capacity int) { + if s.records == nil { + s.records = make(map[string]rawMeshRecord, capacity) + } + if s.heads == nil { + s.heads = make(map[string]rawMeshRecordRevision, capacity) + } + if s.slotRecords == nil { + s.slotRecords = make(map[rawMeshSquashSlot]map[string]struct{}) + } + if s.squashFloors == nil { + s.squashFloors = make(map[rawMeshSquashSlot]time.Time) + } +} + +func newRawMeshRecordSet(entries []json.RawMessage, pathHint string) (*rawMeshRecordSet, error) { + set := &rawMeshRecordSet{} + set.initLocked(len(entries)) + if _, err := set.addEntries(entries, pathHint); err != nil { + return nil, err + } + return set, nil +} + +// addEntries atomically normalizes RecordsWrite query entries before merging +// them. Sorting by the DWN revision order makes reconstruction deterministic +// and lets a visible squash record rebuild its slot floor. +func (s *rawMeshRecordSet) addEntries(entries []json.RawMessage, pathHint string) (changed bool, err error) { + if s == nil { + return false, fmt.Errorf("%w: nil record set", errMalformedRawMeshRecord) + } + records := make([]rawMeshRecord, 0, len(entries)) + for i, entry := range entries { + record, err := normalizeRawMeshRecord(entry, pathHint) + if err != nil { + return false, fmt.Errorf("entry %d: %w", i, err) + } + records = append(records, record) + } + sort.SliceStable(records, func(i, j int) bool { + if comparison := compareRawMeshRecordRevision(records[i].head(), records[j].head()); comparison != 0 { + return comparison < 0 + } + return records[i].recordID < records[j].recordID + }) + + s.mu.Lock() + defer s.mu.Unlock() + s.initLocked(len(records)) + + // A RecordsQuery batch is an already-visible snapshot. Floors that predate + // this batch still fence stale data, but squash writes within the batch are + // resolved together so equal-timestamp siblings deliberately retained by + // the server are not made order-dependent. + floorsBefore := make(map[rawMeshSquashSlot]time.Time, len(s.squashFloors)) + for slot, floor := range s.squashFloors { + floorsBefore[slot] = floor + } + var squashes []rawMeshRecord + for _, record := range records { + if floor, ok := floorsBefore[record.slot()]; ok && !record.revision.After(floor) { + if current, exists := s.heads[record.recordID]; !exists || compareRawMeshRecordRevision(record.head(), current) != 0 { + continue + } + } + if !s.writeCouldApplyLocked(record) { + continue + } + s.storeWriteLocked(record) + changed = true + if record.squash { + squashes = append(squashes, record) + } + } + for _, squash := range squashes { + current, ok := s.heads[squash.recordID] + if !ok || current.method != rawMeshRecordWrite || current.messageCID != squash.messageCID { + continue + } + changed = s.applySquashLocked(squash) || changed + } + return changed, nil +} + +func (s *rawMeshRecordSet) clone() *rawMeshRecordSet { + if s == nil { + return nil + } + s.mu.RLock() + defer s.mu.RUnlock() + + cloned := &rawMeshRecordSet{} + cloned.initLocked(len(s.records)) + for id, record := range s.records { + cloned.records[id] = record.clone() + } + for id, head := range s.heads { + cloned.heads[id] = head + } + for slot, ids := range s.slotRecords { + clonedIDs := make(map[string]struct{}, len(ids)) + for id := range ids { + clonedIDs[id] = struct{}{} + } + cloned.slotRecords[slot] = clonedIDs + } + for slot, floor := range s.squashFloors { + cloned.squashFloors[slot] = floor + } + return cloned +} + +func (s *rawMeshRecordSet) len() int { + if s == nil { + return 0 + } + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.records) +} + +func (s *rawMeshRecordSet) get(recordID string) (rawMeshRecord, bool) { + if s == nil { + return rawMeshRecord{}, false + } + s.mu.RLock() + defer s.mu.RUnlock() + record, ok := s.records[recordID] + return record.clone(), ok +} + +// all returns a deterministic, deeply cloned snapshot ordered by record ID. +func (s *rawMeshRecordSet) all() []rawMeshRecord { + if s == nil { + return nil + } + s.mu.RLock() + defer s.mu.RUnlock() + + ids := make([]string, 0, len(s.records)) + for id := range s.records { + ids = append(ids, id) + } + sort.Strings(ids) + + records := make([]rawMeshRecord, 0, len(ids)) + for _, id := range ids { + records = append(records, s.records[id].clone()) + } + return records +} + +// applySubscriptionMessage applies one Records Write/Delete event. An explicit +// non-latest row is ignorable, but every accepted event is still fenced by the +// canonical DWN revision and delete-wins lattice because replay can overlap a +// newer full-reconciliation baseline. +func (s *rawMeshRecordSet) applySubscriptionMessage(message *dwn.SubscriptionMessage, pathHint string) (changed bool, err error) { + if s == nil { + return false, fmt.Errorf("%w: nil record set", errMalformedRawMeshRecord) + } + if message != nil && message.IsLatestBaseState != nil && !*message.IsLatestBaseState { + return false, nil + } + if message == nil { + return false, fmt.Errorf("%w: nil subscription message", errMalformedRawMeshRecord) + } + if message.Type != dwn.SubscriptionEventType { + return false, fmt.Errorf("%w: subscription message type %q", errUnsupportedRawMeshRecordEvent, message.Type) + } + if message.Event == nil || len(message.Event.Message) == 0 { + return false, fmt.Errorf("%w: subscription event is missing event.message", errMalformedRawMeshRecord) + } + + computedCID, err := validateSubscriptionRecordMessageCID(message) + if err != nil { + return false, err + } + messageType, err := classifyRawRecordMessage(message.Event.Message) + if err != nil { + return false, err + } + switch messageType { + case "Write": + identity, err := normalizeRawMeshRecordIdentity(message.Event.Message, pathHint) + if err != nil { + return false, err + } + if identity.messageCID != computedCID { + return false, fmt.Errorf("%w: RecordsWrite CID changed while normalizing", errMalformedRawMeshRecord) + } + s.mu.RLock() + couldApply := s.writeCouldApplyLocked(identity) + s.mu.RUnlock() + if !couldApply { + return false, nil + } + + raw, err := injectSubscriptionEncodedData(message.Event.Message, message.EncodedData) + if err != nil { + return false, err + } + if !rawRecordHasEncodedData(raw) { + return false, fmt.Errorf("%w: RecordsWrite event is missing encoded data", errRawMeshRecordDataUnavailable) + } + record, err := normalizeRawMeshRecord(raw, pathHint) + if err != nil { + return false, err + } + if record.messageCID != computedCID { + return false, fmt.Errorf("%w: RecordsWrite CID changed while attaching data", errMalformedRawMeshRecord) + } + s.mu.Lock() + defer s.mu.Unlock() + s.initLocked(1) + return s.applyWriteLocked(record), nil + case "Delete": + deletion, err := normalizeRawMeshRecordDelete(message.Event.Message) + if err != nil { + return false, err + } + if deletion.messageCID != computedCID { + return false, fmt.Errorf("%w: RecordsDelete CID changed while normalizing", errMalformedRawMeshRecord) + } + if len(message.Event.InitialWrite) != 0 { + identity, identityErr := normalizeRawMeshRecordIdentity(message.Event.InitialWrite, pathHint) + if identityErr != nil { + return false, fmt.Errorf("%w: invalid delete initialWrite: %v", errMalformedRawMeshRecord, identityErr) + } + if identity.recordID != deletion.recordID { + return false, fmt.Errorf("%w: delete recordId %q does not match initialWrite %q", errMalformedRawMeshRecord, deletion.recordID, identity.recordID) + } + deletion.slot = identity.slot() + } + s.mu.Lock() + defer s.mu.Unlock() + s.initLocked(1) + return s.applyDeleteLocked(deletion), nil + default: + return false, fmt.Errorf("%w: Records method %q", errUnsupportedRawMeshRecordEvent, messageType) + } +} + +func (s *rawMeshRecordSet) writeCouldApplyLocked(record rawMeshRecord) bool { + incoming := record.head() + if current, ok := s.heads[record.recordID]; ok { + if current.method == rawMeshRecordDelete { + return false + } + if compareRawMeshRecordRevision(incoming, current) <= 0 { + return false + } + } + if floor, ok := s.squashFloors[record.slot()]; ok && !record.revision.After(floor) { + return false + } + return true +} + +func (s *rawMeshRecordSet) storeWriteLocked(record rawMeshRecord) { + slot := record.slot() + if previous, ok := s.heads[record.recordID]; ok && previous.slot != slot { + s.removeSlotRecordLocked(previous.slot, record.recordID) + } + s.records[record.recordID] = record.clone() + s.heads[record.recordID] = record.head() + s.addSlotRecordLocked(slot, record.recordID) +} + +func (s *rawMeshRecordSet) applyWriteLocked(record rawMeshRecord) bool { + if !s.writeCouldApplyLocked(record) { + return false + } + s.storeWriteLocked(record) + changed := true + if record.squash { + changed = s.applySquashLocked(record) || changed + } + return changed +} + +func (s *rawMeshRecordSet) applySquashLocked(record rawMeshRecord) bool { + slot := record.slot() + changed := false + for siblingID := range s.slotRecords[slot] { + if siblingID == record.recordID { + continue + } + sibling, ok := s.heads[siblingID] + if ok && sibling.method == rawMeshRecordWrite && sibling.revision.Before(record.revision) { + s.removeHeadLocked(siblingID) + changed = true + } + } + if floor, ok := s.squashFloors[slot]; !ok || record.revision.After(floor) { + s.squashFloors[slot] = record.revision + } + return changed +} + +func (s *rawMeshRecordSet) applyDeleteLocked(incoming rawMeshRecordDeleteData) bool { + current, exists := s.heads[incoming.recordID] + if exists { + switch current.method { + case rawMeshRecordWrite: + // A stale replay delete must not remove a newer live base state. + if compareRawMeshRecordRevision(incoming.rawMeshRecordRevision, current) <= 0 { + return false + } + case rawMeshRecordDelete: + // Delete is terminal. The sole legal transition is a strictly newer + // prune replacing a plain tombstone. + if current.prune || !incoming.prune || + compareRawMeshRecordRevision(incoming.rawMeshRecordRevision, current) <= 0 { + return false + } + } + } + + if exists { + incoming.slot = current.slot + } + _, visible := s.records[incoming.recordID] + delete(s.records, incoming.recordID) + if exists && current.slot != incoming.slot { + s.removeSlotRecordLocked(current.slot, incoming.recordID) + } + s.heads[incoming.recordID] = incoming.rawMeshRecordRevision + s.addSlotRecordLocked(incoming.slot, incoming.recordID) + return visible +} + +func (s *rawMeshRecordSet) addSlotRecordLocked(slot rawMeshSquashSlot, recordID string) { + if slot == (rawMeshSquashSlot{}) { + return + } + ids := s.slotRecords[slot] + if ids == nil { + ids = make(map[string]struct{}) + s.slotRecords[slot] = ids + } + ids[recordID] = struct{}{} +} + +func (s *rawMeshRecordSet) removeSlotRecordLocked(slot rawMeshSquashSlot, recordID string) { + ids := s.slotRecords[slot] + delete(ids, recordID) + if len(ids) == 0 { + delete(s.slotRecords, slot) + } +} + +func (s *rawMeshRecordSet) removeHeadLocked(recordID string) { + if head, ok := s.heads[recordID]; ok { + s.removeSlotRecordLocked(head.slot, recordID) + } + delete(s.records, recordID) + delete(s.heads, recordID) +} + +type rawMeshRecordDescriptor struct { + Interface string `json:"interface"` + Method string `json:"method"` + Protocol string `json:"protocol"` + ProtocolPath string `json:"protocolPath"` + ParentID string `json:"parentId"` + Recipient string `json:"recipient"` + RecordID string `json:"recordId"` + DateCreated string `json:"dateCreated"` + MessageTimestamp string `json:"messageTimestamp"` + Squash bool `json:"squash"` + Prune bool `json:"prune"` +} + +type rawMeshRecordMessage struct { + RecordID string `json:"recordId"` + ContextID string `json:"contextId"` + Descriptor *rawMeshRecordDescriptor `json:"descriptor"` + EncodedData *json.RawMessage `json:"encodedData"` +} + +func normalizeRawMeshRecord(entry json.RawMessage, pathHint string) (rawMeshRecord, error) { + record, err := normalizeRawMeshRecordIdentity(entry, pathHint) + if err != nil { + return rawMeshRecord{}, err + } + if !rawRecordHasEncodedData(entry) { + return rawMeshRecord{}, fmt.Errorf("%w: RecordsWrite is missing encoded data", errRawMeshRecordDataUnavailable) + } + record.raw = cloneRawJSON(entry) + return record, nil +} + +func normalizeRawMeshRecordIdentity(entry json.RawMessage, pathHint string) (rawMeshRecord, error) { + message, _, err := unwrapRawRecordMessage(entry, "recordsWrite") + if err != nil { + return rawMeshRecord{}, err + } + if message.Descriptor == nil { + return rawMeshRecord{}, fmt.Errorf("%w: RecordsWrite is missing descriptor", errMalformedRawMeshRecord) + } + descriptor := message.Descriptor + if descriptor.Interface != "Records" || descriptor.Method != "Write" { + return rawMeshRecord{}, fmt.Errorf("%w: expected RecordsWrite, got %s%s", errMalformedRawMeshRecord, descriptor.Interface, descriptor.Method) + } + if message.RecordID == "" || descriptor.Protocol == "" { + return rawMeshRecord{}, fmt.Errorf("%w: RecordsWrite is missing recordId or protocol", errMalformedRawMeshRecord) + } + protocolPath := descriptor.ProtocolPath + if protocolPath == "" { + protocolPath = pathHint + } + if protocolPath == "" { + return rawMeshRecord{}, fmt.Errorf("%w: RecordsWrite is missing protocolPath", errMalformedRawMeshRecord) + } + parentContextID, err := validateRawMeshRecordContext(message.RecordID, message.ContextID, descriptor.ParentID) + if err != nil { + return rawMeshRecord{}, err + } + revision, err := parseRawMeshRecordRevision(descriptor.MessageTimestamp) + if err != nil { + return rawMeshRecord{}, err + } + dateCreated, err := parseRawMeshRecordDateCreated(descriptor.DateCreated) + if err != nil { + return rawMeshRecord{}, err + } + messageCID, err := computeRawRecordMessageCID(entry) + if err != nil { + return rawMeshRecord{}, err + } + return rawMeshRecord{ + raw: cloneRawJSON(entry), + recordID: message.RecordID, + protocol: descriptor.Protocol, + protocolPath: protocolPath, + contextID: message.ContextID, + parentContextID: parentContextID, + parentID: descriptor.ParentID, + recipient: descriptor.Recipient, + dateCreated: descriptor.DateCreated, + dateCreatedTime: dateCreated, + messageTimestamp: descriptor.MessageTimestamp, + revision: revision, + messageCID: messageCID, + squash: descriptor.Squash, + }, nil +} + +func normalizeRawMeshRecordDelete(entry json.RawMessage) (rawMeshRecordDeleteData, error) { + message, _, err := unwrapRawRecordMessage(entry, "recordsDelete") + if err != nil { + return rawMeshRecordDeleteData{}, err + } + if message.Descriptor == nil { + return rawMeshRecordDeleteData{}, fmt.Errorf("%w: RecordsDelete is missing descriptor", errMalformedRawMeshRecord) + } + descriptor := message.Descriptor + if descriptor.Interface != "Records" || descriptor.Method != "Delete" { + return rawMeshRecordDeleteData{}, fmt.Errorf("%w: expected RecordsDelete, got %s%s", errMalformedRawMeshRecord, descriptor.Interface, descriptor.Method) + } + if descriptor.RecordID == "" { + return rawMeshRecordDeleteData{}, fmt.Errorf("%w: RecordsDelete descriptor is missing recordId", errMalformedRawMeshRecord) + } + revision, err := parseRawMeshRecordRevision(descriptor.MessageTimestamp) + if err != nil { + return rawMeshRecordDeleteData{}, err + } + messageCID, err := computeRawRecordMessageCID(entry) + if err != nil { + return rawMeshRecordDeleteData{}, err + } + return rawMeshRecordDeleteData{ + recordID: descriptor.RecordID, + rawMeshRecordRevision: rawMeshRecordRevision{ + method: rawMeshRecordDelete, + messageTimestamp: descriptor.MessageTimestamp, + revision: revision, + messageCID: messageCID, + prune: descriptor.Prune, + }, + }, nil +} + +func compareRawMeshRecordRevision(a, b rawMeshRecordRevision) int { + if a.revision.Before(b.revision) { + return -1 + } + if a.revision.After(b.revision) { + return 1 + } + return strings.Compare(a.messageCID, b.messageCID) +} + +func validateRawMeshRecordContext(recordID, contextID, parentID string) (string, error) { + if contextID == "" { + return "", fmt.Errorf("%w: RecordsWrite is missing contextId", errMalformedRawMeshRecord) + } + if contextID == recordID { + if parentID != "" { + return "", fmt.Errorf("%w: root RecordsWrite has parentId %q", errMalformedRawMeshRecord, parentID) + } + return "", nil + } + suffix := "/" + recordID + if !strings.HasSuffix(contextID, suffix) { + return "", fmt.Errorf("%w: contextId %q does not end in recordId %q", errMalformedRawMeshRecord, contextID, recordID) + } + parentContextID := strings.TrimSuffix(contextID, suffix) + if parentContextID == "" || strings.HasSuffix(parentContextID, "/") { + return "", fmt.Errorf("%w: invalid parent context %q", errMalformedRawMeshRecord, parentContextID) + } + lastSlash := strings.LastIndexByte(parentContextID, '/') + expectedParentID := parentContextID[lastSlash+1:] + if parentID == "" || parentID != expectedParentID { + return "", fmt.Errorf("%w: parentId %q does not match context parent %q", errMalformedRawMeshRecord, parentID, expectedParentID) + } + return parentContextID, nil +} + +func computeRawRecordMessageCID(entry json.RawMessage) (string, error) { + raw, _, err := unwrapRawRecordMessageJSON(entry, "") + if err != nil { + return "", err + } + var canonical map[string]any + if err := json.Unmarshal(raw, &canonical); err != nil || canonical == nil { + return "", fmt.Errorf("%w: decoding canonical record message", errMalformedRawMeshRecord) + } + delete(canonical, "encodedData") + delete(canonical, "initialWrite") + delete(canonical, "messageCid") + cid, err := dwn.ComputeCID(canonical) + if err != nil { + return "", fmt.Errorf("%w: computing record message CID: %v", errMalformedRawMeshRecord, err) + } + return cid, nil +} + +func validateSubscriptionRecordMessageCID(message *dwn.SubscriptionMessage) (string, error) { + computed, err := computeRawRecordMessageCID(message.Event.Message) + if err != nil { + return "", err + } + topLevel := message.MessageCID + cursor := "" + if message.Cursor != nil { + cursor = message.Cursor.MessageCID + } + if topLevel != "" && cursor != "" && topLevel != cursor { + return "", fmt.Errorf("%w: subscription messageCid %q disagrees with cursor %q", errMalformedRawMeshRecord, topLevel, cursor) + } + provided := topLevel + if provided == "" { + provided = cursor + } + if provided != "" && provided != computed { + return "", fmt.Errorf("%w: subscription messageCid %q does not match computed %q", errMalformedRawMeshRecord, provided, computed) + } + return computed, nil +} + +func classifyRawRecordMessage(entry json.RawMessage) (string, error) { + message, wrapper, err := unwrapRawRecordMessage(entry, "") + if err != nil { + return "", err + } + if message.Descriptor == nil { + return "", fmt.Errorf("%w: event message is missing descriptor", errMalformedRawMeshRecord) + } + descriptor := message.Descriptor + if descriptor.Interface == "" || descriptor.Method == "" { + return "", fmt.Errorf("%w: event descriptor is missing interface or method", errMalformedRawMeshRecord) + } + if descriptor.Interface != "Records" { + return "", fmt.Errorf("%w: interface %q", errUnsupportedRawMeshRecordEvent, descriptor.Interface) + } + if wrapper != "" { + expected := "records" + descriptor.Method + if wrapper != expected { + return "", fmt.Errorf("%w: wrapper %q contains Records%s", errMalformedRawMeshRecord, wrapper, descriptor.Method) + } + } + if descriptor.Method != "Write" && descriptor.Method != "Delete" { + return "", fmt.Errorf("%w: Records method %q", errUnsupportedRawMeshRecordEvent, descriptor.Method) + } + return descriptor.Method, nil +} + +func unwrapRawRecordMessageJSON(entry json.RawMessage, requestedWrapper string) (json.RawMessage, string, error) { + var object map[string]json.RawMessage + if len(entry) == 0 || json.Unmarshal(entry, &object) != nil || object == nil { + return nil, "", fmt.Errorf("%w: record message is not a JSON object", errMalformedRawMeshRecord) + } + + wrapper := requestedWrapper + if wrapper == "" { + if _, ok := object["recordsWrite"]; ok { + wrapper = "recordsWrite" + } else if _, ok := object["recordsDelete"]; ok { + wrapper = "recordsDelete" + } + } + raw := entry + if wrapper != "" { + wrapped, ok := object[wrapper] + if !ok || len(wrapped) == 0 || string(wrapped) == "null" { + if requestedWrapper == "" { + return nil, "", fmt.Errorf("%w: %s wrapper is empty", errMalformedRawMeshRecord, wrapper) + } + wrapper = "" + } else { + raw = wrapped + } + } + return raw, wrapper, nil +} + +// unwrapRawRecordMessage accepts both flat messages and query/read-style +// {"recordsWrite": {...}} / {"recordsDelete": {...}} wrappers. +func unwrapRawRecordMessage(entry json.RawMessage, requestedWrapper string) (rawMeshRecordMessage, string, error) { + raw, wrapper, err := unwrapRawRecordMessageJSON(entry, requestedWrapper) + if err != nil { + return rawMeshRecordMessage{}, "", err + } + var message rawMeshRecordMessage + if err := json.Unmarshal(raw, &message); err != nil { + return rawMeshRecordMessage{}, "", fmt.Errorf("%w: decoding record message: %v", errMalformedRawMeshRecord, err) + } + return message, wrapper, nil +} + +func injectSubscriptionEncodedData(entry json.RawMessage, encodedData string) (json.RawMessage, error) { + cloned := cloneRawJSON(entry) + if encodedData == "" || rawRecordHasEncodedData(cloned) { + return cloned, nil + } + + var object map[string]json.RawMessage + if err := json.Unmarshal(cloned, &object); err != nil || object == nil { + return nil, fmt.Errorf("%w: RecordsWrite is not a JSON object", errMalformedRawMeshRecord) + } + encoded, err := json.Marshal(encodedData) + if err != nil { + return nil, fmt.Errorf("%w: encoding subscription data: %v", errMalformedRawMeshRecord, err) + } + if wrapped, ok := object["recordsWrite"]; ok { + var write map[string]json.RawMessage + if err := json.Unmarshal(wrapped, &write); err != nil || write == nil { + return nil, fmt.Errorf("%w: recordsWrite wrapper is not a JSON object", errMalformedRawMeshRecord) + } + write["encodedData"] = encoded + object["recordsWrite"], err = json.Marshal(write) + if err != nil { + return nil, fmt.Errorf("%w: encoding recordsWrite wrapper: %v", errMalformedRawMeshRecord, err) + } + } else { + object["encodedData"] = encoded + } + result, err := json.Marshal(object) + if err != nil { + return nil, fmt.Errorf("%w: encoding RecordsWrite: %v", errMalformedRawMeshRecord, err) + } + return result, nil +} + +func rawRecordHasEncodedData(entry json.RawMessage) bool { + message, _, err := unwrapRawRecordMessage(entry, "recordsWrite") + if err != nil || message.EncodedData == nil { + return false + } + var encoded string + return json.Unmarshal(*message.EncodedData, &encoded) == nil && encoded != "" +} + +func parseRawMeshRecordRevision(value string) (time.Time, error) { + if value == "" { + return time.Time{}, fmt.Errorf("%w: record is missing messageTimestamp", errMalformedRawMeshRecord) + } + revision, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return time.Time{}, fmt.Errorf("%w: invalid messageTimestamp %q: %v", errMalformedRawMeshRecord, value, err) + } + return revision, nil +} + +func parseRawMeshRecordDateCreated(value string) (time.Time, error) { + if value == "" { + return time.Time{}, fmt.Errorf("%w: record is missing dateCreated", errMalformedRawMeshRecord) + } + created, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return time.Time{}, fmt.Errorf("%w: invalid dateCreated %q: %v", errMalformedRawMeshRecord, value, err) + } + return created, nil +} + +func cloneRawJSON(raw json.RawMessage) json.RawMessage { + if raw == nil { + return nil + } + return append(json.RawMessage(nil), raw...) +} diff --git a/internal/control/raw_record_set_test.go b/internal/control/raw_record_set_test.go new file mode 100644 index 0000000..09d408d --- /dev/null +++ b/internal/control/raw_record_set_test.go @@ -0,0 +1,785 @@ +package control + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/enboxorg/meshd/internal/dwn" +) + +const rawRecordTestProtocol = "https://example.com/protocol" + +func TestNewRawMeshRecordSetNormalizesEntriesAndKeepsLatestRevision(t *testing.T) { + older := rawRecordTestWrite(t, "record-a", "", "2026-07-11T12:00:00Z", "old") + newer := rawRecordTestWrite(t, "record-a", "", "2026-07-11T12:00:02.123456Z", "new") + wrapped := rawRecordTestWrappedWrite(t, "record-b", "network/member", "2026-07-11T12:00:01Z", "wrapped") + + set, err := newRawMeshRecordSet([]json.RawMessage{newer, older, wrapped}, "network/node") + if err != nil { + t.Fatalf("newRawMeshRecordSet: %v", err) + } + if got := set.len(); got != 2 { + t.Fatalf("len = %d, want 2", got) + } + + recordA, ok := set.get("record-a") + if !ok { + t.Fatal("record-a missing") + } + if recordA.protocol != rawRecordTestProtocol || recordA.protocolPath != "network/node" { + t.Fatalf("record-a protocol/path = %q/%q", recordA.protocol, recordA.protocolPath) + } + if recordA.contextID != "parent-record-a/record-a" || recordA.parentID != "parent-record-a" || recordA.recipient != "did:example:record-a" { + t.Fatalf("record-a metadata = %+v", recordA) + } + if recordA.messageTimestamp != "2026-07-11T12:00:02.123456Z" { + t.Fatalf("record-a timestamp = %q", recordA.messageTimestamp) + } + if got := rawRecordEncodedData(t, recordA.raw); got != "new" { + t.Fatalf("record-a encodedData = %q, want new", got) + } + + recordB, ok := set.get("record-b") + if !ok { + t.Fatal("record-b missing") + } + if recordB.protocolPath != "network/member" { + t.Fatalf("wrapped path = %q", recordB.protocolPath) + } + if got := rawRecordEncodedData(t, recordB.raw); got != "wrapped" { + t.Fatalf("wrapped encodedData = %q", got) + } + + all := set.all() + if len(all) != 2 || all[0].recordID != "record-a" || all[1].recordID != "record-b" { + t.Fatalf("all order = %#v", all) + } +} + +func TestRawMeshRecordSetAddEntriesIsAtomic(t *testing.T) { + set, err := newRawMeshRecordSet([]json.RawMessage{ + rawRecordTestWrite(t, "record-a", "network/node", "2026-07-11T12:00:00Z", "initial"), + }, "") + if err != nil { + t.Fatal(err) + } + + valid := rawRecordTestWrite(t, "record-b", "network/member", "2026-07-11T12:00:01Z", "valid") + malformed := json.RawMessage(`{"descriptor":{"interface":"Records","method":"Write"}}`) + changed, err := set.addEntries([]json.RawMessage{valid, malformed}, "") + if changed || !errors.Is(err, errMalformedRawMeshRecord) { + t.Fatalf("failed batch changed=%v err=%v", changed, err) + } + if set.len() != 1 { + t.Fatalf("failed batch partially mutated set: len=%d", set.len()) + } + if _, ok := set.get("record-b"); ok { + t.Fatal("valid prefix of failed batch was applied") + } + + changed, err = set.addEntries([]json.RawMessage{valid}, "") + if err != nil || !changed || set.len() != 2 { + t.Fatalf("valid batch changed=%v len=%d err=%v", changed, set.len(), err) + } + changed, err = set.addEntries([]json.RawMessage{valid}, "") + if err != nil || changed { + t.Fatalf("idempotent batch changed=%v err=%v", changed, err) + } +} + +func TestRawMeshRecordSetAppliesWriteUpdateAndDelete(t *testing.T) { + set, err := newRawMeshRecordSet(nil, "") + if err != nil { + t.Fatalf("newRawMeshRecordSet: %v", err) + } + + createRaw := rawRecordTestWrite(t, "record-a", "network/node", "2026-07-11T12:00:00Z", "") + create := rawRecordTestSubscription(createRaw, "created") + changed, err := set.applySubscriptionMessage(create, "unused/path") + if err != nil { + t.Fatalf("apply create: %v", err) + } + if !changed { + t.Fatal("create did not report a change") + } + record, ok := set.get("record-a") + if !ok { + t.Fatal("created record missing") + } + if got := rawRecordEncodedData(t, record.raw); got != "created" { + t.Fatalf("injected encodedData = %q", got) + } + + updateRaw := rawRecordTestWrite(t, "record-a", "network/node", "2026-07-11T12:00:01Z", "in-message") + changed, err = set.applySubscriptionMessage(rawRecordTestSubscription(updateRaw, "top-level"), "") + if err != nil { + t.Fatalf("apply update: %v", err) + } + if !changed { + t.Fatal("update did not report a change") + } + record, _ = set.get("record-a") + if got := rawRecordEncodedData(t, record.raw); got != "in-message" { + t.Fatalf("existing encodedData was overwritten: %q", got) + } + + deleteRaw := rawRecordTestDelete(t, "record-a", "2026-07-11T12:00:02Z") + changed, err = set.applySubscriptionMessage(rawRecordTestSubscription(deleteRaw, ""), "") + if err != nil { + t.Fatalf("apply delete: %v", err) + } + if !changed || set.len() != 0 { + t.Fatalf("delete changed=%v len=%d", changed, set.len()) + } + + changed, err = set.applySubscriptionMessage(rawRecordTestSubscription(deleteRaw, ""), "") + if err != nil { + t.Fatalf("replay delete: %v", err) + } + if changed { + t.Fatal("idempotent delete reported a change") + } +} + +func TestRawMeshRecordSetAppliesWrappedWriteEvent(t *testing.T) { + set, _ := newRawMeshRecordSet(nil, "") + event := rawRecordTestSubscription( + rawRecordTestWrappedWrite(t, "record-a", "network/node", "2026-07-11T12:00:00Z", ""), + "from-subscription", + ) + changed, err := set.applySubscriptionMessage(event, "") + if err != nil { + t.Fatalf("apply wrapped write: %v", err) + } + if !changed { + t.Fatal("wrapped write did not report a change") + } + record, _ := set.get("record-a") + if got := rawRecordEncodedData(t, record.raw); got != "from-subscription" { + t.Fatalf("wrapped encodedData = %q", got) + } +} + +func TestRawMeshRecordSetRejectsStaleWritesAndNeverResurrectsTombstone(t *testing.T) { + set, err := newRawMeshRecordSet([]json.RawMessage{ + rawRecordTestWrite(t, "record-a", "network/node", "2026-07-11T12:00:02Z", "current"), + }, "") + if err != nil { + t.Fatal(err) + } + + staleWrite := rawRecordTestSubscription( + rawRecordTestWrite(t, "record-a", "network/node", "2026-07-11T12:00:01Z", "stale"), + "", + ) + if changed, err := set.applySubscriptionMessage(staleWrite, ""); err != nil || changed { + t.Fatalf("stale write changed=%v err=%v", changed, err) + } + + // A stale replay delete cannot remove a newer live base state when the + // optional isLatestBaseState hint is absent. + olderDelete := rawRecordTestSubscription(rawRecordTestDelete(t, "record-a", "2026-07-11T12:00:01Z"), "") + if changed, err := set.applySubscriptionMessage(olderDelete, ""); err != nil || changed { + t.Fatalf("stale delete changed=%v err=%v", changed, err) + } + if record, ok := set.get("record-a"); !ok || rawRecordEncodedData(t, record.raw) != "current" { + t.Fatalf("stale delete removed current write: record=%#v ok=%v", record, ok) + } + + newerDelete := rawRecordTestSubscription(rawRecordTestDelete(t, "record-a", "2026-07-11T12:00:03Z"), "") + if changed, err := set.applySubscriptionMessage(newerDelete, ""); err != nil || !changed { + t.Fatalf("newer delete changed=%v err=%v", changed, err) + } + if set.len() != 0 { + t.Fatalf("newer delete retained record: len=%d", set.len()) + } + + for _, write := range []*dwn.SubscriptionMessage{ + staleWrite, + rawRecordTestSubscription(rawRecordTestWrite(t, "record-a", "network/node", "2026-07-11T12:00:04Z", "later"), ""), + } { + if changed, err := set.applySubscriptionMessage(write, ""); err != nil || changed || set.len() != 0 { + t.Fatalf("write resurrected tombstone changed=%v len=%d err=%v", changed, set.len(), err) + } + } +} + +func TestRawMeshRecordSetDeleteAndPruneTransitionsFollowBaseStateOrder(t *testing.T) { + set, err := newRawMeshRecordSet([]json.RawMessage{ + rawRecordTestWrite(t, "record-a", "network/node", "2026-07-11T12:00:01Z", "current"), + }, "") + if err != nil { + t.Fatal(err) + } + + plain := rawRecordTestSubscription(rawRecordTestDelete(t, "record-a", "2026-07-11T12:00:02Z"), "") + if changed, err := set.applySubscriptionMessage(plain, ""); err != nil || !changed { + t.Fatalf("plain delete changed=%v err=%v", changed, err) + } + + newerPlain := rawRecordTestSubscription(rawRecordTestDelete(t, "record-a", "2026-07-11T12:00:03Z"), "") + if changed, err := set.applySubscriptionMessage(newerPlain, ""); err != nil || changed { + t.Fatalf("second plain delete changed=%v err=%v", changed, err) + } + olderPrune := rawRecordTestDeleteWith(t, "record-a", "2026-07-11T12:00:01Z", true, "") + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(olderPrune, ""), ""); err != nil || changed { + t.Fatalf("older prune changed=%v err=%v", changed, err) + } + + newerPrune := rawRecordTestDeleteWith(t, "record-a", "2026-07-11T12:00:04Z", true, "") + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(newerPrune, ""), ""); err != nil || changed { + // The visible record was already deleted, so accepting the stronger + // tombstone does not change the visible projection. + t.Fatalf("newer prune visible change=%v err=%v", changed, err) + } + head := set.heads["record-a"] + if !head.prune || head.messageTimestamp != "2026-07-11T12:00:04Z" { + t.Fatalf("head after prune = %#v", head) + } + + terminalPrune := rawRecordTestDeleteWith(t, "record-a", "2026-07-11T12:00:05Z", true, "") + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(terminalPrune, ""), ""); err != nil || changed { + t.Fatalf("second prune changed=%v err=%v", changed, err) + } + if got := set.heads["record-a"].messageTimestamp; got != "2026-07-11T12:00:04Z" { + t.Fatalf("terminal prune was replaced: %s", got) + } +} + +func TestRawMeshRecordSetIgnoresNonLatestBaseState(t *testing.T) { + set, _ := newRawMeshRecordSet(nil, "") + latest := false + changed, err := set.applySubscriptionMessage(&dwn.SubscriptionMessage{ + Type: "not-an-event", + IsLatestBaseState: &latest, + }, "") + if err != nil || changed || set.len() != 0 { + t.Fatalf("ignored event changed=%v len=%d err=%v", changed, set.len(), err) + } +} + +func TestRawMeshRecordSetMissingWriteDataIsMalformedAndDoesNotMutate(t *testing.T) { + missingData := rawRecordTestWrite(t, "record-a", "network/node", "2026-07-11T12:00:00Z", "") + if _, err := newRawMeshRecordSet([]json.RawMessage{missingData}, ""); !errors.Is(err, errRawMeshRecordDataUnavailable) { + t.Fatalf("initial missing-data error = %v", err) + } + + set, _ := newRawMeshRecordSet(nil, "") + event := rawRecordTestSubscription(missingData, "") + changed, err := set.applySubscriptionMessage(event, "") + if changed || !errors.Is(err, errRawMeshRecordDataUnavailable) { + t.Fatalf("missing data changed=%v err=%v", changed, err) + } + if errors.Is(err, errMalformedRawMeshRecord) { + t.Fatalf("missing data was classified as malformed: %v", err) + } + if set.len() != 0 { + t.Fatalf("missing-data event mutated set: len=%d", set.len()) + } +} + +func TestRawMeshRecordSetDeepCopiesInputsOutputsAndClones(t *testing.T) { + source := rawRecordTestWrite(t, "record-a", "network/node", "2026-07-11T12:00:00Z", "original") + set, err := newRawMeshRecordSet([]json.RawMessage{source}, "") + if err != nil { + t.Fatal(err) + } + for i := range source { + source[i] = 'x' + } + record, _ := set.get("record-a") + if got := rawRecordEncodedData(t, record.raw); got != "original" { + t.Fatalf("input mutation reached set: %q", got) + } + + record.raw[0] = 'x' + all := set.all() + all[0].raw[0] = 'x' + record, _ = set.get("record-a") + if got := rawRecordEncodedData(t, record.raw); got != "original" { + t.Fatalf("output mutation reached set: %q", got) + } + + cloned := set.clone() + update := rawRecordTestSubscription( + rawRecordTestWrite(t, "record-a", "network/node", "2026-07-11T12:00:01Z", "clone-only"), + "", + ) + if changed, err := cloned.applySubscriptionMessage(update, ""); err != nil || !changed { + t.Fatalf("update clone changed=%v err=%v", changed, err) + } + originalRecord, _ := set.get("record-a") + cloneRecord, _ := cloned.get("record-a") + if rawRecordEncodedData(t, originalRecord.raw) != "original" || rawRecordEncodedData(t, cloneRecord.raw) != "clone-only" { + t.Fatal("clone and original share record storage") + } + + eventRaw := rawRecordTestWrite(t, "record-b", "network/node", "2026-07-11T12:00:02Z", "event") + event := rawRecordTestSubscription(eventRaw, "") + if _, err := set.applySubscriptionMessage(event, ""); err != nil { + t.Fatal(err) + } + for i := range eventRaw { + eventRaw[i] = 'x' + } + eventRecord, _ := set.get("record-b") + if got := rawRecordEncodedData(t, eventRecord.raw); got != "event" { + t.Fatalf("event mutation reached set: %q", got) + } +} + +func TestRawMeshRecordSetClassifiesMalformedAndUnsupportedEvents(t *testing.T) { + set, _ := newRawMeshRecordSet(nil, "") + tests := []struct { + name string + msg *dwn.SubscriptionMessage + want error + }{ + {name: "nil", msg: nil, want: errMalformedRawMeshRecord}, + {name: "eose", msg: &dwn.SubscriptionMessage{Type: dwn.SubscriptionEOSEType}, want: errUnsupportedRawMeshRecordEvent}, + {name: "missing event", msg: &dwn.SubscriptionMessage{Type: dwn.SubscriptionEventType}, want: errMalformedRawMeshRecord}, + {name: "invalid json", msg: rawRecordTestSubscription(json.RawMessage(`{`), "data"), want: errMalformedRawMeshRecord}, + { + name: "unsupported interface", + msg: rawRecordTestSubscription(json.RawMessage(`{"descriptor":{"interface":"Protocols","method":"Configure"}}`), "data"), + want: errUnsupportedRawMeshRecordEvent, + }, + { + name: "unsupported records method", + msg: rawRecordTestSubscription(json.RawMessage(`{"descriptor":{"interface":"Records","method":"Read"}}`), "data"), + want: errUnsupportedRawMeshRecordEvent, + }, + { + name: "delete missing descriptor record ID", + msg: rawRecordTestSubscription(json.RawMessage(`{"recordId":"wrong-location","descriptor":{"interface":"Records","method":"Delete","messageTimestamp":"2026-07-11T12:00:00Z"}}`), ""), + want: errMalformedRawMeshRecord, + }, + { + name: "write missing record ID", + msg: rawRecordTestSubscription(json.RawMessage(`{"descriptor":{"interface":"Records","method":"Write","protocol":"https://example.com/protocol","protocolPath":"network/node","messageTimestamp":"2026-07-11T12:00:00Z"}}`), "data"), + want: errMalformedRawMeshRecord, + }, + { + name: "invalid timestamp", + msg: rawRecordTestSubscription(rawRecordTestWrite(t, "record-a", "network/node", "not-time", "data"), ""), + want: errMalformedRawMeshRecord, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + changed, err := set.applySubscriptionMessage(test.msg, "") + if changed || !errors.Is(err, test.want) { + t.Fatalf("changed=%v err=%v, want errors.Is(..., %v)", changed, err, test.want) + } + }) + } +} + +func TestNewRawMeshRecordSetRejectsMalformedEntries(t *testing.T) { + tests := []json.RawMessage{ + json.RawMessage(`not-json`), + json.RawMessage(`{"descriptor":{"interface":"Records","method":"Write","protocol":"https://example.com/protocol","protocolPath":"network/node","messageTimestamp":"2026-07-11T12:00:00Z"}}`), + rawRecordTestWrite(t, "record-a", "", "2026-07-11T12:00:00Z", "data"), + } + for i, entry := range tests { + _, err := newRawMeshRecordSet([]json.RawMessage{entry}, "") + if !errors.Is(err, errMalformedRawMeshRecord) { + t.Fatalf("case %d: err=%v", i, err) + } + } +} + +func TestRawMeshRecordSetConcurrentCloneReadAndApply(t *testing.T) { + set, err := newRawMeshRecordSet([]json.RawMessage{ + rawRecordTestWrite(t, "record-a", "network/node", "2026-07-11T12:00:00Z", "initial"), + }, "") + if err != nil { + t.Fatal(err) + } + + const iterations = 50 + var wg sync.WaitGroup + for worker := 0; worker < 4; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + _, _ = set.get("record-a") + _ = set.all() + _ = set.clone() + } + }() + } + for worker := 0; worker < 2; worker++ { + worker := worker + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + timestamp := time.Date(2026, 7, 11, 12, 1, worker*iterations+i, 0, time.UTC).Format(time.RFC3339Nano) + raw := rawRecordTestWrite(t, "record-a", "network/node", timestamp, fmt.Sprintf("%d-%d", worker, i)) + _, _ = set.applySubscriptionMessage(rawRecordTestSubscription(raw, ""), "") + } + }() + } + wg.Wait() + if set.len() != 1 { + t.Fatalf("len after concurrent updates = %d", set.len()) + } +} + +func rawRecordTestWrite(t *testing.T, recordID, protocolPath, timestamp, encodedData string) json.RawMessage { + t.Helper() + parentContext := "parent-" + recordID + message := map[string]any{ + "recordId": recordID, + "contextId": parentContext + "/" + recordID, + "descriptor": map[string]any{ + "interface": "Records", + "method": "Write", + "protocol": rawRecordTestProtocol, + "protocolPath": protocolPath, + "parentId": parentContext, + "recipient": "did:example:" + recordID, + "dateCreated": timestamp, + "messageTimestamp": timestamp, + "dataCid": "data-" + encodedData, + }, + } + if encodedData != "" { + message["encodedData"] = encodedData + } + return rawRecordTestJSON(t, message) +} + +func rawRecordTestWrappedWrite(t *testing.T, recordID, protocolPath, timestamp, encodedData string) json.RawMessage { + t.Helper() + var write any + if err := json.Unmarshal(rawRecordTestWrite(t, recordID, protocolPath, timestamp, encodedData), &write); err != nil { + t.Fatal(err) + } + return rawRecordTestJSON(t, map[string]any{"recordsWrite": write}) +} + +func rawRecordTestDelete(t *testing.T, recordID, timestamp string) json.RawMessage { + t.Helper() + return rawRecordTestJSON(t, map[string]any{ + "descriptor": map[string]any{ + "interface": "Records", + "method": "Delete", + "recordId": recordID, + "messageTimestamp": timestamp, + }, + }) +} + +func rawRecordTestSubscription(raw json.RawMessage, encodedData string) *dwn.SubscriptionMessage { + latest := true + return &dwn.SubscriptionMessage{ + Type: dwn.SubscriptionEventType, + IsLatestBaseState: &latest, + EncodedData: encodedData, + Event: &dwn.RecordEvent{Message: raw}, + } +} + +func rawRecordEncodedData(t *testing.T, raw json.RawMessage) string { + t.Helper() + message, _, err := unwrapRawRecordMessage(raw, "recordsWrite") + if err != nil { + t.Fatalf("unwrap record: %v", err) + } + if message.EncodedData == nil { + return "" + } + var value string + if err := json.Unmarshal(*message.EncodedData, &value); err != nil { + t.Fatalf("decode encodedData: %v", err) + } + return value +} + +func rawRecordTestJSON(t *testing.T, value any) json.RawMessage { + t.Helper() + raw, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return raw +} + +func TestRawMeshRecordSetCanonicalCIDAndSubscriptionAgreement(t *testing.T) { + raw := rawRecordTestWrite(t, "record-cid", "network/node", "2026-07-11T12:00:00Z", "data") + cid, err := computeRawRecordMessageCID(raw) + if err != nil { + t.Fatal(err) + } + event := rawRecordTestSubscription(raw, "") + event.MessageCID = cid + event.Cursor = &dwn.ProgressToken{StreamID: "stream", Epoch: "epoch", Position: "1", MessageCID: cid} + set, _ := newRawMeshRecordSet(nil, "") + if changed, err := set.applySubscriptionMessage(event, ""); err != nil || !changed { + t.Fatalf("matching CID changed=%v err=%v", changed, err) + } + + mismatch := rawRecordTestSubscription(rawRecordTestWrite(t, "other", "network/node", "2026-07-11T12:00:01Z", "data"), "") + mismatch.MessageCID = "bafy-wrong" + mismatch.Cursor = &dwn.ProgressToken{StreamID: "stream", Epoch: "epoch", Position: "2", MessageCID: "bafy-other"} + if changed, err := set.applySubscriptionMessage(mismatch, ""); changed || !errors.Is(err, errMalformedRawMeshRecord) { + t.Fatalf("mismatched CIDs changed=%v err=%v", changed, err) + } + + agreedWrong := rawRecordTestSubscription(rawRecordTestWrite(t, "agreed-wrong", "network/node", "2026-07-11T12:00:02Z", "data"), "") + agreedWrong.MessageCID = "bafy-agreed-but-wrong" + agreedWrong.Cursor = &dwn.ProgressToken{StreamID: "stream", Epoch: "epoch", Position: "3", MessageCID: agreedWrong.MessageCID} + if changed, err := set.applySubscriptionMessage(agreedWrong, ""); changed || !errors.Is(err, errMalformedRawMeshRecord) { + t.Fatalf("agreed noncanonical CID changed=%v err=%v", changed, err) + } +} + +func TestComputeRawRecordMessageCIDCanonicalBoundary(t *testing.T) { + var canonical map[string]any + if err := json.Unmarshal(rawRecordTestWrite(t, "record-boundary", "network/node", "2026-07-11T12:00:00Z", "payload"), &canonical); err != nil { + t.Fatal(err) + } + canonical["authorization"] = map[string]any{"signature": "authorization-a"} + canonical["encryption"] = map[string]any{"algorithm": "encryption-a"} + canonical["attestation"] = map[string]any{"signature": "attestation-a"} + base := rawRecordTestJSON(t, canonical) + baseCID, err := computeRawRecordMessageCID(base) + if err != nil { + t.Fatal(err) + } + + decorated := mapsCloneForRawRecordTest(t, canonical) + decorated["encodedData"] = "different-reply-data" + decorated["initialWrite"] = map[string]any{"descriptor": map[string]any{"messageTimestamp": "1999-01-01T00:00:00Z"}} + decorated["messageCid"] = "bafy-reply-decoration" + decoratedCID, err := computeRawRecordMessageCID(rawRecordTestJSON(t, decorated)) + if err != nil { + t.Fatal(err) + } + if decoratedCID != baseCID { + t.Fatalf("reply decorations changed canonical CID: got %q, want %q", decoratedCID, baseCID) + } + + for _, field := range []string{"authorization", "encryption", "attestation"} { + t.Run(field, func(t *testing.T) { + changed := mapsCloneForRawRecordTest(t, canonical) + changed[field] = map[string]any{"changed": field + "-b"} + changedCID, err := computeRawRecordMessageCID(rawRecordTestJSON(t, changed)) + if err != nil { + t.Fatal(err) + } + if changedCID == baseCID { + t.Fatalf("changing canonical %s did not change CID %q", field, baseCID) + } + }) + } +} + +func mapsCloneForRawRecordTest(t *testing.T, value map[string]any) map[string]any { + t.Helper() + var cloned map[string]any + if err := json.Unmarshal(rawRecordTestJSON(t, value), &cloned); err != nil { + t.Fatal(err) + } + return cloned +} + +func TestRawMeshRecordSetEqualTimestampUsesCanonicalCID(t *testing.T) { + a := rawRecordTestWrite(t, "record-tie", "network/node", "2026-07-11T12:00:00Z", "alpha") + b := rawRecordTestWrite(t, "record-tie", "network/node", "2026-07-11T12:00:00Z", "beta") + aCID, _ := computeRawRecordMessageCID(a) + bCID, _ := computeRawRecordMessageCID(b) + lowRaw, highRaw := a, b + lowCID, highCID := aCID, bCID + if lowCID > highCID { + lowRaw, highRaw = highRaw, lowRaw + lowCID, highCID = highCID, lowCID + } + if lowCID == highCID { + t.Fatal("fixture did not produce distinct canonical CIDs") + } + + set, err := newRawMeshRecordSet([]json.RawMessage{lowRaw}, "") + if err != nil { + t.Fatal(err) + } + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(highRaw, ""), ""); err != nil || !changed { + t.Fatalf("higher CID changed=%v err=%v", changed, err) + } + if got, _ := set.get("record-tie"); got.messageCID != highCID { + t.Fatalf("winning CID = %q, want %q", got.messageCID, highCID) + } + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(lowRaw, ""), ""); err != nil || changed { + t.Fatalf("lower CID replay changed=%v err=%v", changed, err) + } +} + +func TestRawMeshRecordSetDeleteToPruneUsesCanonicalBaseStateOrder(t *testing.T) { + set, err := newRawMeshRecordSet([]json.RawMessage{ + rawRecordTestWrite(t, "record-delete", "network/node", "2026-07-11T12:00:00Z", "visible"), + }, "") + if err != nil { + t.Fatal(err) + } + const tieTimestamp = "2026-07-11T12:00:01Z" + plain := rawRecordTestDeleteWith(t, "record-delete", tieTimestamp, false, "plain") + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(plain, ""), ""); err != nil || !changed { + t.Fatalf("plain delete changed=%v err=%v", changed, err) + } + plainCID, err := computeRawRecordMessageCID(plain) + if err != nil { + t.Fatal(err) + } + + tiedPrune := rawRecordTestDeleteWith(t, "record-delete", tieTimestamp, true, "prune-tie") + tiedPruneCID, err := computeRawRecordMessageCID(tiedPrune) + if err != nil { + t.Fatal(err) + } + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(tiedPrune, ""), ""); err != nil || changed { + t.Fatalf("tied prune visible change=%v err=%v", changed, err) + } + head := set.heads["record-delete"] + if wantPrune := tiedPruneCID > plainCID; head.prune != wantPrune { + t.Fatalf("equal-time prune=%v, want %v from CID order plain=%q prune=%q", head.prune, wantPrune, plainCID, tiedPruneCID) + } + if !head.prune { + newerPrune := rawRecordTestDeleteWith(t, "record-delete", "2026-07-11T12:00:02Z", true, "prune-newer") + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(newerPrune, ""), ""); err != nil || changed { + t.Fatalf("newer prune visible change=%v err=%v", changed, err) + } + head = set.heads["record-delete"] + if !head.prune { + t.Fatal("strictly newer prune did not replace plain tombstone") + } + } + + terminal := head + laterPrune := rawRecordTestDeleteWith(t, "record-delete", "2026-07-11T13:00:00Z", true, "prune-terminal") + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(laterPrune, ""), ""); err != nil || changed { + t.Fatalf("terminal prune visible change=%v err=%v", changed, err) + } + if got := set.heads["record-delete"]; got.messageCID != terminal.messageCID || !got.prune { + t.Fatalf("terminal prune was replaced: got=%#v want=%#v", got, terminal) + } +} + +func TestRawMeshRecordSetSquashSnapshotFloorAndContextIsolation(t *testing.T) { + const timestamp = "2026-07-11T12:01:00Z" + old := rawRecordTestWriteAt(t, "old", "network/node/endpoint", "network/node-a", "2026-07-11T12:00:00Z", "old", false) + equal := rawRecordTestWriteAt(t, "equal", "network/node/endpoint", "network/node-a", timestamp, "equal", false) + squash := rawRecordTestWriteAt(t, "squash", "network/node/endpoint", "network/node-a", timestamp, "new", true) + isolated := rawRecordTestWriteAt(t, "isolated", "network/node/endpoint", "network/node-b", "2026-07-11T11:00:00Z", "isolated", false) + + // Query snapshot ingest must retain equal-time siblings regardless of CID/order. + set, err := newRawMeshRecordSet([]json.RawMessage{squash, old, isolated, equal}, "") + if err != nil { + t.Fatal(err) + } + if _, ok := set.get("old"); ok { + t.Fatal("squash retained strictly older sibling") + } + for _, id := range []string{"equal", "squash", "isolated"} { + if _, ok := set.get(id); !ok { + t.Fatalf("snapshot lost retained record %q", id) + } + } + + delayedEqual := rawRecordTestWriteAt(t, "delayed", "network/node/endpoint", "network/node-a", timestamp, "delayed", false) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(delayedEqual, ""), ""); err != nil || changed { + t.Fatalf("floor-equal delayed write changed=%v err=%v", changed, err) + } + newer := rawRecordTestWriteAt(t, "newer", "network/node/endpoint", "network/node-a", "2026-07-11T12:02:00Z", "newer", false) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(newer, ""), ""); err != nil || !changed { + t.Fatalf("above-floor write changed=%v err=%v", changed, err) + } +} + +func TestRawMeshRecordSetSquashDoesNotDiscardDeleteTombstone(t *testing.T) { + const parent = "network/node-a" + set, err := newRawMeshRecordSet([]json.RawMessage{ + rawRecordTestWriteAt(t, "deleted-sibling", "network/node/endpoint", parent, "2026-07-11T12:00:00Z", "old", false), + }, "") + if err != nil { + t.Fatal(err) + } + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription( + rawRecordTestDelete(t, "deleted-sibling", "2026-07-11T12:01:00Z"), "", + ), ""); err != nil || !changed { + t.Fatalf("delete sibling changed=%v err=%v", changed, err) + } + squash := rawRecordTestWriteAt(t, "replacement", "network/node/endpoint", parent, "2026-07-11T12:02:00Z", "new", true) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(squash, ""), ""); err != nil || !changed { + t.Fatalf("squash changed=%v err=%v", changed, err) + } + if head, ok := set.heads["deleted-sibling"]; !ok || head.method != rawMeshRecordDelete { + t.Fatalf("squash discarded delete tombstone: %#v, present=%v", head, ok) + } + resurrection := rawRecordTestWriteAt(t, "deleted-sibling", "network/node/endpoint", parent, "2026-07-11T12:03:00Z", "", false) + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(resurrection, ""), ""); err != nil || changed { + t.Fatalf("post-squash resurrection changed=%v err=%v", changed, err) + } +} + +func TestRawMeshRecordSetStaleDataLessWriteNeedsNoHydration(t *testing.T) { + set, err := newRawMeshRecordSet([]json.RawMessage{ + rawRecordTestWrite(t, "record-no-read", "network/node", "2026-07-11T12:02:00Z", "current"), + }, "") + if err != nil { + t.Fatal(err) + } + stale := rawRecordTestWrite(t, "record-no-read", "network/node", "2026-07-11T12:01:00Z", "") + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(stale, ""), ""); err != nil || changed { + t.Fatalf("stale data-less write changed=%v err=%v", changed, err) + } + + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription( + rawRecordTestDelete(t, "record-no-read", "2026-07-11T12:03:00Z"), "", + ), ""); err != nil || !changed { + t.Fatalf("newer delete changed=%v err=%v", changed, err) + } + futureButForbidden := rawRecordTestWrite(t, "record-no-read", "network/node", "2026-07-11T13:00:00Z", "") + if changed, err := set.applySubscriptionMessage(rawRecordTestSubscription(futureButForbidden, ""), ""); err != nil || changed { + t.Fatalf("tombstoned data-less write changed=%v err=%v", changed, err) + } +} + +func rawRecordTestWriteAt(t *testing.T, recordID, protocolPath, parentContext, timestamp, encodedData string, squash bool) json.RawMessage { + t.Helper() + var message map[string]any + if err := json.Unmarshal(rawRecordTestWrite(t, recordID, protocolPath, timestamp, encodedData), &message); err != nil { + t.Fatal(err) + } + message["contextId"] = parentContext + "/" + recordID + descriptor := message["descriptor"].(map[string]any) + segments := strings.Split(parentContext, "/") + descriptor["parentId"] = segments[len(segments)-1] + if squash { + descriptor["squash"] = true + } + return rawRecordTestJSON(t, message) +} + +func rawRecordTestDeleteWith(t *testing.T, recordID, timestamp string, prune bool, permissionGrantID string) json.RawMessage { + t.Helper() + var message map[string]any + if err := json.Unmarshal(rawRecordTestDelete(t, recordID, timestamp), &message); err != nil { + t.Fatal(err) + } + descriptor := message["descriptor"].(map[string]any) + if prune { + descriptor["prune"] = true + } + if permissionGrantID != "" { + descriptor["permissionGrantId"] = permissionGrantID + } + return rawRecordTestJSON(t, message) +} diff --git a/internal/control/state_transaction_test.go b/internal/control/state_transaction_test.go index ad10570..505f2eb 100644 --- a/internal/control/state_transaction_test.go +++ b/internal/control/state_transaction_test.go @@ -5,9 +5,6 @@ import ( "encoding/base64" "encoding/json" "errors" - "net/http" - "net/http/httptest" - "sync/atomic" "testing" "time" @@ -15,79 +12,18 @@ import ( ) func TestLoadStateRollsBackCachedStateOnReplyRateLimit(t *testing.T) { - identity, signer, _, _ := sealedTestOwner(t) + network, entries := rawCaptureLoadFixture(t) + control := &rawCaptureServerControl{ + queryRateLimitPath: "network/node/nodeInfo", + queryRateLimitDetail: "RateLimitExceeded: tenant rate limit exceeded, retry after 6s", + } + client := newRawCaptureLoadClient(t, network, entries, control) oldNetwork := &NetworkConfig{Name: "old-network", MeshCIDR: "10.200.0.0/16"} - oldNode := &NodeRecord{DID: identity.URI, MeshIP: "10.200.0.2", Label: "old-node", RecordID: "old-node-record"} + oldNode := &NodeRecord{DID: materializerSelfDID, MeshIP: "10.200.0.2", Label: "old-node", RecordID: "old-node-record"} oldMember := &MemberRecord{DID: "did:example:old-member", Label: "old-member", RecordID: "old-member-record"} oldRelay := &RelayData{URL: "https://old-relay.example", Region: "old-region"} oldACL := &ACLPolicyData{Version: 1, DefaultAction: "deny"} - - newNetworkEntry := stateTransactionEntry(t, "", "", NetworkConfig{ - Name: "new-network", - MeshCIDR: "10.201.0.0/16", - }) - newNodeEntry := stateTransactionEntry(t, "new-node-record", identity.URI, NodeRecord{ - MeshIP: "10.201.0.2", - Label: "new-node", - }) - newMemberEntry := stateTransactionEntry(t, "new-member-record", "did:example:new-member", MemberRecord{ - Label: "new-member", - AddedAt: "2026-07-11T00:00:00Z", - }) - newRelayEntry := stateTransactionEntry(t, "", "", RelayData{ - URL: "https://new-relay.example", - Region: "new-region", - }) - newACLEntry := stateTransactionEntry(t, "", "", ACLPolicyData{ - Version: 2, - DefaultAction: "accept", - }) - - var requestCount atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var request dwn.JsonRpcRequest - if err := json.Unmarshal([]byte(r.Header.Get("dwn-request")), &request); err != nil { - t.Errorf("decode DWN request: %v", err) - http.Error(w, "bad request", http.StatusBadRequest) - return - } - - reply := &dwn.DwnReply{Status: dwn.Status{Code: http.StatusOK, Detail: "OK"}} - switch requestCount.Add(1) { - case 1: // Network record read. - reply.Entry = newNetworkEntry - case 2: // Owner node query. - reply.Entries = stateTransactionEntries(t, newNodeEntry) - case 3: // Member query. - reply.Entries = stateTransactionEntries(t, newMemberEntry) - case 4: // Member-associated node query. - reply.Entries = stateTransactionEntries(t) - case 5: // Relay query. - reply.Entries = stateTransactionEntries(t, newRelayEntry) - case 6: // ACL policy query. - reply.Entries = stateTransactionEntries(t, newACLEntry) - case 7: // First node child query, after every cached state field changed. - reply.Status = dwn.Status{ - Code: http.StatusTooManyRequests, - Detail: "RateLimitExceeded: tenant rate limit exceeded, retry after 6s", - } - default: - t.Errorf("unexpected DWN request %d", requestCount.Load()) - reply.Status = dwn.Status{Code: http.StatusInternalServerError, Detail: "unexpected request"} - } - - if err := json.NewEncoder(w).Encode(dwn.JsonRpcResponse{ - JSONRPC: "2.0", - ID: request.ID, - Result: &dwn.JsonRpcResult{Reply: reply}, - }); err != nil { - t.Errorf("encode DWN response: %v", err) - } - })) - defer server.Close() - - client := NewDWNClient(server.URL, identity.URI, "network-record", identity.URI, signer) client.network = oldNetwork client.nodes = map[string]*NodeRecord{oldNode.DID: oldNode} client.members = map[string]*MemberRecord{oldMember.DID: oldMember} @@ -99,32 +35,17 @@ func TestLoadStateRollsBackCachedStateOnReplyRateLimit(t *testing.T) { t.Fatalf("LoadState error = %v, want ErrRateLimited", err) } var rateErr *dwn.RateLimitError - if !errors.As(err, &rateErr) { - t.Fatalf("LoadState error = %T %v, want *dwn.RateLimitError", err, err) - } - if rateErr.RetryAfter != 6*time.Second { - t.Fatalf("retry delay = %v, want 6s", rateErr.RetryAfter) - } - if got := requestCount.Load(); got != 7 { - t.Fatalf("DWN request count = %d, want 7", got) + if !errors.As(err, &rateErr) || rateErr.RetryAfter != 6*time.Second { + t.Fatalf("LoadState error = %#v, want 6s retry", err) } client.mu.RLock() defer client.mu.RUnlock() - if client.network != oldNetwork { - t.Errorf("network = %#v, want original cached pointer %#v", client.network, oldNetwork) - } - if len(client.nodes) != 1 || client.nodes[oldNode.DID] != oldNode { - t.Errorf("nodes = %#v, want only original cached node %#v", client.nodes, oldNode) - } - if len(client.members) != 1 || client.members[oldMember.DID] != oldMember { - t.Errorf("members = %#v, want only original cached member %#v", client.members, oldMember) - } - if len(client.relays) != 1 || client.relays[0] != oldRelay { - t.Errorf("relays = %#v, want only original cached relay %#v", client.relays, oldRelay) - } - if client.acl != oldACL { - t.Errorf("ACL = %#v, want original cached pointer %#v", client.acl, oldACL) + if client.network != oldNetwork || len(client.nodes) != 1 || client.nodes[oldNode.DID] != oldNode || + len(client.members) != 1 || client.members[oldMember.DID] != oldMember || + len(client.relays) != 1 || client.relays[0] != oldRelay || client.acl != oldACL { + t.Fatalf("rate-limited full load changed cached state: network=%#v nodes=%#v members=%#v relays=%#v ACL=%#v", + client.network, client.nodes, client.members, client.relays, client.acl) } } diff --git a/internal/dwn/subscribe.go b/internal/dwn/subscribe.go index 3bc714d..3266ca2 100644 --- a/internal/dwn/subscribe.go +++ b/internal/dwn/subscribe.go @@ -19,6 +19,12 @@ const ( SubscriptionEventType = "event" SubscriptionEOSEType = "eose" SubscriptionErrorType = "error" + + // subscriptionMaxMessageBytes bounds every inbound WebSocket message. It + // leaves headroom above the topology queue's encoded event budget while + // preventing a peer from forcing an unbounded allocation before JSON + // validation can run. + subscriptionMaxMessageBytes int64 = 16 << 20 ) // @@ -491,6 +497,7 @@ func (s *Subscription) connect(ctx context.Context, endpoint string) (bool, erro if err != nil { return false, fmt.Errorf("dialing websocket: %w", err) } + conn.SetReadLimit(subscriptionMaxMessageBytes) s.mu.Lock() s.conn = conn diff --git a/internal/dwn/subscribe_read_limit_test.go b/internal/dwn/subscribe_read_limit_test.go new file mode 100644 index 0000000..a4430ce --- /dev/null +++ b/internal/dwn/subscribe_read_limit_test.go @@ -0,0 +1,218 @@ +package dwn + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" +) + +func TestSubscriptionOversizedMessageReconnectsWithoutHandlingOrAck(t *testing.T) { + var connections atomic.Int32 + serverErrors := make(chan error, 8) + oversizedRejected := make(chan struct{}) + validAcked := make(chan struct{}) + serverFinished := make(chan struct{}) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Accept(w, r, nil) + if err != nil { + serverErrors <- fmt.Errorf("accept: %w", err) + return + } + defer conn.CloseNow() + + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + request, err := readWSRequest(ctx, conn) + if err != nil { + serverErrors <- err + return + } + + switch connection := connections.Add(1); connection { + case 1: + if err := writeSubscribeReply(ctx, conn, request.ID, "oversized-frame-subscription"); err != nil { + serverErrors <- err + return + } + token := ProgressToken{StreamID: "stream", Epoch: "epoch", Position: "1"} + message := &SubscriptionMessage{ + Type: SubscriptionEventType, + Cursor: &token, + EncodedData: strings.Repeat("A", int(subscriptionMaxMessageBytes)), + Event: &RecordEvent{ + Message: json.RawMessage("{\"recordId\":\"oversized-must-not-be-handled\"}"), + }, + } + if err := writeSubscriptionMessage(ctx, conn, request.Subscription.ID, message); err != nil { + serverErrors <- err + return + } + + // The read limit closes the connection at the WebSocket layer. An + // application rpc.ack must never be emitted for the rejected frame. + _, _, err = conn.Read(ctx) + if got := websocket.CloseStatus(err); got != websocket.StatusMessageTooBig { + serverErrors <- fmt.Errorf("close status after oversized frame = %d (%v), want %d", got, err, websocket.StatusMessageTooBig) + return + } + close(oversizedRejected) + case 2: + defer close(serverFinished) + if _, ok := request.Params.Message.Descriptor["cursor"]; ok { + serverErrors <- fmt.Errorf("reconnect after unhandled frame unexpectedly carried a cursor") + return + } + if err := writeSubscribeReply(ctx, conn, request.ID, "valid-frame-subscription"); err != nil { + serverErrors <- err + return + } + + validToken := ProgressToken{StreamID: "stream", Epoch: "epoch", Position: "1"} + valid := &SubscriptionMessage{ + Type: SubscriptionEventType, + Cursor: &validToken, + Event: &RecordEvent{ + Message: json.RawMessage("{\"recordId\":\"valid-after-reconnect\"}"), + }, + } + if err := writeSubscriptionMessage(ctx, conn, request.Subscription.ID, valid); err != nil { + serverErrors <- err + return + } + if err := expectAck(ctx, conn, request.Subscription.ID, validToken); err != nil { + serverErrors <- err + return + } + close(validAcked) + + terminalToken := ProgressToken{StreamID: "stream", Epoch: "epoch", Position: "2"} + terminal := &SubscriptionMessage{ + Type: SubscriptionErrorType, + Cursor: &terminalToken, + Error: &SubscriptionError{Code: "Closed", Detail: "test complete"}, + } + if err := writeSubscriptionMessage(ctx, conn, request.Subscription.ID, terminal); err != nil { + serverErrors <- err + return + } + if err := expectAck(ctx, conn, request.Subscription.ID, terminalToken); err != nil { + serverErrors <- err + } + default: + serverErrors <- fmt.Errorf("unexpected connection %d", connection) + } + })) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + lifecycle := make(chan SubscriptionLifecycleEvent, 8) + handled := make(chan SubscriptionMessage, 4) + manager := NewSubscriptionManager(server.URL, slog.Default()) + sub, err := manager.SubscribeWithAuthAndLifecycle( + ctx, + "did:dht:target", + newTestSigner(t), + RecordsFilter{Protocol: "https://example.com/protocol"}, + MessageAuth{}, + func(message *SubscriptionMessage) error { + copyMessage := *message + if message.Event != nil { + copyEvent := *message.Event + copyEvent.Message = append(json.RawMessage(nil), message.Event.Message...) + copyMessage.Event = ©Event + } + handled <- copyMessage + return nil + }, + func(event SubscriptionLifecycleEvent) { + lifecycle <- event + }, + ) + if err != nil { + cancel() + t.Fatalf("SubscribeWithAuthAndLifecycle: %v", err) + } + defer func() { + cancel() + sub.Close() + }() + + var gotLifecycle []SubscriptionLifecycleKind + for { + select { + case event := <-lifecycle: + gotLifecycle = append(gotLifecycle, event.Kind) + if event.Kind == SubscriptionLifecycleRetrying && !errors.Is(event.Err, websocket.ErrMessageTooBig) { + t.Fatalf("retry error = %v, want websocket.ErrMessageTooBig", event.Err) + } + if event.Kind == SubscriptionLifecycleTerminal { + goto terminal + } + case <-ctx.Done(): + t.Fatalf("waiting for reconnect lifecycle: %v; lifecycle=%v", ctx.Err(), gotLifecycle) + } + } + +terminal: + wantLifecycle := []SubscriptionLifecycleKind{ + SubscriptionLifecycleEstablished, + SubscriptionLifecycleRetrying, + SubscriptionLifecycleEstablished, + SubscriptionLifecycleTerminal, + } + if fmt.Sprint(gotLifecycle) != fmt.Sprint(wantLifecycle) { + t.Fatalf("lifecycle = %v, want %v", gotLifecycle, wantLifecycle) + } + + select { + case <-oversizedRejected: + case <-ctx.Done(): + t.Fatalf("waiting for oversized rejection: %v", ctx.Err()) + } + select { + case <-validAcked: + case <-ctx.Done(): + t.Fatalf("waiting for valid acknowledgement: %v", ctx.Err()) + } + select { + case <-serverFinished: + case <-ctx.Done(): + t.Fatalf("waiting for server completion: %v", ctx.Err()) + } + + gotHandled := make([]SubscriptionMessage, 0, 2) + for len(gotHandled) < 2 { + select { + case message := <-handled: + gotHandled = append(gotHandled, message) + case <-ctx.Done(): + t.Fatalf("waiting for handled messages: %v", ctx.Err()) + } + } + if len(handled) != 0 { + t.Fatalf("unexpected extra handled message after oversized frame") + } + if gotHandled[0].Type != SubscriptionEventType || + gotHandled[0].Event == nil || + !strings.Contains(string(gotHandled[0].Event.Message), "valid-after-reconnect") { + t.Fatalf("first handled message = %+v, want valid event from reconnect", gotHandled[0]) + } + if gotHandled[1].Type != SubscriptionErrorType { + t.Fatalf("second handled message type = %q, want terminal error", gotHandled[1].Type) + } + if got := connections.Load(); got != 2 { + t.Fatalf("connections = %d, want one reconnect", got) + } + drainServerErrors(t, serverErrors) +} diff --git a/internal/dwn/transport.go b/internal/dwn/transport.go index 366e8d8..bd37d9f 100644 --- a/internal/dwn/transport.go +++ b/internal/dwn/transport.go @@ -15,6 +15,7 @@ import ( "errors" "fmt" "io" + "math" "net/http" "strconv" "strings" @@ -29,6 +30,17 @@ var ( ErrRateLimited = errors.New("rate limited") ) +const ( + // maxHTTPResponseBodyBytes accommodates large encrypted RecordsRead data + // and paginated JSON envelopes, including their wire-format overhead, while + // keeping every response allocation within a fixed production bound. + maxHTTPResponseBodyBytes int64 = 128 << 20 + + // maxHTTPResponseErrorPreviewBytes keeps malformed or rate-limit responses + // from being copied wholesale into errors and logs. + maxHTTPResponseErrorPreviewBytes = 4 << 10 +) + // RateLimitError reports a DWN rate-limit response and carries the delay the // server asked the caller to observe before retrying. // @@ -224,8 +236,9 @@ func WithTransportHTTPClient(c *http.Client) HTTPTransportOption { // - For RecordsRead responses with data: JSON-RPC response is in // "dwn-response" header and binary data is the HTTP body type HTTPTransport struct { - endpoint string - httpClient *http.Client + endpoint string + httpClient *http.Client + maxResponseBodyBytes int64 } // NewHTTPTransport creates a new HTTP transport for the given DWN endpoint. @@ -238,8 +251,9 @@ func NewHTTPTransport(endpoint string, opts ...HTTPTransportOption) *HTTPTranspo } return &HTTPTransport{ - endpoint: endpoint, - httpClient: options.httpClient, + endpoint: endpoint, + httpClient: options.httpClient, + maxResponseBodyBytes: maxHTTPResponseBodyBytes, } } @@ -300,6 +314,10 @@ func (t *HTTPTransport) Send(ctx context.Context, target string, msg *Message, d // - dwn-response header present: JSON-RPC in header, binary data in body // - dwn-response header absent: JSON-RPC in body func (t *HTTPTransport) parseResponse(resp *http.Response) (*SendResult, error) { + maxBodyBytes := t.maxResponseBodyBytes + if maxBodyBytes == 0 { + maxBodyBytes = maxHTTPResponseBodyBytes + } dwnResponseHeader := resp.Header.Get("dwn-response") if dwnResponseHeader != "" { @@ -320,9 +338,9 @@ func (t *HTTPTransport) parseResponse(resp *http.Response) (*SendResult, error) } // Read the binary data from the body. - data, err := io.ReadAll(resp.Body) + data, err := readBoundedResponseBody(resp.Body, maxBodyBytes) if err != nil { - return nil, fmt.Errorf("%w: reading response data: %v", ErrTransport, err) + return nil, fmt.Errorf("reading response data: %w", err) } result := &SendResult{Data: data} @@ -333,9 +351,9 @@ func (t *HTTPTransport) parseResponse(resp *http.Response) (*SendResult, error) } // Standard response: JSON-RPC in body. - respBody, err := io.ReadAll(resp.Body) + respBody, err := readBoundedResponseBody(resp.Body, maxBodyBytes) if err != nil { - return nil, fmt.Errorf("%w: reading response body: %v", ErrTransport, err) + return nil, fmt.Errorf("reading response body: %w", err) } var rpcResp JsonRpcResponse @@ -349,15 +367,15 @@ func (t *HTTPTransport) parseResponse(resp *http.Response) (*SendResult, error) rpcErr = rpcResp.Error } detail := "HTTP 429" - if rpcErr == nil && len(strings.TrimSpace(string(respBody))) > 0 { - detail += ": " + strings.TrimSpace(string(respBody)) + if rpcErr == nil && len(bytes.TrimSpace(respBody)) > 0 { + detail += ": " + strings.TrimSpace(responseBodyErrorPreview(respBody)) } return nil, newRateLimitError(resp, rpcErr, detail) } if parseErr != nil { return nil, fmt.Errorf("%w: parsing response body (HTTP %d): %s: %v", - ErrTransport, resp.StatusCode, string(respBody), parseErr) + ErrTransport, resp.StatusCode, responseBodyErrorPreview(respBody), parseErr) } if rpcResp.Error != nil { @@ -374,6 +392,31 @@ func (t *HTTPTransport) parseResponse(resp *http.Response) (*SendResult, error) return result, nil } +// readBoundedResponseBody reads at most limit+1 bytes so it can distinguish an +// exact-boundary response from an oversized one without buffering the rest of +// the stream. Errors never return a partial body for callers to decode. +func readBoundedResponseBody(reader io.Reader, limit int64) ([]byte, error) { + if limit < 0 || limit == math.MaxInt64 { + return nil, fmt.Errorf("%w: invalid response body limit %d", ErrTransport, limit) + } + + body, err := io.ReadAll(io.LimitReader(reader, limit+1)) + if err != nil { + return nil, fmt.Errorf("%w: reading bounded response body: %w", ErrTransport, err) + } + if int64(len(body)) > limit { + return nil, fmt.Errorf("%w: response body exceeds %d-byte limit", ErrTransport, limit) + } + return body, nil +} + +func responseBodyErrorPreview(body []byte) string { + if len(body) <= maxHTTPResponseErrorPreviewBytes { + return string(body) + } + return fmt.Sprintf("%s... (%d bytes total)", body[:maxHTTPResponseErrorPreviewBytes], len(body)) +} + func newRateLimitError(resp *http.Response, rpcErr *JsonRpcError, fallbackDetail string) *RateLimitError { retryAfter, ok := parseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) if !ok && rpcErr != nil { diff --git a/internal/dwn/transport_response_limit_test.go b/internal/dwn/transport_response_limit_test.go new file mode 100644 index 0000000..2da0933 --- /dev/null +++ b/internal/dwn/transport_response_limit_test.go @@ -0,0 +1,167 @@ +package dwn + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "net/http/httptest" + "strings" + "testing" + "testing/iotest" +) + +func TestReadBoundedResponseBody(t *testing.T) { + t.Run("exact boundary", func(t *testing.T) { + body, err := readBoundedResponseBody(strings.NewReader("12345"), 5) + if err != nil { + t.Fatalf("readBoundedResponseBody: %v", err) + } + if got := string(body); got != "12345" { + t.Fatalf("body = %q, want exact boundary content", got) + } + }) + + t.Run("one byte over", func(t *testing.T) { + body, err := readBoundedResponseBody(strings.NewReader("123456"), 5) + if body != nil { + t.Fatalf("body = %q, want nil on overflow", body) + } + if !errors.Is(err, ErrTransport) { + t.Fatalf("error = %v, want ErrTransport", err) + } + if err == nil || !strings.Contains(err.Error(), "exceeds 5-byte limit") { + t.Fatalf("error = %v, want response-size detail", err) + } + }) + + t.Run("reader error discards partial data", func(t *testing.T) { + readFailure := errors.New("test reader failed") + reader := io.MultiReader( + strings.NewReader("partial"), + iotest.ErrReader(readFailure), + ) + body, err := readBoundedResponseBody(reader, 32) + if body != nil { + t.Fatalf("body = %q, want nil after reader error", body) + } + if !errors.Is(err, ErrTransport) { + t.Fatalf("error = %v, want ErrTransport", err) + } + if !errors.Is(err, readFailure) { + t.Fatalf("error = %v, want wrapped reader failure", err) + } + }) + + t.Run("limit plus one overflow is rejected", func(t *testing.T) { + body, err := readBoundedResponseBody(strings.NewReader("unused"), math.MaxInt64) + if body != nil { + t.Fatalf("body = %q, want nil for invalid limit", body) + } + if !errors.Is(err, ErrTransport) { + t.Fatalf("error = %v, want ErrTransport", err) + } + }) +} + +func TestHTTPTransportRejectsOversizedResponseBodies(t *testing.T) { + signer := newTestSigner(t) + message, err := BuildRecordsQuery(signer, RecordsFilter{Protocol: "test"}, "", nil, "") + if err != nil { + t.Fatalf("BuildRecordsQuery: %v", err) + } + + rpcResponse, err := json.Marshal(JsonRpcResponse{ + JSONRPC: "2.0", + ID: "response-limit-test", + Result: &JsonRpcResult{Reply: &DwnReply{ + Status: Status{Code: http.StatusOK, Detail: "OK"}, + }}, + }) + if err != nil { + t.Fatalf("marshal JSON-RPC response: %v", err) + } + + tests := []struct { + name string + body []byte + header string + limit int64 + wantDetail string + }{ + { + name: "normal JSON response", + body: rpcResponse, + limit: int64(len(rpcResponse) - 1), + wantDetail: "reading response body", + }, + { + name: "dwn-response header with binary data", + body: []byte("binary-record-data"), + header: string(rpcResponse), + limit: int64(len("binary-record-data") - 1), + wantDetail: "reading response data", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tc.header != "" { + w.Header().Set("dwn-response", tc.header) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write(tc.body) + })) + defer server.Close() + + transport := NewHTTPTransport(server.URL) + transport.maxResponseBodyBytes = tc.limit + result, err := transport.Send(context.Background(), "did:dht:target", message, nil) + if result != nil { + t.Fatalf("result = %+v, want nil on oversized response", result) + } + if !errors.Is(err, ErrTransport) { + t.Fatalf("error = %v, want ErrTransport", err) + } + if err == nil || !strings.Contains(err.Error(), tc.wantDetail) { + t.Fatalf("error = %v, want %q context", err, tc.wantDetail) + } + }) + } +} + +func TestHTTPTransportTruncatesMalformedResponsePreview(t *testing.T) { + const tailMarker = "tail-marker-must-not-appear" + body := strings.Repeat("x", maxHTTPResponseErrorPreviewBytes) + tailMarker + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte(body)) + })) + defer server.Close() + + signer := newTestSigner(t) + message, err := BuildRecordsQuery(signer, RecordsFilter{Protocol: "test"}, "", nil, "") + if err != nil { + t.Fatalf("BuildRecordsQuery: %v", err) + } + transport := NewHTTPTransport(server.URL) + transport.maxResponseBodyBytes = int64(len(body)) + result, err := transport.Send(context.Background(), "did:dht:target", message, nil) + if result != nil { + t.Fatalf("result = %+v, want nil for malformed response", result) + } + if !errors.Is(err, ErrTransport) { + t.Fatalf("error = %v, want ErrTransport", err) + } + if strings.Contains(err.Error(), tailMarker) { + t.Fatalf("error contains content beyond diagnostic preview: %v", err) + } + wantSize := fmt.Sprintf("(%d bytes total)", len(body)) + if !strings.Contains(err.Error(), wantSize) { + t.Fatalf("error = %v, want size diagnostic %q", err, wantSize) + } +} diff --git a/internal/engine/convert_test.go b/internal/engine/convert_test.go index fd97f8c..040b1d4 100644 --- a/internal/engine/convert_test.go +++ b/internal/engine/convert_test.go @@ -49,6 +49,27 @@ func TestConvertNodeKeyExpiry(t *testing.T) { } } +func TestConvertRevokedSelfDownMap(t *testing.T) { + now := time.Now().UTC() + selfIP := netip.MustParseAddr("10.200.1.1") + resp := &control.MapResponse{Node: &control.Node{ + ID: 1, StableID: "self", Name: "self", DID: "did:jwk:self", + Key: testWireGuardKey(), MeshIP: selfIP, + AllowedIPs: []netip.Prefix{netip.PrefixFrom(selfIP, selfIP.BitLen())}, + ExpiresAt: time.Unix(0, 0).UTC().Format(time.RFC3339Nano), + }} + nm, err := NewConverter("test").Convert(resp) + if err != nil { + t.Fatalf("Convert: %v", err) + } + if !nm.SelfNode.Valid() || !nm.SelfKeyExpiry().Before(now) { + t.Fatalf("revoked self = valid %v expiry %v, want past expiry", nm.SelfNode.Valid(), nm.SelfKeyExpiry()) + } + if len(nm.Peers) != 0 { + t.Fatalf("revoked peers = %d, want 0", len(nm.Peers)) + } +} + // testWireGuardKey returns a valid base64-encoded 32-byte key for testing. func testWireGuardKey() string { // Generate a real key pair so we have a valid 32-byte public key. diff --git a/internal/engine/dwncontrol.go b/internal/engine/dwncontrol.go index 746c34a..19ed983 100644 --- a/internal/engine/dwncontrol.go +++ b/internal/engine/dwncontrol.go @@ -40,6 +40,15 @@ type DWNControlConfig struct { // Calls are serialized and coalesced by RefreshCoordinator. MapResponseFunc func(ctx context.Context) (*netmap.NetworkMap, error) + // RefreshMapResponseFunc is called to obtain the current network state + // with the exact invalidation batch that triggered the rebuild. When set, + // it is preferred over MapResponseFunc. This lets loaders update local + // materialized state incrementally while retaining MapResponseFunc for + // legacy full-snapshot loaders. + // + // Calls are serialized and coalesced by RefreshCoordinator. + RefreshMapResponseFunc func(ctx context.Context, batch RefreshBatch) (*netmap.NetworkMap, error) + // OnMapResult observes every completed map load. It is called after the // control-client observer has received the result, and is useful for // reconciling host routing readiness with the exact map that was loaded. @@ -69,6 +78,11 @@ type DWNControlConfig struct { // bounded chance to establish before the initial full rebuild. StartupSubscriptionWait time.Duration + // ExpiryClock supplies time and timers for local peer-expiry projection. + // It defaults to the system clock and is primarily exposed for deterministic + // tests. + ExpiryClock RefreshClock + // NodePrivateKey, if set, overrides the auto-generated WireGuard node // key with the given key. This is essential when the WireGuard key has // already been published to a coordination store (e.g. DWN records) and @@ -133,6 +147,7 @@ type DWNControl struct { shutdownOnce sync.Once cancel context.CancelFunc coordinator *RefreshCoordinator + peerExpiry *peerExpiryScheduler initialMapApplied atomic.Bool } @@ -151,7 +166,7 @@ func NewDWNControl(config *DWNControlConfig, opts controlclient.Options) (*DWNCo if config == nil { return nil, fmt.Errorf("DWN control config is required") } - if config.MapResponseFunc == nil { + if config.RefreshMapResponseFunc == nil && config.MapResponseFunc == nil { return nil, fmt.Errorf("DWN control map response function is required") } if config.RefreshTimeout < 0 || config.StartupSubscriptionWait < 0 { @@ -194,12 +209,14 @@ func NewDWNControl(config *DWNControlConfig, opts controlclient.Options) (*DWNCo cc := &DWNControl{ config: DWNControlConfig{ MapResponseFunc: config.MapResponseFunc, + RefreshMapResponseFunc: config.RefreshMapResponseFunc, OnMapResult: config.OnMapResult, EndpointUpdateFunc: config.EndpointUpdateFunc, PollInterval: pollInterval, HealthyPollInterval: healthyPollInterval, RefreshTimeout: refreshTimeout, StartupSubscriptionWait: config.StartupSubscriptionWait, + ExpiryClock: config.ExpiryClock, NodePrivateKey: config.NodePrivateKey, DiscoKeyRegistry: config.DiscoKeyRegistry, Logf: logf, @@ -224,6 +241,9 @@ func NewDWNControl(config *DWNControlConfig, opts controlclient.Options) (*DWNCo return nil, err } cc.coordinator = coordinator + cc.peerExpiry = newPeerExpiryScheduler(config.ExpiryClock, func() { + coordinator.Notify(RefreshReasonExpiry) + }) cc.disco = opts.DiscoPublicKey @@ -303,7 +323,7 @@ func refreshStreamsReadyForStartup(health RefreshCoordinatorHealth) bool { // refreshControlState performs one bounded remote rebuild. The first usable // map is applied locally a second time after the event bus has observed its peer // views; this preserves the startup ordering workaround without a second DWN load. -func (cc *DWNControl) refreshControlState(ctx context.Context, _ RefreshBatch) error { +func (cc *DWNControl) refreshControlState(ctx context.Context, batch RefreshBatch) error { refreshCtx := ctx cancel := func() {} if cc.config.RefreshTimeout > 0 { @@ -311,7 +331,7 @@ func (cc *DWNControl) refreshControlState(ctx context.Context, _ RefreshBatch) e } defer cancel() - replay, err := cc.loadAndPush(refreshCtx) + replay, err := cc.loadAndPush(refreshCtx, batch) if err != nil { return err } @@ -327,15 +347,23 @@ func (cc *DWNControl) refreshControlState(ctx context.Context, _ RefreshBatch) e // loadAndPush reads DWN state, applies it once, and returns an observer-safe // clone for the one-time startup replay. -func (cc *DWNControl) loadAndPush(ctx context.Context) (*netmap.NetworkMap, error) { - if cc.config.MapResponseFunc == nil { - cc.logf("dwn-control: no MapResponseFunc configured") +func (cc *DWNControl) loadAndPush(ctx context.Context, batch RefreshBatch) (*netmap.NetworkMap, error) { + if cc.config.RefreshMapResponseFunc == nil && cc.config.MapResponseFunc == nil { + cc.logf("dwn-control: no map response function configured") return nil, nil } - nm, err := cc.config.MapResponseFunc(ctx) + var nm *netmap.NetworkMap + var err error + loaderName := "MapResponseFunc" + if cc.config.RefreshMapResponseFunc != nil { + loaderName = "RefreshMapResponseFunc" + nm, err = cc.config.RefreshMapResponseFunc(ctx, batch) + } else { + nm, err = cc.config.MapResponseFunc(ctx) + } if err != nil { - cc.logf("dwn-control: MapResponseFunc error: %v", err) + cc.logf("dwn-control: %s error: %v", loaderName, err) if cc.observer != nil { cc.observer.SetControlClientStatus(cc, controlclient.Status{ Err: err, @@ -358,6 +386,7 @@ func (cc *DWNControl) loadAndPush(ctx context.Context) (*netmap.NetworkMap, erro cc.prepareNetMap(nm) replay := cloneNetMapForReplay(nm) cc.pushNetMap(nm) + cc.peerExpiry.Schedule(nm) if cc.config.OnMapResult != nil { cc.config.OnMapResult(ctx, nm, nil) } @@ -439,6 +468,7 @@ func (cc *DWNControl) RefreshHealth() RefreshCoordinatorHealth { func (cc *DWNControl) Shutdown() { cc.shutdownOnce.Do(func() { + cc.peerExpiry.Stop() cc.cancel() cc.coordinator.Stop() }) @@ -483,13 +513,12 @@ func (cc *DWNControl) SetTKAHead(headHash string) { func (cc *DWNControl) UpdateEndpoints(endpoints []tailcfg.Endpoint) { if cc.config.EndpointUpdateFunc != nil { - go func() { + owned := slices.Clone(endpoints) + go func(endpoints []tailcfg.Endpoint) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() cc.config.EndpointUpdateFunc(ctx, endpoints) - // Coalesce the local write with its subscription echo. - cc.coordinator.Notify(RefreshReasonEndpoint) - }() + }(owned) } } diff --git a/internal/engine/dwncontrol_coordinator_test.go b/internal/engine/dwncontrol_coordinator_test.go index 1d72601..5c19a47 100644 --- a/internal/engine/dwncontrol_coordinator_test.go +++ b/internal/engine/dwncontrol_coordinator_test.go @@ -3,6 +3,7 @@ package engine import ( "context" "errors" + "net/netip" "reflect" "sync/atomic" "testing" @@ -46,6 +47,56 @@ func TestDWNControlRejectsInvalidCoordinatorConfig(t *testing.T) { } } +func TestDWNControlRefreshMapResponseFuncReceivesExactBatch(t *testing.T) { + startedAt := time.Date(2026, time.July, 11, 12, 34, 56, 789, time.UTC) + want := RefreshBatch{ + Reasons: []RefreshReason{RefreshReasonTopology, RefreshReasonDelivery}, + Sequence: 42, + Attempt: 3, + StartedAt: startedAt, + } + var got RefreshBatch + var legacyCalls atomic.Int32 + + cc, err := NewDWNControl(&DWNControlConfig{ + MapResponseFunc: func(context.Context) (*netmap.NetworkMap, error) { + legacyCalls.Add(1) + return nil, nil + }, + RefreshMapResponseFunc: func(_ context.Context, batch RefreshBatch) (*netmap.NetworkMap, error) { + got = batch + return nil, nil + }, + Logf: func(string, ...any) {}, + }, controlclient.Options{SkipStartForTests: true}) + if err != nil { + t.Fatalf("NewDWNControl: %v", err) + } + defer cc.Shutdown() + + if err := cc.refreshControlState(context.Background(), want); err != nil { + t.Fatalf("refreshControlState: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("refresh batch = %#v, want %#v", got, want) + } + if calls := legacyCalls.Load(); calls != 0 { + t.Fatalf("legacy map response calls = %d, want 0", calls) + } +} + +func TestDWNControlAcceptsRefreshMapResponseFuncWithoutLegacyLoader(t *testing.T) { + cc, err := NewDWNControl(&DWNControlConfig{ + RefreshMapResponseFunc: func(context.Context, RefreshBatch) (*netmap.NetworkMap, error) { + return nil, nil + }, + }, controlclient.Options{SkipStartForTests: true}) + if err != nil { + t.Fatalf("NewDWNControl: %v", err) + } + cc.Shutdown() +} + func TestDWNControlCoordinatorFirstSuccessReplaysLoadedMap(t *testing.T) { var loads atomic.Int32 var mapResults atomic.Int32 @@ -550,6 +601,112 @@ func TestDWNControlCoordinatorUsesConfiguredRefreshIntervals(t *testing.T) { } } +func TestDWNControlUpdateEndpointsReliesOnSubscriptionEcho(t *testing.T) { + published := make(chan struct{}) + var loads atomic.Int32 + cc, err := NewDWNControl(&DWNControlConfig{ + MapResponseFunc: func(context.Context) (*netmap.NetworkMap, error) { + loads.Add(1) + return nil, nil + }, + EndpointUpdateFunc: func(context.Context, []tailcfg.Endpoint) { close(published) }, + Logf: func(string, ...any) {}, + }, controlclient.Options{SkipStartForTests: true}) + if err != nil { + t.Fatalf("NewDWNControl: %v", err) + } + defer cc.Shutdown() + + cc.UpdateEndpoints(nil) + select { + case <-published: + case <-time.After(time.Second): + t.Fatal("endpoint publication did not run") + } + if got := loads.Load(); got != 0 { + t.Fatalf("endpoint publication triggered %d map loads, want 0", got) + } + if pending := cc.RefreshHealth().PendingReasons; len(pending) != 0 { + t.Fatalf("endpoint publication queued refresh reasons: %v", pending) + } +} + +func TestDWNControlUpdateEndpointsOwnsCallerSlice(t *testing.T) { + release := make(chan struct{}) + published := make(chan []tailcfg.Endpoint, 1) + cc, err := NewDWNControl(&DWNControlConfig{ + MapResponseFunc: func(context.Context) (*netmap.NetworkMap, error) { return nil, nil }, + EndpointUpdateFunc: func(_ context.Context, endpoints []tailcfg.Endpoint) { + <-release + published <- endpoints + }, + Logf: func(string, ...any) {}, + }, controlclient.Options{SkipStartForTests: true}) + if err != nil { + t.Fatalf("NewDWNControl: %v", err) + } + defer cc.Shutdown() + + original := netip.MustParseAddrPort("192.0.2.1:4242") + callerOwned := []tailcfg.Endpoint{{Addr: original}} + cc.UpdateEndpoints(callerOwned) + callerOwned[0].Addr = netip.MustParseAddrPort("198.51.100.2:5252") + close(release) + + select { + case got := <-published: + if len(got) != 1 || got[0].Addr != original { + t.Fatalf("published endpoints = %#v, want owned copy of %v", got, original) + } + case <-time.After(time.Second): + t.Fatal("endpoint publication did not run") + } +} + +func TestDWNControlPublishesRevokedMapAndRenewalAsSuccess(t *testing.T) { + now := time.Now().UTC() + selfAddress := netip.MustParsePrefix("10.200.70.205/32") + revoked := &netmap.NetworkMap{SelfNode: (&tailcfg.Node{ + Addresses: []netip.Prefix{selfAddress}, KeyExpiry: now.Add(-time.Second), + }).View()} + renewed := &netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{Addresses: []netip.Prefix{selfAddress}, KeyExpiry: now.Add(time.Hour)}).View(), + Peers: []tailcfg.NodeView{(&tailcfg.Node{Addresses: []netip.Prefix{netip.MustParsePrefix("10.200.176.93/32")}}).View()}, + } + maps := []*netmap.NetworkMap{revoked, renewed} + var calls atomic.Int32 + statuses := make(chan controlclient.Status, len(maps)) + cc, err := NewDWNControl(&DWNControlConfig{ + MapResponseFunc: func(context.Context) (*netmap.NetworkMap, error) { + return maps[int(calls.Add(1))-1], nil + }, + Logf: func(string, ...any) {}, + }, controlclient.Options{ + Observer: dwnControlObserverFunc(func(_ controlclient.Client, status controlclient.Status) { statuses <- status }), + SkipStartForTests: true, + }) + if err != nil { + t.Fatalf("NewDWNControl: %v", err) + } + defer cc.Shutdown() + + for i, want := range maps { + if _, err := cc.loadAndPush(context.Background(), RefreshBatch{}); err != nil { + t.Fatalf("loadAndPush %d: %v", i, err) + } + status := receiveDWNControlStatus(t, statuses) + if status.Err != nil || !status.LoggedIn || status.NetMap != want { + t.Fatalf("status %d = loggedIn %v map %p err %v, want success map %p", i, status.LoggedIn, status.NetMap, status.Err, want) + } + } + if !revoked.SelfKeyExpiry().Before(now) || len(revoked.Peers) != 0 { + t.Fatalf("revoked map = expiry %v peers %d", revoked.SelfKeyExpiry(), len(revoked.Peers)) + } + if !renewed.SelfKeyExpiry().After(now) || len(renewed.Peers) != 1 { + t.Fatalf("renewed map = expiry %v peers %d", renewed.SelfKeyExpiry(), len(renewed.Peers)) + } +} + type dwnControlObserverFunc func(controlclient.Client, controlclient.Status) func (f dwnControlObserverFunc) SetControlClientStatus(client controlclient.Client, status controlclient.Status) { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 7885381..960e2bd 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -464,23 +464,25 @@ func New(cfg Config) (*Engine, error) { // This replaces pure polling with event-driven updates when the DWN // server supports WebSocket subscriptions. subWatcher := NewSubscriptionWatcher(SubscriptionWatcherConfig{ - AnchorEndpoint: cfg.AnchorEndpoint, - AnchorTenant: cfg.AnchorTenant, - NetworkRecordID: cfg.NetworkRecordID, - SelfDID: cfg.SelfDID, - Signer: cfg.Signer, - ReadAuth: readAuth, - Logger: l, + AnchorEndpoint: cfg.AnchorEndpoint, + AnchorTenant: cfg.AnchorTenant, + NetworkRecordID: cfg.NetworkRecordID, + SelfDID: cfg.SelfDID, + Signer: cfg.Signer, + ReadAuth: readAuth, + TopologyEventHandler: dwnClient.StageTopologyEvent, + TopologyRepairHandler: dwnClient.RequireFullReconciliation, + Logger: l, }) // Wire the DWN control client into the LocalBackend. - // MapResponseFunc closes over our DWNClient and Converter to produce - // NetworkMaps from DWN records. + // RefreshMapResponseFunc uses staged topology deltas only for pure topology + // batches and performs a full reconciliation for every other trigger. snapshots := &meshSnapshotStore{} - mapFn := mapResponseFunc(dwnClient.LoadState, converter, snapshots.record) + mapFn := refreshMapResponseFunc(dwnClient, converter, snapshots.record) var engineRef *Engine dwnControlConfig := &DWNControlConfig{ - MapResponseFunc: mapFn, + RefreshMapResponseFunc: mapFn, PollInterval: pollInterval, HealthyPollInterval: healthyPollInterval, RefreshTimeout: refreshTimeout, diff --git a/internal/engine/peer_expiry.go b/internal/engine/peer_expiry.go new file mode 100644 index 0000000..d3e0e6d --- /dev/null +++ b/internal/engine/peer_expiry.go @@ -0,0 +1,148 @@ +package engine + +import ( + "sync" + "time" + + "github.com/enboxorg/meshnet/types/netmap" +) + +// peerExpiryScheduler reprojects the local materialized topology when a peer +// membership expires. DWNControl replaces the schedule after every committed +// map, so callbacks from an older generation must never invalidate the map. +// +// Self expiry is deliberately not scheduled here. meshnet's LocalBackend +// SetControlClientStatus path owns a generation-fenced netmap expiry timer; it +// marks the local key expired and blocks engine updates. Sending self expiry +// through the materializer would reject the expired self record and turn a +// terminal local condition into a futile full-DWN reconciliation loop. A later +// membership renewal installs a new map and a new peer-expiry schedule. +type peerExpiryScheduler struct { + clock RefreshClock + notify func() + + mu sync.Mutex + timer RefreshTimer + cancel chan struct{} + generation uint64 + stopped bool +} + +func newPeerExpiryScheduler(clock RefreshClock, notify func()) *peerExpiryScheduler { + if clock == nil { + clock = systemRefreshClock{} + } + return &peerExpiryScheduler{clock: clock, notify: notify} +} + +// Schedule atomically replaces the active deadline with the nearest future +// peer expiry that occurs before self expires. Zero, past, already-expired, +// and post-self-expiry peer deadlines cannot require a local refresh. +func (s *peerExpiryScheduler) Schedule(nm *netmap.NetworkMap) { + now := s.clock.Now() + deadline := nextLocalPeerExpiry(nm, now) + + s.mu.Lock() + defer s.mu.Unlock() + if s.stopped { + return + } + + s.generation++ + s.stopActiveLocked() + if deadline.IsZero() { + return + } + + timer := s.clock.NewTimer(deadline.Sub(now)) + cancel := make(chan struct{}) + generation := s.generation + s.timer = timer + s.cancel = cancel + go s.wait(timer, cancel, generation) +} + +func (s *peerExpiryScheduler) wait(timer RefreshTimer, cancel <-chan struct{}, generation uint64) { + select { + case <-timer.C(): + s.fire(generation) + case <-cancel: + } +} + +func (s *peerExpiryScheduler) fire(generation uint64) { + s.mu.Lock() + if s.stopped || generation != s.generation || s.timer == nil { + s.mu.Unlock() + return + } + s.timer = nil + s.cancel = nil + notify := s.notify + s.mu.Unlock() + + if notify != nil { + notify() + } +} + +// Stop permanently cancels the active schedule. It is idempotent. +func (s *peerExpiryScheduler) Stop() { + s.mu.Lock() + defer s.mu.Unlock() + if s.stopped { + return + } + s.stopped = true + s.generation++ + s.stopActiveLocked() +} + +func (s *peerExpiryScheduler) stopActiveLocked() { + if s.timer != nil { + s.timer.Stop() + s.timer = nil + } + if s.cancel != nil { + close(s.cancel) + s.cancel = nil + } +} + +func nextLocalPeerExpiry(nm *netmap.NetworkMap, now time.Time) time.Time { + if nm == nil { + return time.Time{} + } + + var nearest time.Time + for _, peer := range nm.Peers { + if !peer.Valid() || peer.Expired() { + continue + } + expiry := peer.KeyExpiry() + if !expiry.After(now) { + continue + } + if nearest.IsZero() || expiry.Before(nearest) { + nearest = expiry + } + } + if nearest.IsZero() { + return time.Time{} + } + if !nm.SelfNode.Valid() { + return time.Time{} + } + + selfExpiry := nm.SelfNode.KeyExpiry() + if selfExpiry.IsZero() { + return nearest + } + // Self authorization is terminal at its deadline. Do not queue a peer + // projection at or after it: LocalBackend independently fails the engine + // closed, and a renewed membership will install and schedule a fresh map. + if !selfExpiry.After(now) || !nearest.Before(selfExpiry) { + return time.Time{} + } + return nearest +} diff --git a/internal/engine/peer_expiry_test.go b/internal/engine/peer_expiry_test.go new file mode 100644 index 0000000..4f6d42b --- /dev/null +++ b/internal/engine/peer_expiry_test.go @@ -0,0 +1,376 @@ +package engine + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/enboxorg/meshd/internal/control" + "github.com/enboxorg/meshnet/control/controlclient" + "github.com/enboxorg/meshnet/tailcfg" + "github.com/enboxorg/meshnet/types/netmap" +) + +func TestPeerExpirySchedulerSchedulesNearestFuturePeer(t *testing.T) { + clock := newCoordinatorFakeClock() + now := clock.Now() + notified := make(chan struct{}, 1) + scheduler := newPeerExpiryScheduler(clock, func() { notified <- struct{}{} }) + t.Cleanup(scheduler.Stop) + + expired := &tailcfg.Node{KeyExpiry: now.Add(5 * time.Minute), Expired: true} + scheduler.Schedule(&netmap.NetworkMap{ + SelfNode: (&tailcfg.Node{KeyExpiry: now.Add(time.Hour)}).View(), + Peers: []tailcfg.NodeView{ + (&tailcfg.Node{}).View(), + (&tailcfg.Node{KeyExpiry: now.Add(-time.Minute)}).View(), + (&tailcfg.Node{KeyExpiry: now}).View(), + expired.View(), + (&tailcfg.Node{KeyExpiry: now.Add(20 * time.Minute)}).View(), + (&tailcfg.Node{KeyExpiry: now.Add(10 * time.Minute)}).View(), + }, + }) + + timer, _ := activePeerExpiryTimer(t, scheduler) + if got, want := timer.deadline, now.Add(10*time.Minute); !got.Equal(want) { + t.Fatalf("deadline = %v, want nearest peer expiry %v", got, want) + } + + clock.Advance(10 * time.Minute) + receivePeerExpiryNotification(t, notified) + if timer, _ := currentPeerExpiryTimer(scheduler); timer != nil { + t.Fatal("fired timer remains active") + } +} + +func TestPeerExpirySchedulerDefersToSelfExpiry(t *testing.T) { + clock := newCoordinatorFakeClock() + now := clock.Now() + var calls atomic.Int32 + scheduler := newPeerExpiryScheduler(clock, func() { calls.Add(1) }) + t.Cleanup(scheduler.Stop) + + tests := []struct { + name string + nm *netmap.NetworkMap + }{ + { + name: "self only", + nm: expiryTestNetMap(now.Add(10 * time.Minute)), + }, + { + name: "peer at self deadline", + nm: expiryTestNetMap(now.Add(10*time.Minute), now.Add(10*time.Minute)), + }, + { + name: "peer after self", + nm: expiryTestNetMap(now.Add(10*time.Minute), now.Add(20*time.Minute)), + }, + { + name: "self already expired", + nm: expiryTestNetMap(now.Add(-time.Minute), now.Add(20*time.Minute)), + }, + { + name: "self expires now", + nm: expiryTestNetMap(now, now.Add(20*time.Minute)), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + scheduler.Schedule(test.nm) + if timer, _ := currentPeerExpiryTimer(scheduler); timer != nil { + t.Fatalf("scheduled peer projection after terminal self expiry: %#v", timer) + } + }) + } + + clock.Advance(time.Hour) + if got := calls.Load(); got != 0 { + t.Fatalf("self expiry queued %d local refreshes, want 0", got) + } + + scheduler.Schedule(expiryTestNetMap(now.Add(2*time.Hour), clock.Now().Add(10*time.Minute))) + timer, _ := activePeerExpiryTimer(t, scheduler) + if got, want := timer.deadline, clock.Now().Add(10*time.Minute); !got.Equal(want) { + t.Fatalf("renewed schedule deadline = %v, want %v", got, want) + } +} + +func TestPeerExpirySchedulerReplacementCancelsAndFencesStaleCallback(t *testing.T) { + clock := newCoordinatorFakeClock() + now := clock.Now() + notified := make(chan struct{}, 1) + scheduler := newPeerExpiryScheduler(clock, func() { notified <- struct{}{} }) + t.Cleanup(scheduler.Stop) + + scheduler.Schedule(expiryTestNetMap(time.Time{}, now.Add(10*time.Minute))) + oldTimer, oldGeneration := activePeerExpiryTimer(t, scheduler) + scheduler.mu.Lock() + oldCancel := scheduler.cancel + scheduler.mu.Unlock() + + scheduler.Schedule(expiryTestNetMap(time.Time{}, now.Add(20*time.Minute))) + newTimer, newGeneration := activePeerExpiryTimer(t, scheduler) + if newGeneration == oldGeneration || newTimer == oldTimer { + t.Fatalf("replacement did not create a new generation: old=%d new=%d", oldGeneration, newGeneration) + } + if !fakeTimerStopped(clock, oldTimer) { + t.Fatal("replaced timer was not stopped") + } + select { + case <-oldCancel: + default: + t.Fatal("replaced timer waiter was not canceled") + } + + // Model an AfterFunc-style callback that was already runnable when Stop + // won. The generation and active-timer fence must reject it. + scheduler.fire(oldGeneration) + assertNoPeerExpiryNotification(t, notified) + + clock.Advance(20 * time.Minute) + receivePeerExpiryNotification(t, notified) +} + +func TestPeerExpirySchedulerStopCancelsAndPreventsReschedule(t *testing.T) { + clock := newCoordinatorFakeClock() + now := clock.Now() + notified := make(chan struct{}, 1) + scheduler := newPeerExpiryScheduler(clock, func() { notified <- struct{}{} }) + + scheduler.Schedule(expiryTestNetMap(time.Time{}, now.Add(10*time.Minute))) + timer, generation := activePeerExpiryTimer(t, scheduler) + scheduler.mu.Lock() + cancel := scheduler.cancel + scheduler.mu.Unlock() + + scheduler.Stop() + scheduler.Stop() + if !fakeTimerStopped(clock, timer) { + t.Fatal("shutdown did not stop active expiry timer") + } + select { + case <-cancel: + default: + t.Fatal("shutdown did not cancel expiry waiter") + } + + scheduler.fire(generation) + scheduler.Schedule(expiryTestNetMap(time.Time{}, now.Add(20*time.Minute))) + if timer, _ := currentPeerExpiryTimer(scheduler); timer != nil { + t.Fatal("stopped scheduler accepted a new deadline") + } + assertNoPeerExpiryNotification(t, notified) +} + +func TestDWNControlReplacesPeerExpiryScheduleAndStopsItOnShutdown(t *testing.T) { + clock := newCoordinatorFakeClock() + now := clock.Now() + maps := []*netmap.NetworkMap{ + expiryTestNetMap(time.Time{}, now.Add(10*time.Minute)), + expiryTestNetMap(time.Time{}, now.Add(20*time.Minute)), + } + var loads atomic.Int32 + observer := dwnControlObserverFunc(func(controlclient.Client, controlclient.Status) {}) + + cc, err := NewDWNControl(&DWNControlConfig{ + MapResponseFunc: func(context.Context) (*netmap.NetworkMap, error) { + index := int(loads.Add(1)) - 1 + return maps[index], nil + }, + ExpiryClock: clock, + Logf: func(string, ...any) {}, + }, controlclient.Options{Observer: observer, SkipStartForTests: true}) + if err != nil { + t.Fatalf("NewDWNControl: %v", err) + } + + if _, err := cc.loadAndPush(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonStartup}}); err != nil { + t.Fatalf("first loadAndPush: %v", err) + } + first, firstGeneration := activePeerExpiryTimer(t, cc.peerExpiry) + if got, want := first.deadline, now.Add(10*time.Minute); !got.Equal(want) { + t.Fatalf("first deadline = %v, want %v", got, want) + } + + if _, err := cc.loadAndPush(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonPeriodic}}); err != nil { + t.Fatalf("second loadAndPush: %v", err) + } + second, secondGeneration := activePeerExpiryTimer(t, cc.peerExpiry) + if secondGeneration == firstGeneration || !fakeTimerStopped(clock, first) { + t.Fatalf("successful replacement did not cancel first schedule: generations %d -> %d", firstGeneration, secondGeneration) + } + if got, want := second.deadline, now.Add(20*time.Minute); !got.Equal(want) { + t.Fatalf("second deadline = %v, want %v", got, want) + } + + cc.Shutdown() + if !fakeTimerStopped(clock, second) { + t.Fatal("DWNControl shutdown did not stop expiry schedule") + } + cc.peerExpiry.mu.Lock() + stopped := cc.peerExpiry.stopped + cc.peerExpiry.mu.Unlock() + if !stopped { + t.Fatal("DWNControl shutdown did not stop expiry scheduler") + } +} + +func TestDWNControlSelfOnlyExpiryQueuesNoRefresh(t *testing.T) { + clock := newCoordinatorFakeClock() + now := clock.Now() + var loads atomic.Int32 + cc, err := NewDWNControl(&DWNControlConfig{ + MapResponseFunc: func(context.Context) (*netmap.NetworkMap, error) { + loads.Add(1) + return expiryTestNetMap(now.Add(10 * time.Minute)), nil + }, + ExpiryClock: clock, + Logf: func(string, ...any) {}, + }, controlclient.Options{ + Observer: dwnControlObserverFunc(func(controlclient.Client, controlclient.Status) {}), + SkipStartForTests: true, + }) + if err != nil { + t.Fatalf("NewDWNControl: %v", err) + } + defer cc.Shutdown() + + if _, err := cc.loadAndPush(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonStartup}}); err != nil { + t.Fatalf("loadAndPush: %v", err) + } + if timer, _ := currentPeerExpiryTimer(cc.peerExpiry); timer != nil { + t.Fatal("self-only expiry created a coordinator timer") + } + clock.Advance(time.Hour) + if got := loads.Load(); got != 1 { + t.Fatalf("self-only expiry caused %d loads, want initial load only", got) + } + if pending := cc.RefreshHealth().PendingReasons; len(pending) != 0 { + t.Fatalf("self-only expiry queued refresh reasons: %v", pending) + } +} + +func TestExpiryOnlyRefreshUsesLocalMaterializer(t *testing.T) { + response := &control.MapResponse{} + loader := &recordingControlStateLoader{ + applyResponse: response, + loadError: errors.New("full load must not run"), + } + fn := refreshMapResponseFunc(loader, NewConverter("mesh.test"), nil) + + if _, err := fn(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonExpiry}}); err != nil { + t.Fatalf("expiry refresh: %v", err) + } + if len(loader.calls) != 1 || loader.calls[0] != "apply" { + t.Fatalf("loader calls = %v, want [apply]", loader.calls) + } +} + +func TestExpiryRepairSentinelFallsBackToFullReconciliation(t *testing.T) { + loader := &recordingControlStateLoader{ + applyError: control.ErrFullReconciliationRequired, + loadResponse: &control.MapResponse{}, + } + fn := refreshMapResponseFunc(loader, NewConverter("mesh.test"), nil) + + if _, err := fn(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonExpiry}}); err != nil { + t.Fatalf("expiry repair refresh: %v", err) + } + if len(loader.calls) != 2 || loader.calls[0] != "apply" || loader.calls[1] != "load" { + t.Fatalf("loader calls = %v, want [apply load]", loader.calls) + } +} + +func expiryTestNetMap(selfExpiry time.Time, peerExpiries ...time.Time) *netmap.NetworkMap { + nm := &netmap.NetworkMap{SelfNode: (&tailcfg.Node{KeyExpiry: selfExpiry}).View()} + for _, expiry := range peerExpiries { + nm.Peers = append(nm.Peers, (&tailcfg.Node{KeyExpiry: expiry}).View()) + } + return nm +} + +func activePeerExpiryTimer(t *testing.T, scheduler *peerExpiryScheduler) (*coordinatorFakeTimer, uint64) { + t.Helper() + timer, generation := currentPeerExpiryTimer(scheduler) + if timer == nil { + t.Fatal("no active peer expiry timer") + } + return timer, generation +} + +func currentPeerExpiryTimer(scheduler *peerExpiryScheduler) (*coordinatorFakeTimer, uint64) { + scheduler.mu.Lock() + defer scheduler.mu.Unlock() + if scheduler.timer == nil { + return nil, scheduler.generation + } + return scheduler.timer.(*coordinatorFakeTimer), scheduler.generation +} + +func fakeTimerStopped(clock *coordinatorFakeClock, timer *coordinatorFakeTimer) bool { + clock.mu.Lock() + defer clock.mu.Unlock() + return timer.stopped +} + +func receivePeerExpiryNotification(t *testing.T, notified <-chan struct{}) { + t.Helper() + select { + case <-notified: + case <-time.After(time.Second): + t.Fatal("timed out waiting for peer expiry notification") + } +} + +func assertNoPeerExpiryNotification(t *testing.T, notified <-chan struct{}) { + t.Helper() + select { + case <-notified: + t.Fatal("unexpected peer expiry notification") + default: + } +} + +func TestDWNControlPeerExpiryQueuesOnlyLocalReason(t *testing.T) { + clock := newCoordinatorFakeClock() + now := clock.Now() + var loads atomic.Int32 + cc, err := NewDWNControl(&DWNControlConfig{ + MapResponseFunc: func(context.Context) (*netmap.NetworkMap, error) { + loads.Add(1) + return expiryTestNetMap(time.Time{}, now.Add(10*time.Minute)), nil + }, + ExpiryClock: clock, + Logf: func(string, ...any) {}, + }, controlclient.Options{ + Observer: dwnControlObserverFunc(func(controlclient.Client, controlclient.Status) {}), + SkipStartForTests: true, + }) + if err != nil { + t.Fatalf("NewDWNControl: %v", err) + } + defer cc.Shutdown() + + if _, err := cc.loadAndPush(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonStartup}}); err != nil { + t.Fatalf("loadAndPush: %v", err) + } + _, generation := activePeerExpiryTimer(t, cc.peerExpiry) + before := cc.RefreshHealth() + cc.peerExpiry.fire(generation) + after := cc.RefreshHealth() + + if len(after.PendingReasons) != 1 || after.PendingReasons[0] != RefreshReasonExpiry { + t.Fatalf("pending reasons = %v, want [%s]", after.PendingReasons, RefreshReasonExpiry) + } + for _, stream := range []RefreshStream{RefreshStreamTopology, RefreshStreamDelivery} { + if after.Streams[stream] != before.Streams[stream] { + t.Fatalf("%s stream health changed on local expiry: before=%+v after=%+v", stream, before.Streams[stream], after.Streams[stream]) + } + } + if got := loads.Load(); got != 1 { + t.Fatalf("expiry notification ran %d loads before coordinator execution, want initial load only", got) + } +} diff --git a/internal/engine/refresh_coordinator.go b/internal/engine/refresh_coordinator.go index daa7e49..b3ec55f 100644 --- a/internal/engine/refresh_coordinator.go +++ b/internal/engine/refresh_coordinator.go @@ -20,6 +20,7 @@ const ( RefreshReasonStartup RefreshReason = "startup" RefreshReasonPeriodic RefreshReason = "periodic" RefreshReasonTopology RefreshReason = "topology" + RefreshReasonExpiry RefreshReason = "expiry" RefreshReasonDelivery RefreshReason = "delivery" RefreshReasonEndpoint RefreshReason = "endpoint" RefreshReasonManual RefreshReason = "manual" diff --git a/internal/engine/refresh_loader.go b/internal/engine/refresh_loader.go new file mode 100644 index 0000000..5db532a --- /dev/null +++ b/internal/engine/refresh_loader.go @@ -0,0 +1,102 @@ +package engine + +import ( + "context" + "errors" + "fmt" + + "github.com/enboxorg/meshd/internal/control" + "github.com/enboxorg/meshd/internal/dwn" + "github.com/enboxorg/meshnet/types/netmap" +) + +type controlStateLoader interface { + LoadStateValidated(context.Context, control.PendingStateValidator) (*control.MapResponse, error) + ApplyPendingStateValidated(context.Context, control.PendingStateValidator) (*control.MapResponse, error) +} + +type controlStateConverter interface { + Convert(*control.MapResponse) (*netmap.NetworkMap, error) +} + +// refreshMapResponseFunc chooses the cheapest authoritative state transition +// for a coordinator batch, then converts and publishes exactly one final +// result. Topology and local expiry batches can be satisfied by staged local +// state; every other invalidation requires a full DWN reconciliation. +func refreshMapResponseFunc( + loader controlStateLoader, + converter controlStateConverter, + onResult func(*control.MapResponse, error), +) func(context.Context, RefreshBatch) (*netmap.NetworkMap, error) { + return func(ctx context.Context, batch RefreshBatch) (*netmap.NetworkMap, error) { + if batchCanUsePendingState(batch) { + var converted *netmap.NetworkMap + var validationErr error + resp, err := loader.ApplyPendingStateValidated(ctx, func(candidate *control.MapResponse) error { + converted, validationErr = converter.Convert(candidate) + return validationErr + }) + if err == nil { + if onResult != nil { + onResult(resp, nil) + } + return converted, nil + } + if validationErr != nil { + if onResult != nil { + onResult(resp, err) + } + return nil, err + } + if deferFullReconciliation(ctx, err) || !errors.Is(err, control.ErrFullReconciliationRequired) { + err = fmt.Errorf("loading DWN state: %w", err) + if onResult != nil { + onResult(nil, err) + } + return nil, err + } + } + + var converted *netmap.NetworkMap + var validationErr error + resp, err := loader.LoadStateValidated(ctx, func(candidate *control.MapResponse) error { + converted, validationErr = converter.Convert(candidate) + return validationErr + }) + if err != nil { + if validationErr != nil { + if onResult != nil { + onResult(resp, err) + } + return nil, err + } + err = fmt.Errorf("loading DWN state: %w", err) + if onResult != nil { + onResult(resp, err) + } + return nil, err + } + if onResult != nil { + onResult(resp, nil) + } + return converted, nil + } +} + +func deferFullReconciliation(ctx context.Context, err error) bool { + return ctx.Err() != nil || errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) || errors.Is(err, dwn.ErrRateLimited) || + errors.Is(err, dwn.ErrTransport) +} + +func batchCanUsePendingState(batch RefreshBatch) bool { + if len(batch.Reasons) == 0 { + return false + } + for _, reason := range batch.Reasons { + if reason != RefreshReasonTopology && reason != RefreshReasonExpiry { + return false + } + } + return true +} diff --git a/internal/engine/refresh_loader_test.go b/internal/engine/refresh_loader_test.go new file mode 100644 index 0000000..7999913 --- /dev/null +++ b/internal/engine/refresh_loader_test.go @@ -0,0 +1,306 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/enboxorg/meshd/internal/control" + "github.com/enboxorg/meshd/internal/dwn" + "github.com/enboxorg/meshnet/types/netmap" +) + +type recordingControlStateLoader struct { + applyResponse *control.MapResponse + applyError error + postValidationError error + loadResponse *control.MapResponse + loadError error + calls []string +} + +type recordingControlStateConverter struct { + responses []*control.MapResponse + maps []*netmap.NetworkMap + err error +} + +func (c *recordingControlStateConverter) Convert(response *control.MapResponse) (*netmap.NetworkMap, error) { + c.responses = append(c.responses, response) + if c.err != nil { + return nil, c.err + } + result := &netmap.NetworkMap{} + c.maps = append(c.maps, result) + return result, nil +} + +func (l *recordingControlStateLoader) ApplyPendingStateValidated(_ context.Context, validate control.PendingStateValidator) (*control.MapResponse, error) { + l.calls = append(l.calls, "apply") + if l.applyError != nil { + return l.applyResponse, l.applyError + } + if validate != nil { + if err := validate(l.applyResponse); err != nil { + return l.applyResponse, err + } + } + if l.postValidationError != nil { + return l.applyResponse, l.postValidationError + } + return l.applyResponse, nil +} + +func (l *recordingControlStateLoader) LoadStateValidated(_ context.Context, validate control.PendingStateValidator) (*control.MapResponse, error) { + l.calls = append(l.calls, "load") + if l.loadError != nil { + return l.loadResponse, l.loadError + } + if validate != nil { + if err := validate(l.loadResponse); err != nil { + return l.loadResponse, err + } + } + return l.loadResponse, nil +} + +func TestRefreshMapResponseFuncUsesPendingStateForTopologyOnlyBatch(t *testing.T) { + response := &control.MapResponse{} + loader := &recordingControlStateLoader{ + applyResponse: response, + loadError: errors.New("full load must not run"), + } + resultCalls := 0 + fn := refreshMapResponseFunc(loader, NewConverter("mesh.test"), func(got *control.MapResponse, err error) { + resultCalls++ + if got != response || err != nil { + t.Fatalf("result = (%p, %v), want (%p, nil)", got, err, response) + } + }) + + if _, err := fn(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonTopology}}); err != nil { + t.Fatalf("refresh: %v", err) + } + if !reflect.DeepEqual(loader.calls, []string{"apply"}) { + t.Fatalf("loader calls = %v, want [apply]", loader.calls) + } + if resultCalls != 1 { + t.Fatalf("result calls = %d, want 1", resultCalls) + } +} + +func TestRefreshMapResponseFuncUsesFullStateForNonTopologyBatches(t *testing.T) { + tests := []struct { + name string + reasons []RefreshReason + }{ + {name: "empty"}, + {name: "startup", reasons: []RefreshReason{RefreshReasonStartup}}, + {name: "periodic", reasons: []RefreshReason{RefreshReasonPeriodic}}, + {name: "delivery", reasons: []RefreshReason{RefreshReasonDelivery}}, + {name: "manual", reasons: []RefreshReason{RefreshReasonManual}}, + {name: "endpoint", reasons: []RefreshReason{RefreshReasonEndpoint}}, + {name: "mixed", reasons: []RefreshReason{RefreshReasonTopology, RefreshReasonDelivery}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loader := &recordingControlStateLoader{ + applyError: errors.New("pending state must not run"), + loadResponse: &control.MapResponse{}, + } + resultCalls := 0 + fn := refreshMapResponseFunc(loader, NewConverter("mesh.test"), func(got *control.MapResponse, err error) { + resultCalls++ + if got != loader.loadResponse || err != nil { + t.Fatalf("result = (%p, %v), want full response", got, err) + } + }) + if _, err := fn(context.Background(), RefreshBatch{Reasons: test.reasons}); err != nil { + t.Fatalf("refresh: %v", err) + } + if !reflect.DeepEqual(loader.calls, []string{"load"}) { + t.Fatalf("loader calls = %v, want [load]", loader.calls) + } + if resultCalls != 1 { + t.Fatalf("result calls = %d, want 1", resultCalls) + } + }) + } +} + +func TestRefreshMapResponseFuncFullConverterFailurePublishesOnce(t *testing.T) { + convertErr := errors.New("full candidate rejected") + response := &control.MapResponse{} + loader := &recordingControlStateLoader{loadResponse: response} + converter := &recordingControlStateConverter{err: convertErr} + resultCalls := 0 + fn := refreshMapResponseFunc(loader, converter, func(got *control.MapResponse, err error) { + resultCalls++ + if got != response || !errors.Is(err, convertErr) { + t.Fatalf("result = (%p, %v), want candidate and converter error", got, err) + } + }) + + if _, err := fn(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonStartup}}); !errors.Is(err, convertErr) { + t.Fatalf("refresh error = %v", err) + } + if !reflect.DeepEqual(loader.calls, []string{"load"}) { + t.Fatalf("loader calls = %v, want [load]", loader.calls) + } + if len(converter.responses) != 1 || converter.responses[0] != response || resultCalls != 1 { + t.Fatalf("converter responses=%#v result calls=%d", converter.responses, resultCalls) + } +} + +func TestRefreshMapResponseFuncFallsBackOnceForFullReconciliationSentinel(t *testing.T) { + fullResponse := &control.MapResponse{} + loader := &recordingControlStateLoader{ + applyError: fmt.Errorf("wrapped: %w", control.ErrFullReconciliationRequired), + loadResponse: fullResponse, + } + converter := &recordingControlStateConverter{} + resultCalls := 0 + fn := refreshMapResponseFunc(loader, converter, func(got *control.MapResponse, err error) { + resultCalls++ + if got != fullResponse || err != nil { + t.Fatalf("result = (%p, %v), want final full response", got, err) + } + }) + + if _, err := fn(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonTopology}}); err != nil { + t.Fatalf("refresh: %v", err) + } + if !reflect.DeepEqual(loader.calls, []string{"apply", "load"}) { + t.Fatalf("loader calls = %v, want [apply load]", loader.calls) + } + if resultCalls != 1 { + t.Fatalf("result calls = %d, want one final publication", resultCalls) + } + if len(converter.responses) != 1 || converter.responses[0] != fullResponse { + t.Fatalf("converted responses = %#v, want final response exactly once", converter.responses) + } +} + +func TestRefreshMapResponseFuncCommitFenceFailureDiscardsCandidate(t *testing.T) { + candidate := &control.MapResponse{} + fullResponse := &control.MapResponse{} + loader := &recordingControlStateLoader{ + applyResponse: candidate, + postValidationError: control.ErrFullReconciliationRequired, + loadResponse: fullResponse, + } + converter := &recordingControlStateConverter{} + resultCalls := 0 + fn := refreshMapResponseFunc(loader, converter, func(got *control.MapResponse, err error) { + resultCalls++ + if got != fullResponse || err != nil { + t.Fatalf("published result = (%p, %v), want final full response", got, err) + } + }) + + got, err := fn(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonTopology}}) + if err != nil { + t.Fatalf("refresh: %v", err) + } + if !reflect.DeepEqual(loader.calls, []string{"apply", "load"}) { + t.Fatalf("loader calls = %v, want [apply load]", loader.calls) + } + if len(converter.responses) != 2 || converter.responses[0] != candidate || converter.responses[1] != fullResponse { + t.Fatalf("converted responses = %#v, want candidate then full", converter.responses) + } + if len(converter.maps) != 2 || got != converter.maps[1] || got == converter.maps[0] { + t.Fatalf("returned map = %p, converted maps = %#v", got, converter.maps) + } + if resultCalls != 1 { + t.Fatalf("result calls = %d, want only the full outcome", resultCalls) + } +} + +func TestRefreshMapResponseFuncConverterFailureDoesNotCommitOrLoadFull(t *testing.T) { + convertErr := errors.New("converter rejected candidate") + response := &control.MapResponse{} + loader := &recordingControlStateLoader{applyResponse: response, loadError: errors.New("full load must not run")} + converter := &recordingControlStateConverter{err: convertErr} + resultCalls := 0 + fn := refreshMapResponseFunc(loader, converter, func(got *control.MapResponse, err error) { + resultCalls++ + if got != response || !errors.Is(err, convertErr) { + t.Fatalf("result = (%p, %v)", got, err) + } + }) + + if _, err := fn(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonTopology}}); !errors.Is(err, convertErr) { + t.Fatalf("refresh error = %v", err) + } + if !reflect.DeepEqual(loader.calls, []string{"apply"}) { + t.Fatalf("loader calls = %v, want [apply]", loader.calls) + } + if len(converter.responses) != 1 || resultCalls != 1 { + t.Fatalf("converter calls=%d result calls=%d", len(converter.responses), resultCalls) + } +} + +func TestRefreshMapResponseFuncDefersFullLoadForStructuredRetryErrors(t *testing.T) { + tests := []struct { + name string + err error + want error + }{ + {name: "rate limit", err: errors.Join(control.ErrFullReconciliationRequired, &dwn.RateLimitError{RetryAfter: time.Second}), want: dwn.ErrRateLimited}, + {name: "delta hydration HTTP 500", err: errors.Join(control.ErrFullReconciliationRequired, fmt.Errorf("hydrating record: %w", dwn.ErrTransport)), want: dwn.ErrTransport}, + {name: "canceled", err: errors.Join(control.ErrFullReconciliationRequired, context.Canceled), want: context.Canceled}, + {name: "deadline", err: errors.Join(control.ErrFullReconciliationRequired, context.DeadlineExceeded), want: context.DeadlineExceeded}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loader := &recordingControlStateLoader{applyError: test.err, loadError: errors.New("full load must not run")} + resultCalls := 0 + fn := refreshMapResponseFunc(loader, &recordingControlStateConverter{}, func(got *control.MapResponse, err error) { + resultCalls++ + if got != nil || !errors.Is(err, test.want) { + t.Fatalf("result = (%p, %v)", got, err) + } + }) + if _, err := fn(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonTopology}}); !errors.Is(err, test.want) { + t.Fatalf("refresh error = %v", err) + } + if !reflect.DeepEqual(loader.calls, []string{"apply"}) || resultCalls != 1 { + t.Fatalf("loader calls=%v result calls=%d", loader.calls, resultCalls) + } + }) + } +} + +func TestRefreshMapResponseFuncFailurePreservesLastGoodSnapshot(t *testing.T) { + refreshFailure := errors.New("incremental projector failed") + loader := &recordingControlStateLoader{loadResponse: &control.MapResponse{}} + store := &meshSnapshotStore{} + fn := refreshMapResponseFunc(loader, NewConverter("mesh.test"), store.record) + + if _, err := fn(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonStartup}}); err != nil { + t.Fatalf("initial refresh: %v", err) + } + before := store.load() + if before == nil || before.Generation != 1 || before.LastError != "" { + t.Fatalf("initial snapshot = %#v", before) + } + + loader.applyError = refreshFailure + if _, err := fn(context.Background(), RefreshBatch{Reasons: []RefreshReason{RefreshReasonTopology}}); !errors.Is(err, refreshFailure) { + t.Fatalf("incremental refresh error = %v, want %v", err, refreshFailure) + } + after := store.load() + if after.Generation != before.Generation || !after.RefreshedAt.Equal(before.RefreshedAt) { + t.Fatalf("failure replaced last-good snapshot: before=%#v after=%#v", before, after) + } + if !strings.Contains(after.LastError, refreshFailure.Error()) { + t.Fatalf("LastError = %q, want %q", after.LastError, refreshFailure) + } + if !reflect.DeepEqual(loader.calls, []string{"load", "apply"}) { + t.Fatalf("loader calls = %v", loader.calls) + } +} diff --git a/internal/engine/routing_status_test.go b/internal/engine/routing_status_test.go index 1e3ba72..50d8cc4 100644 --- a/internal/engine/routing_status_test.go +++ b/internal/engine/routing_status_test.go @@ -217,9 +217,9 @@ func TestDWNControlReportsMapResults(t *testing.T) { defer cc.Shutdown() ctx := context.Background() - cc.loadAndPush(ctx) - cc.loadAndPush(ctx) - cc.loadAndPush(ctx) + cc.loadAndPush(ctx, RefreshBatch{}) + cc.loadAndPush(ctx, RefreshBatch{}) + cc.loadAndPush(ctx, RefreshBatch{}) if len(results) != 3 { t.Fatalf("map result callbacks = %d, want 3", len(results)) } @@ -234,6 +234,63 @@ func TestDWNControlReportsMapResults(t *testing.T) { } } +func TestHandleControlMapResultRevokedSelfRemovesPeerRoutes(t *testing.T) { + routerRecorder := &recordingRoutingRouter{} + eng := newRoutingTestEngine(true, routerRecorder, discardRoutingLogger()) + ctx := context.Background() + + eng.handleControlMapResult(ctx, routingTestNetMap(), nil) + selfAddress := netip.MustParsePrefix("10.200.70.205/32") + revoked := &netmap.NetworkMap{SelfNode: (&tailcfg.Node{ + Addresses: []netip.Prefix{selfAddress}, + KeyExpiry: time.Now().Add(-time.Second), + }).View()} + eng.handleControlMapResult(ctx, revoked, nil) + + configs := routerRecorder.Configs() + if len(configs) != 2 { + t.Fatalf("router configs = %d, want 2", len(configs)) + } + if len(configs[0].Routes) != 1 { + t.Fatalf("initial routes = %v, want peer route", configs[0].Routes) + } + if len(configs[1].Routes) != 0 { + t.Fatalf("revoked routes = %v, want none", configs[1].Routes) + } + if len(configs[1].LocalAddrs) != 1 || configs[1].LocalAddrs[0] != selfAddress { + t.Fatalf("revoked local addresses = %v, want %v", configs[1].LocalAddrs, selfAddress) + } +} + +type recordingRoutingRouter struct { + mu sync.Mutex + configs []*router.Config +} + +func (r *recordingRoutingRouter) Up() error { return nil } +func (r *recordingRoutingRouter) Close() error { return nil } +func (r *recordingRoutingRouter) Set(cfg *router.Config) error { + r.mu.Lock() + defer r.mu.Unlock() + owned := *cfg + owned.LocalAddrs = append([]netip.Prefix(nil), cfg.LocalAddrs...) + owned.Routes = append([]netip.Prefix(nil), cfg.Routes...) + r.configs = append(r.configs, &owned) + return nil +} +func (r *recordingRoutingRouter) Configs() []*router.Config { + r.mu.Lock() + defer r.mu.Unlock() + configs := make([]*router.Config, len(r.configs)) + for i, cfg := range r.configs { + owned := *cfg + owned.LocalAddrs = append([]netip.Prefix(nil), cfg.LocalAddrs...) + owned.Routes = append([]netip.Prefix(nil), cfg.Routes...) + configs[i] = &owned + } + return configs +} + type scriptedRoutingRouter struct { mu sync.Mutex setErrors []error diff --git a/internal/engine/subscribe.go b/internal/engine/subscribe.go index ea46005..e83d867 100644 --- a/internal/engine/subscribe.go +++ b/internal/engine/subscribe.go @@ -54,14 +54,16 @@ type subscriptionRefreshCoordinator interface { // reducing peer discovery latency from up to 30s to near-instant. // The poll timer remains as a fallback for missed events. type SubscriptionWatcher struct { - endpoint string - anchorTenant string - networkRecordID string - selfDID string - signer *dwn.Signer - readAuth dwn.MessageAuth - logger *slog.Logger - newManager subscriptionManagerFactory + endpoint string + anchorTenant string + networkRecordID string + selfDID string + signer *dwn.Signer + readAuth dwn.MessageAuth + topologyHandler func(*dwn.SubscriptionMessage) error + topologyRepairHandler func() + logger *slog.Logger + newManager subscriptionManagerFactory mu sync.Mutex manager subscriptionManager @@ -100,6 +102,17 @@ type SubscriptionWatcherConfig struct { // subscriptions and therefore keeps polling for topology changes. ReadAuth dwn.MessageAuth + // TopologyEventHandler applies topology event frames to local materialized + // state before the refresh coordinator is invalidated. It runs + // synchronously; returning an error prevents invalidation and propagates to + // the subscription so the event cursor is not acknowledged. + TopologyEventHandler func(*dwn.SubscriptionMessage) error + + // TopologyRepairHandler marks local topology state as requiring an + // authoritative rebuild. It runs before coordinator lifecycle actions can + // release or schedule a topology repair. + TopologyRepairHandler func() + // Logger is the structured logger. Logger *slog.Logger } @@ -113,13 +126,15 @@ func NewSubscriptionWatcher(cfg SubscriptionWatcherConfig) *SubscriptionWatcher } return &SubscriptionWatcher{ - endpoint: cfg.AnchorEndpoint, - anchorTenant: cfg.AnchorTenant, - networkRecordID: cfg.NetworkRecordID, - selfDID: cfg.SelfDID, - signer: cfg.Signer, - readAuth: cloneMessageAuth(cfg.ReadAuth), - logger: l.With(slog.String("component", "subscription-watcher")), + endpoint: cfg.AnchorEndpoint, + anchorTenant: cfg.AnchorTenant, + networkRecordID: cfg.NetworkRecordID, + selfDID: cfg.SelfDID, + signer: cfg.Signer, + readAuth: cloneMessageAuth(cfg.ReadAuth), + topologyHandler: cfg.TopologyEventHandler, + topologyRepairHandler: cfg.TopologyRepairHandler, + logger: l.With(slog.String("component", "subscription-watcher")), newManager: func(endpoint string, logger *slog.Logger) subscriptionManager { return dwn.NewSubscriptionManager(endpoint, logger) }, @@ -202,6 +217,11 @@ func (w *SubscriptionWatcher) handleSubscriptionMessage(stream RefreshStream, me switch message.Type { case "event": + if stream == RefreshStreamTopology && w.topologyHandler != nil { + if err := w.topologyHandler(message); err != nil { + return fmt.Errorf("handling topology event: %w", err) + } + } w.logger.Debug("DWN record changed", slog.String("source", string(stream)), slog.Any("cursor", message.Cursor), @@ -236,6 +256,9 @@ func (w *SubscriptionWatcher) handleSubscriptionLifecycle(stream RefreshStream, slog.String("source", string(stream)), slog.Bool("needsFullRefresh", event.NeedsFullRefresh), ) + if stream == RefreshStreamTopology && event.NeedsFullRefresh && w.topologyRepairHandler != nil { + w.topologyRepairHandler() + } w.setStreamLive(stream, true, event.NeedsFullRefresh) case dwn.SubscriptionLifecycleProgressGap: // Wait for the fresh replacement stream before rebuilding. This keeps @@ -244,6 +267,9 @@ func (w *SubscriptionWatcher) handleSubscriptionLifecycle(stream RefreshStream, slog.String("source", string(stream)), slog.Any("gap", event.Gap), ) + if stream == RefreshStreamTopology && w.topologyRepairHandler != nil { + w.topologyRepairHandler() + } w.setStreamLive(stream, false, true) case dwn.SubscriptionLifecycleRetrying: // The coordinator switches to the fallback cadence while reconnecting. @@ -258,6 +284,9 @@ func (w *SubscriptionWatcher) handleSubscriptionLifecycle(stream RefreshStream, slog.String("source", string(stream)), slog.Any("error", event.Err), ) + if stream == RefreshStreamTopology && w.topologyRepairHandler != nil { + w.topologyRepairHandler() + } w.setStreamLive(stream, false, false) w.notify(reasonForStream(stream)) default: diff --git a/internal/engine/subscribe_test.go b/internal/engine/subscribe_test.go index 5d7f6b1..d5e6fb8 100644 --- a/internal/engine/subscribe_test.go +++ b/internal/engine/subscribe_test.go @@ -684,6 +684,74 @@ func TestSubscriptionWatcherSetupFailureClosesPartialSubscriptions(t *testing.T) } } +func TestSubscriptionWatcherTopologyHandlerRunsBeforeInvalidation(t *testing.T) { + coordinator := newRecordingRefreshCoordinator() + message := &dwn.SubscriptionMessage{Type: dwn.SubscriptionEventType} + handlerCalled := false + w := NewSubscriptionWatcher(SubscriptionWatcherConfig{ + TopologyEventHandler: func(got *dwn.SubscriptionMessage) error { + if got != message { + t.Fatalf("handler message = %p, want %p", got, message) + } + if calls := coordinator.takeCalls(); len(calls) != 0 { + t.Fatalf("coordinator called before topology handler: %#v", calls) + } + handlerCalled = true + return nil + }, + }) + w.SetRefreshCoordinator(coordinator) + + if err := w.handleSubscriptionMessage(RefreshStreamTopology, message); err != nil { + t.Fatalf("handleSubscriptionMessage: %v", err) + } + if !handlerCalled { + t.Fatal("topology handler was not called") + } + requireRefreshCoordinatorCalls(t, coordinator, []refreshCoordinatorCall{{ + method: "invalidate", stream: RefreshStreamTopology, reason: RefreshReasonTopology, + }}) +} + +func TestSubscriptionWatcherTopologyHandlerErrorPreventsInvalidation(t *testing.T) { + handlerErr := errors.New("materializing topology event") + coordinator := newRecordingRefreshCoordinator() + w := NewSubscriptionWatcher(SubscriptionWatcherConfig{ + TopologyEventHandler: func(*dwn.SubscriptionMessage) error { + return handlerErr + }, + }) + w.SetRefreshCoordinator(coordinator) + + err := w.handleSubscriptionMessage(RefreshStreamTopology, &dwn.SubscriptionMessage{Type: dwn.SubscriptionEventType}) + if !errors.Is(err, handlerErr) { + t.Fatalf("handleSubscriptionMessage error = %v, want %v", err, handlerErr) + } + requireRefreshCoordinatorCalls(t, coordinator, nil) +} + +func TestSubscriptionWatcherDeliveryEventSkipsTopologyHandler(t *testing.T) { + var handlerCalls atomic.Int32 + coordinator := newRecordingRefreshCoordinator() + w := NewSubscriptionWatcher(SubscriptionWatcherConfig{ + TopologyEventHandler: func(*dwn.SubscriptionMessage) error { + handlerCalls.Add(1) + return errors.New("topology handler must not receive delivery events") + }, + }) + w.SetRefreshCoordinator(coordinator) + + if err := w.handleSubscriptionMessage(RefreshStreamDelivery, &dwn.SubscriptionMessage{Type: dwn.SubscriptionEventType}); err != nil { + t.Fatalf("handleSubscriptionMessage: %v", err) + } + if calls := handlerCalls.Load(); calls != 0 { + t.Fatalf("topology handler calls = %d, want 0", calls) + } + requireRefreshCoordinatorCalls(t, coordinator, []refreshCoordinatorCall{{ + method: "invalidate", stream: RefreshStreamDelivery, reason: RefreshReasonDelivery, + }}) +} + func TestSubscriptionWatcherMessageCallbacks(t *testing.T) { tests := []struct { name string diff --git a/internal/engine/subscription_repair_test.go b/internal/engine/subscription_repair_test.go new file mode 100644 index 0000000..c25f168 --- /dev/null +++ b/internal/engine/subscription_repair_test.go @@ -0,0 +1,93 @@ +package engine + +import ( + "testing" + + "github.com/enboxorg/meshd/internal/dwn" +) + +func TestSubscriptionWatcherMarksTopologyRepairBeforeCoordinatorLifecycle(t *testing.T) { + tests := []struct { + name string + event dwn.SubscriptionLifecycleEvent + want []refreshCoordinatorCall + }{ + { + name: "fresh establishment", + event: dwn.SubscriptionLifecycleEvent{ + Kind: dwn.SubscriptionLifecycleEstablished, + NeedsFullRefresh: true, + }, + want: []refreshCoordinatorCall{{ + method: "live", stream: RefreshStreamTopology, live: true, needsFullRefresh: true, + }}, + }, + { + name: "progress gap", + event: dwn.SubscriptionLifecycleEvent{ + Kind: dwn.SubscriptionLifecycleProgressGap, + Gap: &dwn.ProgressGapInfo{Reason: "compacted"}, + }, + want: []refreshCoordinatorCall{{ + method: "live", stream: RefreshStreamTopology, needsFullRefresh: true, + }}, + }, + { + name: "terminal", + event: dwn.SubscriptionLifecycleEvent{ + Kind: dwn.SubscriptionLifecycleTerminal, + }, + want: []refreshCoordinatorCall{ + {method: "live", stream: RefreshStreamTopology}, + {method: "notify", reason: RefreshReasonTopology}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + coordinator := newRecordingRefreshCoordinator() + repairCalls := 0 + w := NewSubscriptionWatcher(SubscriptionWatcherConfig{ + TopologyRepairHandler: func() { + if calls := coordinator.takeCalls(); len(calls) != 0 { + t.Fatalf("coordinator called before topology repair: %#v", calls) + } + repairCalls++ + }, + }) + w.SetRefreshCoordinator(coordinator) + + w.handleSubscriptionLifecycle(RefreshStreamTopology, test.event) + + if repairCalls != 1 { + t.Fatalf("repair calls = %d, want 1", repairCalls) + } + requireRefreshCoordinatorCalls(t, coordinator, test.want) + }) + } +} + +func TestSubscriptionWatcherDoesNotMarkRepairForHealthyOrDeliveryLifecycle(t *testing.T) { + repairCalls := 0 + w := NewSubscriptionWatcher(SubscriptionWatcherConfig{ + TopologyRepairHandler: func() { repairCalls++ }, + }) + w.handleSubscriptionLifecycle(RefreshStreamTopology, dwn.SubscriptionLifecycleEvent{ + Kind: dwn.SubscriptionLifecycleEstablished, + }) + w.handleSubscriptionLifecycle(RefreshStreamDelivery, dwn.SubscriptionLifecycleEvent{ + Kind: dwn.SubscriptionLifecycleEstablished, + NeedsFullRefresh: true, + }) + w.handleSubscriptionLifecycle(RefreshStreamDelivery, dwn.SubscriptionLifecycleEvent{ + Kind: dwn.SubscriptionLifecycleProgressGap, + }) + w.handleSubscriptionLifecycle(RefreshStreamDelivery, dwn.SubscriptionLifecycleEvent{ + Kind: dwn.SubscriptionLifecycleTerminal, + }) + + if repairCalls != 0 { + t.Fatalf("repair calls = %d, want 0", repairCalls) + } +}