diff --git a/agent/agent.go b/agent/agent.go index fd7202b..0dab512 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -47,11 +47,18 @@ type SyncRunner func(ctx context.Context, vol *config.Volume, destName string) S // SyncRunReport is the SyncRunner result surfaced to the scheduler. // Kept minimal — anything richer belongs in the runs table the CLI -// inspects via `squirrel runs`, not on the agent's hot path. +// inspects via `squirrel runs`, not on the agent's hot path. Conflicts +// and Contested carry the peer-sync divergence counts so the scheduler +// logs a conflict signal on the initiating machine, not only the hub +// (#158, F27): Conflicts is how many paths this run resolved by +// preserving the loser and landing this node's bytes live, Contested how +// many the receiver refused because a prior freeze still stands. type SyncRunReport struct { - RunID int64 - Status string - Err error + RunID int64 + Status string + Err error + Conflicts int + Contested int } // VerifyRunner is the function shape the scheduler delegates one diff --git a/agent/contested_test.go b/agent/contested_test.go new file mode 100644 index 0000000..ab1c477 --- /dev/null +++ b/agent/contested_test.go @@ -0,0 +1,190 @@ +package agent + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/mbertschler/squirrel/store" + "github.com/mbertschler/squirrel/syncproto" + "github.com/zeebo/blake3" +) + +// seedLocalWrite writes content to path and records a present index row +// attributed to a *local* index run (no peer linkage), so a divergent +// initiator hash classifies as a conflict — the precondition for a freeze. +func (f *preStageFixture) seedLocalWrite(t *testing.T, ctx context.Context, path string, content []byte) []byte { + t.Helper() + abs := filepath.Join(f.vol.Path, path) + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + t.Fatalf("mkdir for %s: %v", path, err) + } + if err := os.WriteFile(abs, content, 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + sum := blake3.Sum256(content) + if err := f.store.Upsert(ctx, store.FileRow{ + VolumeID: f.volID, Path: path, Blake3: sum[:], SizeBytes: int64(len(content)), + MtimeNs: store.NowNs(), Status: store.StatusPresent, + FirstSeenRunID: f.localRun, LastSeenRunID: f.localRun, IndexedAtNs: store.NowNs(), + }, nil); err != nil { + t.Fatalf("Upsert %s: %v", path, err) + } + return sum[:] +} + +// freshSession opens another receiver run and returns a session bound to +// it at the given protocol version, so a test can drive successive /plan +// exchanges the way successive cadence ticks would. +func (f *preStageFixture) freshSession(t *testing.T, protocol int) *peerSession { + t.Helper() + run, err := f.store.BeginPeerSyncRun(context.Background(), f.volID, f.peerID, 7, "peerA") + if err != nil { + t.Fatalf("BeginPeerSyncRun: %v", err) + } + sess := f.newSession() + sess.receiverRunID = run + sess.protocolVersion = protocol + return sess +} + +// countConflictRunDirs returns how many run-/ subdirectories exist +// under .squirrel-conflicts — one per distinct conflict-preserving run, so +// the F27 unbounded-growth regression shows up as this number climbing. +func countConflictRunDirs(t *testing.T, volPath string) int { + t.Helper() + entries, err := os.ReadDir(filepath.Join(volPath, ConflictsDirName)) + if err != nil { + if os.IsNotExist(err) { + return 0 + } + t.Fatalf("read conflicts dir: %v", err) + } + return len(entries) +} + +// TestContestedFreezeStopsPingPong is the #158 acceptance at the /plan +// level: the first conflict preserves the loser and freezes the path; a +// later divergent re-assertion is refused with `contested` (no second +// copy minted); and the frozen winner re-asserting is still allowed +// through so a dropped /close can re-land it. +func TestContestedFreezeStopsPingPong(t *testing.T) { + ctx := context.Background() + f := newPreStageFixture(t) + + contentX := []byte("homepc's version, indexed locally on the hub") + contentZ := []byte("laptop's divergent version — wins the first conflict") + contentW := []byte("homepc re-asserts yet another edit") + f.seedLocalWrite(t, ctx, "doc.md", contentX) + + // Tick 1: laptop's divergent bytes conflict with the local write. + sess1 := f.freshSession(t, syncproto.ProtocolVersionContested) + plan1, err := f.router.planSession(ctx, sess1, []syncproto.IndexEntry{ + {Path: "doc.md", Blake3Hex: blakeHex(contentZ), SizeBytes: int64(len(contentZ))}, + }) + if err != nil { + t.Fatalf("planSession tick 1: %v", err) + } + if d := sess1.dispositions["doc.md"].disposition; d != syncproto.DispositionConflict { + t.Fatalf("tick 1 disposition = %q, want conflict", d) + } + if len(plan1.Conflicts) != 1 { + t.Fatalf("tick 1 conflicts = %d, want 1", len(plan1.Conflicts)) + } + if got := countConflictRunDirs(t, f.vol.Path); got != 1 { + t.Fatalf("conflict run dirs after tick 1 = %d, want 1", got) + } + // The path is now frozen with laptop's bytes as the winner. + latch, err := f.store.GetContestedPath(ctx, f.volID, "doc.md") + if err != nil { + t.Fatalf("GetContestedPath: %v", err) + } + if blakeHex(contentZ) != hexOrEmpty(latch.LiveBlake3) { + t.Fatalf("latch winner = %s, want hash(Z) %s", hexOrEmpty(latch.LiveBlake3), blakeHex(contentZ)) + } + + // Tick 2: homepc re-asserts a third version. The freeze refuses it — + // no bytes move, no second copy is minted (the F27 fix). + sess2 := f.freshSession(t, syncproto.ProtocolVersionContested) + plan2, err := f.router.planSession(ctx, sess2, []syncproto.IndexEntry{ + {Path: "doc.md", Blake3Hex: blakeHex(contentW), SizeBytes: int64(len(contentW))}, + }) + if err != nil { + t.Fatalf("planSession tick 2: %v", err) + } + if d := sess2.dispositions["doc.md"].disposition; d != syncproto.DispositionContested { + t.Fatalf("tick 2 disposition = %q, want contested", d) + } + if len(plan2.Conflicts) != 0 { + t.Fatalf("tick 2 conflicts = %d, want 0 (frozen, not re-conflicted)", len(plan2.Conflicts)) + } + if len(plan2.Contested) != 1 || plan2.Contested[0].Path != "doc.md" { + t.Fatalf("tick 2 contested = %+v, want one doc.md entry", plan2.Contested) + } + if plan2.Contested[0].PreservedAtPath == "" || plan2.Contested[0].LiveBlake3Hex != blakeHex(contentZ) { + t.Fatalf("tick 2 contested detail = %+v, want winner Z + preserved path", plan2.Contested[0]) + } + if got := countConflictRunDirs(t, f.vol.Path); got != 1 { + t.Fatalf("conflict run dirs after tick 2 = %d, want 1 (no new copy minted)", got) + } + + // Tick 3: the frozen winner re-asserts its own bytes — allowed through + // (Transfer), so a /close that never committed can re-land the winner. + sess3 := f.freshSession(t, syncproto.ProtocolVersionContested) + if _, err := f.router.planSession(ctx, sess3, []syncproto.IndexEntry{ + {Path: "doc.md", Blake3Hex: blakeHex(contentZ), SizeBytes: int64(len(contentZ))}, + }); err != nil { + t.Fatalf("planSession tick 3: %v", err) + } + if d := sess3.dispositions["doc.md"].disposition; d != syncproto.DispositionTransfer { + t.Fatalf("tick 3 disposition = %q, want transfer (winner re-asserting)", d) + } +} + +// TestContestedFreezeVersionGated confirms the disposition is gated: on +// one and the same frozen state, a session negotiated below +// ProtocolVersionContested falls back to the legacy `conflict` it +// understands, while a contested-capable session gets the freeze. Older +// peers stay safe. +func TestContestedFreezeVersionGated(t *testing.T) { + ctx := context.Background() + f := newPreStageFixture(t) + contentX := f.seedLocalWrite(t, ctx, "doc.md", []byte("hub-local content")) + + // Freeze doc.md while its local-write bytes stay live (the winner), so + // both sessions below classify against an identical present row. + if err := f.store.RaiseContested(ctx, store.ContestedPath{ + VolumeID: f.volID, Path: "doc.md", LiveBlake3: contentX, RaisedRunID: f.localRun, + }); err != nil { + t.Fatalf("RaiseContested: %v", err) + } + divergent := blakeHex([]byte("a divergent edit")) + + // An older initiator (Merkle-walk protocol) must get the legacy + // conflict, not the contested disposition it can't interpret. + legacy := f.freshSession(t, syncproto.ProtocolVersionMerkleWalk) + plan, err := f.router.planSession(ctx, legacy, []syncproto.IndexEntry{ + {Path: "doc.md", Blake3Hex: divergent, SizeBytes: 16}, + }) + if err != nil { + t.Fatalf("planSession legacy: %v", err) + } + if d := legacy.dispositions["doc.md"].disposition; d != syncproto.DispositionConflict { + t.Fatalf("legacy disposition = %q, want conflict (version gate)", d) + } + if len(plan.Contested) != 0 { + t.Fatalf("legacy plan.Contested = %+v, want empty for an older peer", plan.Contested) + } + + // A contested-capable initiator, same frozen state, gets refused. + modern := f.freshSession(t, syncproto.ProtocolVersionContested) + if _, err := f.router.planSession(ctx, modern, []syncproto.IndexEntry{ + {Path: "doc.md", Blake3Hex: divergent, SizeBytes: 16}, + }); err != nil { + t.Fatalf("planSession modern: %v", err) + } + if d := modern.dispositions["doc.md"].disposition; d != syncproto.DispositionContested { + t.Fatalf("modern disposition = %q, want contested (version gate)", d) + } +} diff --git a/agent/dispatcher.go b/agent/dispatcher.go new file mode 100644 index 0000000..a9708fa --- /dev/null +++ b/agent/dispatcher.go @@ -0,0 +1,212 @@ +package agent + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" +) + +// defaultMaxParallelSyncs bounds how many automatic syncs run at once +// across all destinations. Per-destination serialisation already caps +// parallelism at the number of configured destinations; this is the +// overall ceiling so a host with many destinations can't spawn one rclone +// per destination simultaneously and thrash its uplink and CPU. Four +// covers the reference household — a NAS pushing photos/docs/media to +// cloudbox + s3archive + kopia-mirror + htpc — without a LAN pair ever +// queueing behind a slow offsite, while still bounding a pathological +// config. +const defaultMaxParallelSyncs = 4 + +// syncDispatcher runs (volume, destination) syncs off the scheduler's tick +// goroutine so a slow or wedged destination can never delay the tick loop, +// index runs, peer syncs, or syncs to other destinations (#160, F25). +// +// The concurrency unit is the destination: each destination has a FIFO +// queue drained by a single worker, so at most one sync per destination is +// in flight while a second volume due to the same destination (photos and +// docs both push to cloudbox in the reference setup) waits its turn instead +// of being skipped and starved — issue #160 option 4b is per-destination +// queues, not skip-on-busy. An overall semaphore bounds how many +// destinations transfer at once. +// +// The pre-sync index ordering the scheduler guarantees is unaffected: the +// tick body still runs a volume's index to completion before it hands that +// volume's due syncs to the dispatcher. +type syncDispatcher struct { + run SyncRunner + logger *slog.Logger + now func() time.Time + + sem chan struct{} + + mu sync.Mutex + dests map[string]*destQueue + wg sync.WaitGroup +} + +// destQueue is one destination's serial pipeline: at most one sync active, +// the rest waiting in FIFO order. queued holds every volume name active or +// pending on this destination so a re-dispatch of a pair already in the +// pipeline — the scheduler re-evaluates a still-unfinished pair on every +// tick — is a quiet no-op rather than a duplicate enqueue. +type destQueue struct { + pending []*config.Volume + queued map[string]bool + active bool +} + +func newSyncDispatcher(run SyncRunner, logger *slog.Logger, now func() time.Time, maxParallel int) *syncDispatcher { + if maxParallel <= 0 { + maxParallel = defaultMaxParallelSyncs + } + return &syncDispatcher{ + run: run, + logger: logger, + now: now, + sem: make(chan struct{}, maxParallel), + dests: make(map[string]*destQueue), + } +} + +// dispatch enqueues the sync for (vol, destName) on destName's FIFO queue. +// An idle destination starts its worker immediately; a busy one takes the +// pair onto its queue to run when the current transfer finishes (deduped so +// re-evaluation on later ticks can't queue the same pair twice). It never +// blocks on the transfer — the tick loop returns at once, so a slow +// destination cannot delay any other. +func (d *syncDispatcher) dispatch(ctx context.Context, vol *config.Volume, destName string) { + if d.run == nil { + d.logger.Info("scheduler.skipped", + "kind", "sync", "volume", vol.Name, "destination", destName, + "reason", "sync runner not configured") + return + } + d.mu.Lock() + q := d.dests[destName] + if q == nil { + q = &destQueue{queued: make(map[string]bool)} + d.dests[destName] = q + } + if q.queued[vol.Name] { + d.mu.Unlock() // already active or queued for this destination + return + } + q.queued[vol.Name] = true + if q.active { + q.pending = append(q.pending, vol) + d.mu.Unlock() + return + } + q.active = true + d.mu.Unlock() + d.wg.Add(1) + go d.worker(ctx, destName, vol) +} + +// worker drains destName's queue one sync at a time, so the destination +// never has two transfers in flight. It exits — freeing the destination — +// when the queue empties; on shutdown (ctx cancelled) it drains without +// running, leaving the unfinished pairs to be re-evaluated next start. +func (d *syncDispatcher) worker(ctx context.Context, destName string, vol *config.Volume) { + defer d.wg.Done() + for vol != nil { + if ctx.Err() == nil { + d.runOne(ctx, vol, destName) + } + vol = d.next(destName, vol.Name) + } +} + +// inFlight reports whether (destName, volName) is currently active or queued +// on destName's pipeline. The scheduler consults it alongside the runs table +// so a pair whose dispatch from an earlier tick is still working — but whose +// run row is not yet visible — is recognised as in flight and not dragged +// through a redundant pre-sync index every tick (#160). Safe for concurrent +// use; a nil-runner dispatcher (no dests) always reports false. +func (d *syncDispatcher) inFlight(destName, volName string) bool { + d.mu.Lock() + defer d.mu.Unlock() + q := d.dests[destName] + if q == nil { + return false + } + return q.queued[volName] +} + +// next removes the just-finished volume from destName's pipeline and returns +// the next queued volume, or nil (dropping the now-idle destination) when +// none remain. +func (d *syncDispatcher) next(destName, done string) *config.Volume { + d.mu.Lock() + defer d.mu.Unlock() + q := d.dests[destName] + if q == nil { + return nil + } + delete(q.queued, done) + if len(q.pending) == 0 { + delete(d.dests, destName) + return nil + } + next := q.pending[0] + q.pending = q.pending[1:] + return next +} + +// runOne takes an overall parallelism slot (or bails on shutdown), runs the +// sync, and emits the kicked/finished/error logs. A sync that stalls is +// failed by the runner's transfer timeout (rclone is killed); its +// diagnosable error lands here and in the run row. +func (d *syncDispatcher) runOne(ctx context.Context, vol *config.Volume, destName string) { + select { + case d.sem <- struct{}{}: + defer func() { <-d.sem }() + case <-ctx.Done(): + return + } + if ctx.Err() != nil { + return + } + d.logger.Info("scheduler.kicked", + "kind", "sync", "volume", vol.Name, "destination", destName) + start := d.now() + rep := d.run(ctx, vol, destName) + duration := d.now().Sub(start) + status := rep.Status + if status == "" { + status = store.RunStatusFailed + } + d.logger.Info("scheduler.finished", + "kind", "sync", "volume", vol.Name, "destination", destName, + "run_id", rep.RunID, "status", status, + "duration_ms", duration.Milliseconds(), + "conflicts", rep.Conflicts, "contested", rep.Contested, + ) + // A conflict signal on this (initiating) machine, not just the hub: a + // divergent edit was preserved or a frozen path refused, and a human + // must eventually resolve it (#158, F27). Logged at warn so it stands + // out from the routine finished line in the scheduler's own output. + if rep.Conflicts > 0 || rep.Contested > 0 { + d.logger.Warn("scheduler.conflict", + "kind", "sync", "volume", vol.Name, "destination", destName, + "run_id", rep.RunID, + "conflicts", rep.Conflicts, "contested", rep.Contested, + ) + } + if rep.Err != nil { + d.logger.Error("scheduler.error", + "kind", "sync", "volume", vol.Name, "destination", destName, + "run_id", rep.RunID, "err", rep.Err.Error()) + } +} + +// wait blocks until every destination worker has drained. The scheduler +// calls it during shutdown, after the run context is cancelled (which kills +// any in-flight rclone child), so it returns promptly. +func (d *syncDispatcher) wait() { + d.wg.Wait() +} diff --git a/agent/dispatcher_test.go b/agent/dispatcher_test.go new file mode 100644 index 0000000..c795341 --- /dev/null +++ b/agent/dispatcher_test.go @@ -0,0 +1,151 @@ +package agent + +import ( + "bytes" + "context" + "log/slog" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" +) + +func discardLogger() *slog.Logger { return slog.New(slog.DiscardHandler) } + +// TestSyncDispatcherRunsDestinationsConcurrently proves the core F25 fix: +// two destinations run at the same time, so a slow offsite can never delay +// a LAN pair. Both runner invocations must be in flight before either is +// released. +func TestSyncDispatcherRunsDestinationsConcurrently(t *testing.T) { + var active, maxActive atomic.Int32 + entered := make(chan struct{}, 2) + release := make(chan struct{}) + run := func(ctx context.Context, vol *config.Volume, destName string) SyncRunReport { + n := active.Add(1) + for { + old := maxActive.Load() + if n <= old || maxActive.CompareAndSwap(old, n) { + break + } + } + entered <- struct{}{} + <-release + active.Add(-1) + return SyncRunReport{Status: store.RunStatusSuccess} + } + d := newSyncDispatcher(run, discardLogger(), time.Now, 4) + vol := &config.Volume{Name: "photos"} + d.dispatch(context.Background(), vol, "cloudbox") + d.dispatch(context.Background(), vol, "htpc") + <-entered // both must be running before we release either + <-entered + close(release) + d.wait() + if got := maxActive.Load(); got != 2 { + t.Fatalf("max concurrent syncs = %d; want 2 (different destinations run concurrently)", got) + } +} + +// TestSyncDispatcherSerializesSameDestination proves the concurrency unit +// is the destination: two volumes due to the same destination on one tick +// both run — the second is queued behind the first, never starved — while +// at most one is ever in flight against that destination. +func TestSyncDispatcherSerializesSameDestination(t *testing.T) { + var active, maxActive atomic.Int32 + var photosRan, docsRan atomic.Bool + entered := make(chan string, 2) + release := make(chan struct{}) // one send per sync, in completion order + run := func(ctx context.Context, vol *config.Volume, destName string) SyncRunReport { + n := active.Add(1) + for { + old := maxActive.Load() + if n <= old || maxActive.CompareAndSwap(old, n) { + break + } + } + switch vol.Name { + case "photos": + photosRan.Store(true) + case "docs": + docsRan.Store(true) + } + entered <- vol.Name + <-release + active.Add(-1) + return SyncRunReport{Status: store.RunStatusSuccess} + } + d := newSyncDispatcher(run, discardLogger(), time.Now, 4) + d.dispatch(context.Background(), &config.Volume{Name: "photos"}, "cloudbox") + d.dispatch(context.Background(), &config.Volume{Name: "docs"}, "cloudbox") + + first := <-entered // FIFO: photos was dispatched first + if first != "photos" { + t.Fatalf("first sync = %q; want photos (FIFO order)", first) + } + time.Sleep(20 * time.Millisecond) // give a wrongly-concurrent docs a chance to start + if got := active.Load(); got != 1 { + t.Fatalf("concurrent syncs to one destination = %d; want 1 (serialized)", got) + } + release <- struct{}{} // let photos finish; the worker then dequeues docs + if second := <-entered; second != "docs" { + t.Fatalf("second sync = %q; want docs (queued, not starved)", second) + } + release <- struct{}{} // let docs finish + d.wait() + + if !photosRan.Load() || !docsRan.Load() { + t.Fatalf("both volumes must run to the shared destination; photos=%v docs=%v", + photosRan.Load(), docsRan.Load()) + } + if got := maxActive.Load(); got != 1 { + t.Fatalf("max concurrent syncs to one destination = %d; want 1", got) + } +} + +// TestSyncDispatcherBoundsOverallParallelism proves the semaphore ceiling: +// with a single slot, a second destination waits for the first to free the +// slot rather than running immediately. +func TestSyncDispatcherBoundsOverallParallelism(t *testing.T) { + var bCalled atomic.Bool + aEntered := make(chan struct{}) + release := make(chan struct{}) + run := func(ctx context.Context, vol *config.Volume, destName string) SyncRunReport { + if destName == "a" { + close(aEntered) + <-release + } else { + bCalled.Store(true) + } + return SyncRunReport{Status: store.RunStatusSuccess} + } + d := newSyncDispatcher(run, discardLogger(), time.Now, 1) + vol := &config.Volume{Name: "v"} + d.dispatch(context.Background(), vol, "a") + <-aEntered // a holds the only slot + d.dispatch(context.Background(), vol, "b") + time.Sleep(20 * time.Millisecond) // give b's worker time to reach the slot wait + if bCalled.Load() { + t.Fatal("b ran while the single parallelism slot was held by a") + } + close(release) + d.wait() + if !bCalled.Load() { + t.Fatal("b never ran after the slot was freed") + } +} + +// TestSyncDispatcherNilRunnerSkips covers the index-only host: no runner +// wired, so a dispatch logs a skip and starts no worker. +func TestSyncDispatcherNilRunnerSkips(t *testing.T) { + buf := &bytes.Buffer{} + logger := slog.New(slog.NewTextHandler(buf, nil)) + d := newSyncDispatcher(nil, logger, time.Now, 4) + d.dispatch(context.Background(), &config.Volume{Name: "v"}, "dest") + d.wait() + if !strings.Contains(buf.String(), "sync runner not configured") { + t.Fatalf("expected a nil-runner skip log, got:\n%s", buf.String()) + } +} diff --git a/agent/scheduler.go b/agent/scheduler.go index 9c95b9e..4929e2f 100644 --- a/agent/scheduler.go +++ b/agent/scheduler.go @@ -30,7 +30,7 @@ const DefaultSchedulerTick = 30 * time.Second type scheduler struct { store *store.Store volumes map[string]*config.Volume - syncRun SyncRunner + dispatch *syncDispatcher logger *slog.Logger locks lockHolder tickEvery time.Duration @@ -74,7 +74,7 @@ func newScheduler(srv *Server, tickEvery time.Duration, now func() time.Time) *s return &scheduler{ store: srv.store, volumes: srv.cfg.Volumes, - syncRun: srv.cfg.SyncRunner, + dispatch: newSyncDispatcher(srv.cfg.SyncRunner, srv.cfg.Logger, now, defaultMaxParallelSyncs), logger: srv.cfg.Logger, locks: srv.router, tickEvery: tickEvery, @@ -197,9 +197,12 @@ func (s *scheduler) run(ctx context.Context) { for { select { case <-ctx.Done(): - // Drain in-flight hooks before returning so Serve's shutdown - // wait doesn't race a hook goroutine writing its outcome. + // Drain in-flight hooks and per-destination sync workers before + // returning so Serve's shutdown wait doesn't race a goroutine + // writing its outcome. ctx is already cancelled, so any rclone + // child has been killed and the workers return promptly. s.hooks.wait() + s.dispatch.wait() return case <-t.C: s.tick(ctx) @@ -333,42 +336,87 @@ func (s *scheduler) resolveVolume(ctx context.Context, name, absPath string) (st return created, nil } -// maybeRunSync evaluates the sync_every cadence and, if any destination -// is due, runs a pre-sync index and then the per-destination syncs. -// Returns true when the scheduler took any sync-related action (kicked, -// skipped, or errored) so the caller can suppress a redundant -// standalone-index check. +// maybeRunSync evaluates the sync_every cadence and, for the destinations +// that are due and not already syncing, runs a pre-sync index and then hands +// each pair to the per-destination dispatcher. Returns true when the +// scheduler took any sync-related action (kicked, skipped, or errored) so the +// caller can suppress a redundant standalone-index check. // // Invariant: a scheduled sync always runs an index immediately before -// pushing. If the pre-sync index fails or is skipped, no syncs run -// this tick — the next tick re-evaluates. +// pushing. If the pre-sync index fails or is skipped, no syncs run this tick +// — the next tick re-evaluates. +// +// A due destination whose (volume, destination) sync from an earlier tick is +// still in flight is dropped before the pre-sync index runs. Dispatch is +// asynchronous (#160), so without this the tick loop would re-index the source +// on every 30s tick for the entire duration of a slow multi-hour push — +// burning I/O and flooding the never-pruned runs audit trail with a +// kind='index' row per tick. When every due destination is already syncing, +// the pre-sync index is skipped entirely: there is no new push for it to +// precede, and the in-flight pushes were each already preceded by their own. func (s *scheduler) maybeRunSync(ctx context.Context, vol *config.Volume, volumeID int64) bool { if vol.SyncEvery == 0 { return false } - var dueDests []string + dueDests := s.dueSyncDests(ctx, vol, volumeID) + if len(dueDests) == 0 { + return false + } + dispatchable := s.dispatchableSyncDests(ctx, vol, volumeID, dueDests) + if len(dispatchable) == 0 { + // Every due destination is already syncing (each skip logged per + // pair). Report the sync action so the standalone-index branch stays + // suppressed — otherwise its own re-index would reintroduce the exact + // per-tick flood this guard exists to prevent. + return true + } + if !s.maybeRunIndex(ctx, vol, volumeID, "pre-sync", 0) { + return true + } + for _, destName := range dispatchable { + s.dispatch.dispatch(ctx, vol, destName) + } + return true +} + +// dueSyncDests returns the destinations on the volume's sync_to whose +// sync_every cadence has elapsed. A per-destination lookup error is logged and +// that destination is dropped from this tick's evaluation (the next tick +// re-evaluates), never aborting the others. +func (s *scheduler) dueSyncDests(ctx context.Context, vol *config.Volume, volumeID int64) []string { + var due []string for _, destName := range vol.SyncTo { - due, err := s.syncDue(ctx, volumeID, destName, vol.SyncEvery) + ok, err := s.syncDue(ctx, volumeID, destName, vol.SyncEvery) if err != nil { s.logger.Error("scheduler.error", "kind", "sync", "volume", vol.Name, "destination", destName, "err", err.Error()) continue } - if due { - dueDests = append(dueDests, destName) + if ok { + due = append(due, destName) } } - if len(dueDests) == 0 { - return false - } - if !s.maybeRunIndex(ctx, vol, volumeID, "pre-sync", 0) { - return true - } + return due +} + +// dispatchableSyncDests filters dueDests to the pairs the dispatcher can start +// now, dropping any whose sync is already in flight and logging the same +// per-pair scheduler.skipped the serial scheduler emitted — so an operator +// still sees why a due destination isn't moving, it just no longer drags a +// redundant pre-sync index along with it. +func (s *scheduler) dispatchableSyncDests(ctx context.Context, vol *config.Volume, volumeID int64, dueDests []string) []string { + var dispatchable []string for _, destName := range dueDests { - s.runSync(ctx, vol, volumeID, destName) + if s.syncInFlight(ctx, vol, volumeID, destName) { + s.logger.Info("scheduler.skipped", + "kind", "sync", "volume", vol.Name, "destination", destName, + "reason", "in-flight sync run") + continue + } + dispatchable = append(dispatchable, destName) } - return true + return dispatchable } // syncDue computes (now - last_finished) ≥ cadence for the (volume, @@ -562,56 +610,30 @@ func indexRunStatus(rep index.Report, err error) (string, bool) { return store.RunStatusSuccess, true } -// runSync kicks one (volume, destination) sync via the configured -// SyncRunner. The pre-check against the runs table is the in-flight -// detection the issue specifies; the SyncRunner's downstream call -// (sync.RunPair) adds its own atomic BeginSyncRunIfClear gate, so a -// race that sneaks past the pre-check still produces a clean skip -// rather than a duplicate run. -// -// A nil SyncRunner surfaces as a clean skip rather than an error so -// an agent running pure index-only schedules can omit the sync wiring -// (and the rclone dependency that comes with it) without the -// scheduler logging at error level on each tick. -func (s *scheduler) runSync(ctx context.Context, vol *config.Volume, volumeID int64, destName string) { +// syncInFlight reports whether a sync of this exact (volume, destination) pair +// is already active — a stale row, a concurrent CLI `squirrel sync`, or (the +// common case) a dispatch from an earlier tick still working. It consults two +// sources so neither a not-yet-visible run row nor a just-drained queue slot +// can fool it: the dispatcher's in-memory queue (set the instant a pair is +// enqueued, before its run row exists) and the runs table (authoritative once +// the row is written, and the signal that also catches a stale row or a +// concurrent CLI sync). The downstream dispatch and sync.RunPair's atomic +// BeginSyncRunIfClear gate remain the final guard against a duplicate run, so +// this check only needs to be good enough to suppress a redundant pre-sync +// index. A store error is treated as in-flight so a transient DB hiccup skips +// this tick rather than racing a duplicate push. +func (s *scheduler) syncInFlight(ctx context.Context, vol *config.Volume, volumeID int64, destName string) bool { + if s.dispatch.inFlight(destName, vol.Name) { + return true + } running, err := s.store.HasRunningRun(ctx, store.RunKindSync, volumeID, destName) if err != nil { s.logger.Error("scheduler.error", "kind", "sync", "volume", vol.Name, "destination", destName, "err", err.Error()) - return - } - if running { - s.logger.Info("scheduler.skipped", - "kind", "sync", "volume", vol.Name, "destination", destName, - "reason", "in-flight sync run") - return - } - if s.syncRun == nil { - s.logger.Info("scheduler.skipped", - "kind", "sync", "volume", vol.Name, "destination", destName, - "reason", "sync runner not configured") - return - } - s.logger.Info("scheduler.kicked", - "kind", "sync", "volume", vol.Name, "destination", destName) - start := s.now() - rep := s.syncRun(ctx, vol, destName) - duration := s.now().Sub(start) - status := rep.Status - if status == "" { - status = store.RunStatusFailed - } - s.logger.Info("scheduler.finished", - "kind", "sync", "volume", vol.Name, "destination", destName, - "run_id", rep.RunID, "status", status, - "duration_ms", duration.Milliseconds(), - ) - if rep.Err != nil { - s.logger.Error("scheduler.error", - "kind", "sync", "volume", vol.Name, "destination", destName, - "run_id", rep.RunID, "err", rep.Err.Error()) + return true } + return running } // evaluateVerify walks every destination with a resolved verify cadence diff --git a/agent/scheduler_test.go b/agent/scheduler_test.go index 0bb0f09..f5ba43c 100644 --- a/agent/scheduler_test.go +++ b/agent/scheduler_test.go @@ -124,10 +124,14 @@ func (f *fakeSyncRunner) Calls() []syncCall { // + fake sync runner. The scheduler's tick method drives behaviour // directly; the goroutine-driven run() loop is only used in the // integration test below. -func newSchedulerFixture(t *testing.T, volumeCfg *config.Volume) *schedulerFixture { +func newSchedulerFixture(t *testing.T, vols ...*config.Volume) *schedulerFixture { t.Helper() - if volumeCfg.Path == "" { - volumeCfg.Path = t.TempDir() + volumes := make(map[string]*config.Volume, len(vols)) + for _, v := range vols { + if v.Path == "" { + v.Path = t.TempDir() + } + volumes[v.Name] = v } dbPath := filepath.Join(t.TempDir(), "index.db") s, err := store.Open(dbPath) @@ -144,7 +148,7 @@ func newSchedulerFixture(t *testing.T, volumeCfg *config.Volume) *schedulerFixtu Listen: "127.0.0.1:0", Token: "tok", Version: "test", - Volumes: map[string]*config.Volume{volumeCfg.Name: volumeCfg}, + Volumes: volumes, Logger: logger, SyncRunner: syncRunner.Runner(), }, s) @@ -174,7 +178,7 @@ func (f *schedulerFixture) scheduler() *scheduler { return &scheduler{ store: f.srv.store, volumes: f.srv.cfg.Volumes, - syncRun: f.srv.cfg.SyncRunner, + dispatch: newSyncDispatcher(f.srv.cfg.SyncRunner, f.srv.cfg.Logger, f.clock.Now, defaultMaxParallelSyncs), logger: f.srv.cfg.Logger, locks: f.srv.router, tickEvery: time.Second, @@ -203,7 +207,6 @@ func (f *schedulerFixture) seedFile() { if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { f.t.Fatalf("write %s: %v", p, err) } - return } } @@ -319,6 +322,7 @@ func TestSchedulerSyncRunsPreSyncIndexFirst(t *testing.T) { ctx := context.Background() sch.tick(ctx) + sch.dispatch.wait() // syncs run on per-destination workers; drain them runs, err := f.store.ListRuns(ctx, store.ListRunsOpts{}) if err != nil { @@ -421,9 +425,12 @@ func TestSchedulerSkipsWhenInFlightIndexRun(t *testing.T) { } } -// TestSchedulerSkipsWhenInFlightSyncRun plants a stale running sync -// row and asserts the scheduler skips the sync (but, per the -// invariant, the pre-sync index still ran in this tick). We use a +// TestSchedulerSkipsWhenInFlightSyncRun plants a stale running sync row and +// asserts the scheduler skips the sync — and, since that pair is the volume's +// only due destination, skips the pre-sync index too (#160). Re-indexing the +// source on every tick for the whole duration of an in-flight push would burn +// I/O and flood the never-pruned runs audit trail with a kind='index' row per +// tick; the in-flight push was already preceded by its own index. We use a // fake sync runner so we can verify it never gets invoked. func TestSchedulerSkipsWhenInFlightSyncRun(t *testing.T) { f := newSchedulerFixture(t, &config.Volume{ @@ -446,6 +453,7 @@ func TestSchedulerSkipsWhenInFlightSyncRun(t *testing.T) { sch := f.scheduler() sch.tick(context.Background()) + sch.dispatch.wait() if got := len(f.syncLog.Calls()); got != 0 { t.Fatalf("sync invoked %d times despite running sync row; want 0", got) @@ -453,25 +461,47 @@ func TestSchedulerSkipsWhenInFlightSyncRun(t *testing.T) { if !f.containsLogLine("scheduler.skipped", "kind=sync", "in-flight") { t.Fatalf("missing in-flight sync skip log:\n%s", f.logBuf.String()) } - // Pre-sync index still ran — the invariant says sync always - // indexes first, and the only thing skipped here is the sync - // itself. + // No new pre-sync index: with the sole due destination already in + // flight, there is no new push for an index to precede. Only the planted + // 'running' sync row should exist. runs, _ := f.store.ListRuns(context.Background(), store.ListRunsOpts{}) - var sawIndex, sawSync bool - for _, r := range runs { - switch r.Kind { - case store.RunKindIndex: - sawIndex = true - case store.RunKindSync: - sawSync = true - } + if len(runs) != 1 || runs[0].Kind != store.RunKindSync || runs[0].Status != store.RunStatusRunning { + t.Fatalf("runs=%+v; want only the planted running sync row (no redundant pre-sync index)", runs) } - if !sawIndex { - t.Fatalf("expected a pre-sync index row even when sync is gated: %+v", runs) +} + +// TestSchedulerTwoVolumesSameDestinationBothRun guards the starvation trap +// (#160): the tick iterates volumes by name, so if two volumes are due to +// the same destination on one tick the earlier one wins the destination. +// The dispatcher must queue the later one behind it and run it — not skip +// it every tick forever. "aaa" sorts before "zzz"; both must produce a sync +// run to "backup". +func TestSchedulerTwoVolumesSameDestinationBothRun(t *testing.T) { + f := newSchedulerFixture(t, + &config.Volume{Name: "aaa", SyncTo: []string{"backup"}, SyncEvery: time.Hour}, + &config.Volume{Name: "zzz", SyncTo: []string{"backup"}, SyncEvery: time.Hour}, + ) + f.seedFile() + sch := f.scheduler() + + sch.tick(context.Background()) + sch.dispatch.wait() + + var aaaRan, zzzRan bool + for _, c := range f.syncLog.Calls() { + if c.Destination != "backup" { + continue + } + switch c.Volume { + case "aaa": + aaaRan = true + case "zzz": + zzzRan = true + } } - if !sawSync { - // The planted 'running' sync row counts as a sync; just sanity. - t.Fatalf("expected the planted running sync row to remain: %+v", runs) + if !aaaRan || !zzzRan { + t.Fatalf("both volumes must sync to the shared destination; aaa=%v zzz=%v (calls=%+v)", + aaaRan, zzzRan, f.syncLog.Calls()) } } @@ -526,6 +556,7 @@ func TestSchedulerCadenceUsesLastFinished(t *testing.T) { // Tick 1: nothing prior → fires. sch.tick(ctx) + sch.dispatch.wait() if got := len(f.syncLog.Calls()); got != 1 { t.Fatalf("after tick 1: sync calls = %d; want 1", got) } @@ -533,6 +564,7 @@ func TestSchedulerCadenceUsesLastFinished(t *testing.T) { // Advance 30min, still under cadence → does not fire. f.clock.Add(30 * time.Minute) sch.tick(ctx) + sch.dispatch.wait() if got := len(f.syncLog.Calls()); got != 1 { t.Fatalf("after tick 2 (under cadence): sync calls = %d; want 1", got) } @@ -540,11 +572,157 @@ func TestSchedulerCadenceUsesLastFinished(t *testing.T) { // Advance another hour, past cadence → fires again. f.clock.Add(time.Hour) sch.tick(ctx) + sch.dispatch.wait() if got := len(f.syncLog.Calls()); got != 2 { t.Fatalf("after tick 3 (over cadence): sync calls = %d; want 2", got) } } +// waitUntil polls cond until it returns true or a short deadline passes, +// failing the test on timeout. It synchronises with the dispatcher's async +// workers without a fixed sleep, so a test can wait for one destination's +// queue to drain while another destination's sync is still (deliberately) +// blocked. +func waitUntil(t *testing.T, cond func() bool, what string) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +// TestSchedulerInFlightSyncNotReindexedEachTick is the #160 regression guard: +// once a sync is dispatched and stays in flight across ticks, the scheduler +// must NOT re-run that volume's pre-sync index every tick — otherwise a slow +// multi-hour push would flood the never-pruned runs audit trail with a +// kind='index' row per 30s tick. The pre-sync index runs exactly once (on the +// tick that dispatches the sync), while a different, not-in-flight +// (volume, destination) keeps indexing and dispatching normally on the very +// ticks the wedged one is skipped — proving the guard is per-pair, not global. +func TestSchedulerInFlightSyncNotReindexedEachTick(t *testing.T) { + f := newSchedulerFixture(t, + &config.Volume{Name: "photos", SyncTo: []string{"cloudbox"}, SyncEvery: time.Hour}, + &config.Volume{Name: "docs", SyncTo: []string{"backup"}, SyncEvery: time.Hour}, + ) + f.seedFile() + + release := make(chan struct{}) + photosInFlight := make(chan struct{}, 1) + docsFinished := make(chan struct{}, 8) + f.syncLog.runFn = func(ctx context.Context, vol *config.Volume, destName string) SyncRunReport { + v, err := f.store.GetVolumeByName(ctx, vol.Name) + if err != nil { + return SyncRunReport{Err: err} + } + id, blocker, err := f.store.BeginSyncRunIfClear(ctx, store.SyncRunSpec{ + VolumeID: v.ID, Destination: destName, + }) + if err != nil { + return SyncRunReport{Err: err} + } + if blocker != nil { + return SyncRunReport{RunID: blocker.ID, Err: fmt.Errorf("already running")} + } + if vol.Name == "photos" { + // Announce the running row exists, then block — keeping the + // photos->cloudbox sync in flight across the following ticks + // without ever finishing it (the F25 wedge, minus the kill). + photosInFlight <- struct{}{} + <-release + } + if err := f.store.FinishRun(ctx, id, store.RunStatusSuccess, "", 0); err != nil { + return SyncRunReport{RunID: id, Err: err} + } + if vol.Name == "docs" { + docsFinished <- struct{}{} + } + return SyncRunReport{RunID: id, Status: store.RunStatusSuccess} + } + + sch := f.scheduler() + ctx := context.Background() + + countIndex := func(volName string) int { + t.Helper() + v, err := f.store.GetVolumeByName(ctx, volName) + if err != nil { + t.Fatalf("GetVolumeByName(%s): %v", volName, err) + } + runs, err := f.store.ListRuns(ctx, store.ListRunsOpts{}) + if err != nil { + t.Fatalf("ListRuns: %v", err) + } + n := 0 + for _, r := range runs { + if r.Kind == store.RunKindIndex && r.VolumeID.Valid && r.VolumeID.Int64 == v.ID { + n++ + } + } + return n + } + // drainDocs waits for docs' just-dispatched sync to finish and its queue + // slot to clear, so the next tick sees docs as not-in-flight while photos + // stays wedged. + drainDocs := func(tick int) { + <-docsFinished + waitUntil(t, func() bool { return !sch.dispatch.inFlight("backup", "docs") }, + fmt.Sprintf("docs queue to drain after tick %d", tick)) + } + + // Tick 1: both due, neither in flight -> each indexes once and dispatches. + // photos' sync blocks (stays running); docs' sync finishes. + sch.tick(ctx) + <-photosInFlight + drainDocs(1) + if got := countIndex("photos"); got != 1 { + t.Fatalf("after tick 1: photos index runs = %d; want 1", got) + } + if got := countIndex("docs"); got != 1 { + t.Fatalf("after tick 1: docs index runs = %d; want 1", got) + } + + // Ticks 2 and 3: photos is still in flight -> skipped with NO new + // pre-sync index. docs comes due again each hour and must keep indexing + + // dispatching normally on the very ticks photos is skipped. + for tick := 2; tick <= 3; tick++ { + f.clock.Add(time.Hour + time.Minute) + sch.tick(ctx) + drainDocs(tick) + if got := countIndex("photos"); got != 1 { + t.Fatalf("after tick %d: photos index runs = %d; want 1 (in-flight sync must not be re-indexed)", tick, got) + } + if got := countIndex("docs"); got != tick { + t.Fatalf("after tick %d: docs index runs = %d; want %d (not-in-flight volume indexes each due tick)", tick, got, tick) + } + } + + var photosCalls, docsCalls int + for _, c := range f.syncLog.Calls() { + switch c.Volume { + case "photos": + photosCalls++ + case "docs": + docsCalls++ + } + } + if photosCalls != 1 { + t.Fatalf("photos sync calls = %d; want 1 (dispatched once, then skipped while in flight)", photosCalls) + } + if docsCalls != 3 { + t.Fatalf("docs sync calls = %d; want 3 (once per due tick)", docsCalls) + } + if !f.containsLogLine("scheduler.skipped", "kind=sync", "volume=photos", "in-flight") { + t.Fatalf("missing in-flight skip log for photos:\n%s", f.logBuf.String()) + } + + close(release) + sch.dispatch.wait() +} + // TestSchedulerFailedSyncStillConsumesCadence guards the failure // policy ("Failed runs aren't specially retried — the next tick // re-evaluates"). After a failed sync, the next within-cadence tick @@ -571,6 +749,7 @@ func TestSchedulerFailedSyncStillConsumesCadence(t *testing.T) { ctx := context.Background() sch.tick(ctx) + sch.dispatch.wait() if calls.Load() != 1 { t.Fatalf("tick 1 sync calls = %d; want 1", calls.Load()) } @@ -578,6 +757,7 @@ func TestSchedulerFailedSyncStillConsumesCadence(t *testing.T) { // Within cadence → no re-fire even though the prior run failed. f.clock.Add(10 * time.Minute) sch.tick(ctx) + sch.dispatch.wait() if calls.Load() != 1 { t.Fatalf("under-cadence tick after failure called sync %d times; want 1 (no special retry)", calls.Load()) @@ -586,6 +766,7 @@ func TestSchedulerFailedSyncStillConsumesCadence(t *testing.T) { // Past cadence → re-fires (regardless of prior failure). f.clock.Add(time.Hour) sch.tick(ctx) + sch.dispatch.wait() if calls.Load() != 2 { t.Fatalf("past-cadence tick after failure called sync %d times; want 2", calls.Load()) } @@ -607,6 +788,7 @@ func TestSchedulerBothCadencesShareIndexRun(t *testing.T) { ctx := context.Background() sch.tick(ctx) + sch.dispatch.wait() runs, _ := f.store.ListRuns(ctx, store.ListRunsOpts{}) var indexCount int @@ -636,11 +818,13 @@ func TestSchedulerStandaloneIndexAfterSyncCadenceWaits(t *testing.T) { ctx := context.Background() sch.tick(ctx) // fires sync + pre-sync index. + sch.dispatch.wait() // 10min later → under index_every from the pre-sync index, no // standalone fires. f.clock.Add(10 * time.Minute) sch.tick(ctx) + sch.dispatch.wait() runs, _ := f.store.ListRuns(ctx, store.ListRunsOpts{}) var indexCount int for _, r := range runs { @@ -656,6 +840,7 @@ func TestSchedulerStandaloneIndexAfterSyncCadenceWaits(t *testing.T) { // due yet because sync_every is 1h). f.clock.Add(10 * time.Minute) // total 20min since pre-sync. sch.tick(ctx) + sch.dispatch.wait() runs, _ = f.store.ListRuns(ctx, store.ListRunsOpts{}) indexCount = 0 for _, r := range runs { @@ -801,9 +986,12 @@ func TestSchedulerDropsSyncWhenRunnerNil(t *testing.T) { }) f.seedFile() sch := f.scheduler() - sch.syncRun = nil + // A dispatcher with no runner is the index-only-host case (no rclone + // wired): sync-triggering paths log a skip and create no sync rows. + sch.dispatch = newSyncDispatcher(nil, f.srv.cfg.Logger, f.clock.Now, defaultMaxParallelSyncs) sch.tick(context.Background()) + sch.dispatch.wait() if !f.containsLogLine("scheduler.skipped", "kind=sync", "sync runner not configured") { t.Fatalf("missing nil-runner skip log:\n%s", f.logBuf.String()) diff --git a/agent/sync.go b/agent/sync.go index 29d6a7f..83a2fda 100644 --- a/agent/sync.go +++ b/agent/sync.go @@ -401,7 +401,7 @@ func (r *peerSyncRouter) finishBegin(ctx context.Context, body syncproto.BeginRe // know is clamped down to ProtocolVersionMerkleWalk rather than // rejected, so a partial rollout doesn't break syncs. func negotiateProtocol(requested int) int { - const receiverMax = syncproto.ProtocolVersionMerkleWalk + const receiverMax = syncproto.ProtocolVersionContested if requested <= 0 { return syncproto.ProtocolVersionFlat } @@ -574,9 +574,53 @@ func (r *peerSyncRouter) planSession(ctx context.Context, sess *peerSession, ent }) } resp.Conflicts = collectConflicts(sess) + contested, err := r.collectContested(ctx, sess, entries) + if err != nil { + return syncproto.PlanResponse{}, err + } + resp.Contested = contested return resp, nil } +// collectContested builds the wire-format contested list from the entries +// classified as `contested`, in the order the initiator sent them. Each +// path is frozen, so its contested_paths latch supplies the two versions +// and the preserved-loser location the initiator mirrors into its own +// marker and renders. A latch that vanished between classify and here +// (an operator resolved it mid-plan) is skipped rather than erroring — +// the next sync re-plans cleanly. +func (r *peerSyncRouter) collectContested(ctx context.Context, sess *peerSession, entries []syncproto.IndexEntry) ([]syncproto.ContestedDetail, error) { + var out []syncproto.ContestedDetail + for _, e := range entries { + if sess.dispositions[e.Path].disposition != syncproto.DispositionContested { + continue + } + latch, err := r.srv.store.GetContestedPath(ctx, sess.volumeID, e.Path) + if err != nil { + if store.IsNotFound(err) { + continue + } + return nil, fmt.Errorf("look up contested %s: %w", e.Path, err) + } + out = append(out, syncproto.ContestedDetail{ + Path: e.Path, + LiveBlake3Hex: hexOrEmpty(latch.LiveBlake3), + PreservedBlake3Hex: hexOrEmpty(latch.PreservedBlake3), + PreservedAtPath: latch.PreservedAtPath, + }) + } + return out, nil +} + +// hexOrEmpty renders a raw digest as lowercase hex, or "" for a nil digest +// (the "unknown winner" latch case), keeping the wire field omittable. +func hexOrEmpty(b []byte) string { + if len(b) == 0 { + return "" + } + return hex.EncodeToString(b) +} + // handlePlanFolders implements POST /v1/sync/plan-folders: look up the // receiver's folder rows for each requested path and return the // (shallow, deep, direct children) tuple. The endpoint is the @@ -804,21 +848,31 @@ func collectConflicts(sess *peerSession) []syncproto.ConflictDetail { // strategy is "off" (initiator opted out). func (r *peerSyncRouter) classify(ctx context.Context, sess *peerSession, relPath string, entry *sessionEntry) (string, error) { existing, err := r.srv.store.GetByPath(ctx, sess.volumeID, relPath) - if err != nil { - if store.IsNotFound(err) { - return r.classifyMissingPath(ctx, sess, entry) - } + liveFound := err == nil && existing.Status == store.StatusPresent + if err != nil && !store.IsNotFound(err) { return "", err } - if existing.Status != store.StatusPresent { - // A live row in 'missing' status is treated as "no file here"; - // the initiator's bytes are new content. Dedup still applies — - // the bytes may exist at another path. - return r.classifyMissingPath(ctx, sess, entry) - } - if bytesEqual(existing.Blake3, entry.blake3) { + if liveFound && bytesEqual(existing.Blake3, entry.blake3) { return syncproto.DispositionAlreadyCorrect, nil } + // The initiator's bytes differ from any live row here (or there is no + // live row). Before deciding Supersede/Conflict/Transfer, check the + // contested freeze: a path frozen by a prior conflict refuses a + // divergent re-assertion from any peer, so the flip-flop stops at the + // first conflict instead of minting a fresh copy every cadence tick. + // The frozen winner re-asserting its own digest is *not* refused — it + // falls through so a /close that never committed can re-establish it. + if contested, err := r.contestedRefusal(ctx, sess, relPath, entry); err != nil { + return "", err + } else if contested { + return syncproto.DispositionContested, nil + } + if !liveFound { + // No live row (or a 'missing'/'offloaded' one): the initiator's + // bytes are new content here. Dedup still applies — the bytes may + // exist at another path. + return r.classifyMissingPath(ctx, sess, entry) + } // Different blake3 at this path. Provenance decides the verdict // (Supersede / Conflict). Content at a path takes precedence over // a cross-path dedup — we don't let dedup paper over a divergence @@ -831,6 +885,28 @@ func (r *peerSyncRouter) classify(ctx context.Context, sess *peerSession, relPat return disp, nil } +// contestedRefusal reports whether relPath is frozen and this initiator's +// bytes should be refused with the `contested` disposition. It returns +// false — letting normal classification proceed — when the path isn't +// frozen, when the initiator's digest matches the frozen winner (the +// winner re-asserting, which must be allowed to re-land), or when the +// session negotiated a protocol older than ProtocolVersionContested (an +// initiator that can't interpret the disposition falls back to the legacy +// `conflict`, keeping older peers safe per the version gate). +func (r *peerSyncRouter) contestedRefusal(ctx context.Context, sess *peerSession, relPath string, entry *sessionEntry) (bool, error) { + if sess.protocolVersion < syncproto.ProtocolVersionContested { + return false, nil + } + liveBlake3, contested, err := r.srv.store.IsPathContested(ctx, sess.volumeID, relPath) + if err != nil { + return false, err + } + if !contested { + return false, nil + } + return !bytesEqual(entry.blake3, liveBlake3), nil +} + // classifyMissingPath is the no-live-row branch of classify: try to // satisfy the path from existing content elsewhere in the volume // (CopyFromExisting), or fall back to Transfer. Strategy "off" skips @@ -1188,6 +1264,22 @@ func (r *peerSyncRouter) preStageConflicts(ctx context.Context, sess *peerSessio return fmt.Errorf("record conflict pre-stage for %s: %w", path, err) } entry.preservedAtPath = preservedRel + // Freeze the path so the next cadence tick refuses a divergent + // re-assertion instead of minting another `.squirrel-conflicts/` + // copy (#158, F27). The winner is the initiator's bytes (landing + // live on /close); the loser is the just-preserved prior row. + // Idempotent: a path already frozen keeps its original latch. + if err := r.srv.store.RaiseContested(ctx, store.ContestedPath{ + VolumeID: sess.volumeID, + Path: path, + LiveBlake3: entry.blake3, + PreservedBlake3: entry.priorRow.Blake3, + PreservedAtPath: preservedRel, + PeerNodeID: sess.peerNodeID, + RaisedRunID: sess.receiverRunID, + }); err != nil { + return fmt.Errorf("raise contested for %s: %w", path, err) + } } return nil } diff --git a/cmd/squirrel/agent.go b/cmd/squirrel/agent.go index ebda9ee..f787923 100644 --- a/cmd/squirrel/agent.go +++ b/cmd/squirrel/agent.go @@ -49,6 +49,9 @@ func runAgent(cmd *cobra.Command) error { defer s.Close() logger := slog.New(slog.NewTextHandler(cmd.ErrOrStderr(), &slog.HandlerOptions{Level: slog.LevelInfo})) + if err := reapOrphanedRunsAtStartup(cmd.Context(), s, logger); err != nil { + return err + } rcl, err := resolveSchedulerRclone(cmd, cfg) if err != nil { return err @@ -106,6 +109,25 @@ func serveAgent(cmd *cobra.Command, cfg *config.Config, srv *agent.Server, logge return srv.Serve(cmd.Context(), ln) } +// reapOrphanedRunsAtStartup transitions any 'running' run left behind by a +// previously-killed agent to 'aborted' before the schedulers start (#157, +// F14). A freshly started agent has kicked nothing yet, so every 'running' +// row necessarily predates it; leaving them would block the run gates +// forever and render as a live, elapsed-ticking banner in the TUI hours +// later. The reap is loud — one info line naming the reaped ids — because +// automatic recovery must never be invisible (ux-principle 5). +func reapOrphanedRunsAtStartup(ctx context.Context, s *store.Store, logger *slog.Logger) error { + ids, err := s.AbortRunningRuns(ctx, "reaped at agent startup: the agent that owned this run was killed mid-run") + if err != nil { + return fmt.Errorf("reap orphaned runs: %w", err) + } + if len(ids) > 0 { + logger.Warn("reaped orphaned runs", + "count", len(ids), "run_ids", ids, "status", store.RunStatusAborted) + } + return nil +} + // logSchedulerOnlyStartup emits the listener-less counterpart of the // "agent listening" banner so a journal shows the agent came up in // scheduler-only mode with no bound port. @@ -179,6 +201,10 @@ func resolveSchedulerRclone(cmd *cobra.Command, cfg *config.Config) (*sync.Rclon if err != nil { return nil, fmt.Errorf("scheduler needs rclone for scheduled syncs/verifies: %w", err) } + // Bound every automatic transfer by the no-progress guard so a wedged + // endpoint fails its own run instead of hanging forever (#160, F25). + // Foreground `squirrel sync` leaves this unset — a human can interrupt. + rcl.StallTimeout = sync.DefaultStallTimeout // The version preflight (`--hash blake3`) is a sync concern; a // verify-only schedule reads provider checksums and doesn't need it. if needsSync { @@ -263,7 +289,13 @@ func buildSchedulerSyncRunner(cfg *config.Config, s *store.Store, rcl *sync.Rclo opts.Snapshot = sync.NewSnapshotter(s, rcl, snapshotConfig(cfg, s.Path())) } rep, runErr := sync.RunPair(ctx, s, tools, pair, opts) - return agent.SyncRunReport{RunID: rep.RunID, Status: rep.Status, Err: runErr} + return agent.SyncRunReport{ + RunID: rep.RunID, + Status: rep.Status, + Err: runErr, + Conflicts: len(rep.NodeConflicts), + Contested: len(rep.NodeContested), + } } } diff --git a/cmd/squirrel/conflicts.go b/cmd/squirrel/conflicts.go new file mode 100644 index 0000000..b4344a7 --- /dev/null +++ b/cmd/squirrel/conflicts.go @@ -0,0 +1,134 @@ +package main + +import ( + "database/sql" + "encoding/hex" + "fmt" + "io" + "text/tabwriter" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/store" +) + +// newConflictsCmd returns the `squirrel conflicts` command: the question +// half of the contested-freeze pair (#158, F27). It lists the paths this +// node knows are frozen by a peer-sync conflict — divergent edits that +// stopped ping-ponging once the receiver refused to keep minting +// `.squirrel-conflicts/` copies. Both versions stay preserved; the listing +// shows which is live, which is preserved and where, when the freeze +// landed, and which peer it involves. The change half — `conflicts +// resolve` — is the explicit human act that clears a freeze. +func newConflictsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "conflicts", + Short: "List unresolved contested paths (frozen peer-sync conflicts)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return runConflicts(cmd) + }, + } + cmd.AddCommand(newConflictsResolveCmd()) + return cmd +} + +func runConflicts(cmd *cobra.Command) error { + cfg, err := tryLoadConfig(cmd) + if err != nil { + return err + } + s, err := openStore(cmd, cfg) + if err != nil { + return err + } + defer s.Close() + + contested, err := s.ListContestedPaths(cmd.Context()) + if err != nil { + return fmt.Errorf("list contested paths: %w", err) + } + out := cmd.OutOrStdout() + if len(contested) == 0 { + fmt.Fprintln(out, "no contested paths — nothing frozen") + return nil + } + volumes, err := loadVolumeNames(cmd, s) + if err != nil { + return err + } + nodes := loadContestedNodeNames(cmd, s, contested) + return printContested(out, contested, volumes, nodes) +} + +// loadContestedNodeNames resolves the peer_node_id of every contested row +// to a name. A dropped or unattributed (zero) id is simply omitted; the +// renderer falls back to "—". Best-effort: a lookup error skips the id +// rather than failing the whole listing. +func loadContestedNodeNames(cmd *cobra.Command, s *store.Store, rows []store.ContestedPath) map[int64]string { + out := map[int64]string{} + for _, c := range rows { + if c.PeerNodeID == 0 { + continue + } + if _, ok := out[c.PeerNodeID]; ok { + continue + } + node, err := s.GetNodeByID(cmd.Context(), c.PeerNodeID) + if err != nil { + continue + } + out[c.PeerNodeID] = node.Name + } + return out +} + +func printContested(w io.Writer, rows []store.ContestedPath, volumes, nodes map[int64]string) error { + fmt.Fprintf(w, "%d contested path(s) — resolve each with `squirrel conflicts resolve `:\n\n", len(rows)) + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, "VOLUME\tPATH\tLIVE\tPRESERVED\tPRESERVED AT\tSINCE\tPEER") + for _, c := range rows { + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + volumeLabel(sql.NullInt64{Int64: c.VolumeID, Valid: true}, volumes), + c.Path, + shortDigest(c.LiveBlake3), + shortDigest(c.PreservedBlake3), + dashIfEmpty(c.PreservedAtPath), + formatStarted(c.RaisedAtNs), + contestedPeerLabel(c.PeerNodeID, nodes), + ) + } + return tw.Flush() +} + +// contestedPeerLabel names the peer a freeze involves, or "—" when +// unattributed (zero id) or the node row has vanished. +func contestedPeerLabel(peerNodeID int64, nodes map[int64]string) string { + if peerNodeID == 0 { + return "—" + } + if name, ok := nodes[peerNodeID]; ok { + return name + } + return fmt.Sprintf("id=%d", peerNodeID) +} + +// shortDigest renders the first 12 hex chars of a digest for a compact +// column, or "—" for an unknown (nil) digest. +func shortDigest(b []byte) string { + if len(b) == 0 { + return "—" + } + h := hex.EncodeToString(b) + if len(h) > 12 { + return h[:12] + } + return h +} + +func dashIfEmpty(s string) string { + if s == "" { + return "—" + } + return s +} diff --git a/cmd/squirrel/conflicts_resolve.go b/cmd/squirrel/conflicts_resolve.go new file mode 100644 index 0000000..2cd05a9 --- /dev/null +++ b/cmd/squirrel/conflicts_resolve.go @@ -0,0 +1,96 @@ +package main + +import ( + "encoding/hex" + "fmt" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/store" +) + +// newConflictsResolveCmd returns `squirrel conflicts resolve +// `: the explicit human act that clears a contested freeze (#158, +// F27). Resolution is deliberately a human decision — squirrel never +// resolves a conflict on its own ("irreversible acts remain human"). The +// command clears the standing freeze so syncs flow again; it does not +// itself move bytes between the two versions. Both remain preserved: the +// winner live at the path, the loser under `.squirrel-conflicts/`. To +// adopt the *other* version, restore the preserved copy first — the output +// names exactly where it is. +// +// Clearing loses no history: every conflict run survives in the runs +// table, the raise and this clear survive in runs_audit, and the preserved +// bytes stay on disk. If a peer still holds the divergent version, its +// next sync will re-freeze the path — a fresh, singly-preserved, human- +// notified event, not the unbounded ping-pong F27 described. +func newConflictsResolveCmd() *cobra.Command { + return &cobra.Command{ + Use: "resolve ", + Short: "Clear a contested freeze so syncs flow again (explicit human act)", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return runConflictsResolve(cmd, args[0], args[1]) + }, + } +} + +func runConflictsResolve(cmd *cobra.Command, volumeName, relPath string) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + s, err := openStore(cmd, cfg) + if err != nil { + return err + } + defer s.Close() + + vol, err := s.GetVolumeByName(cmd.Context(), volumeName) + if err != nil { + if store.IsNotFound(err) { + return fmt.Errorf("no volume named %q", volumeName) + } + return fmt.Errorf("lookup volume %q: %w", volumeName, err) + } + latch, err := s.GetContestedPath(cmd.Context(), vol.ID, relPath) + if err != nil { + if store.IsNotFound(err) { + return fmt.Errorf("path %q in volume %q is not contested", relPath, volumeName) + } + return fmt.Errorf("look up contested %q: %w", relPath, err) + } + cleared, err := s.ClearContested(cmd.Context(), vol.ID, relPath, ackOperator()) + if err != nil { + return err + } + if !cleared { + // Raced with another resolve between the read and the write — the + // freeze is gone either way, the outcome the operator asked for. + return fmt.Errorf("path %q in volume %q is not contested", relPath, volumeName) + } + printResolved(cmd, volumeName, relPath, latch) + return nil +} + +// printResolved confirms the clear and points the operator at both +// preserved versions so adopting the non-live one stays a deliberate, +// informed act rather than a surprise the resolve silently performed. +func printResolved(cmd *cobra.Command, volumeName, relPath string, latch store.ContestedPath) { + out := cmd.OutOrStdout() + fmt.Fprintf(out, "resolved %s %s: contested freeze cleared — syncs may flow again\n", volumeName, relPath) + fmt.Fprintf(out, " live version kept: %s\n", digestOrUnknown(latch.LiveBlake3)) + if latch.PreservedAtPath != "" { + fmt.Fprintf(out, " preserved version %s remains at %s (restore it to adopt that version instead)\n", + digestOrUnknown(latch.PreservedBlake3), latch.PreservedAtPath) + } +} + +// digestOrUnknown renders a digest as full hex, or a placeholder when the +// latch recorded no digest (an older or partial freeze record). +func digestOrUnknown(b []byte) string { + if len(b) == 0 { + return "(unknown)" + } + return hex.EncodeToString(b) +} diff --git a/cmd/squirrel/conflicts_test.go b/cmd/squirrel/conflicts_test.go new file mode 100644 index 0000000..dd49e19 --- /dev/null +++ b/cmd/squirrel/conflicts_test.go @@ -0,0 +1,137 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mbertschler/squirrel/store" +) + +// raiseTestContested opens the CLI's DB directly and freezes one path, so +// the conflicts commands have a standing latch to read and clear without +// driving a whole peer sync. The store handle is closed before returning +// so the next CLI invocation has no concurrent connection from this +// process. +func raiseTestContested(t *testing.T, dbPath, volumeName, relPath string) { + t.Helper() + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{}) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer s.Close() + ctx := context.Background() + vol, err := s.GetVolumeByName(ctx, volumeName) + if err != nil { + t.Fatalf("GetVolumeByName %q: %v", volumeName, err) + } + run, err := s.BeginIndexRun(ctx, store.RunKindIndex, vol.ID, false) + if err != nil { + t.Fatalf("BeginIndexRun: %v", err) + } + if err := s.RaiseContested(ctx, store.ContestedPath{ + VolumeID: vol.ID, + Path: relPath, + LiveBlake3: make([]byte, 32), + PreservedBlake3: make([]byte, 32), + PreservedAtPath: ".squirrel-conflicts/run-" + relPath, + RaisedRunID: run, + }); err != nil { + t.Fatalf("RaiseContested: %v", err) + } +} + +// TestCLIConflictsListAndResolve walks the question→change pair: an empty +// listing, a frozen path surfacing in the list, and an explicit resolve +// that clears it so the list empties again. +func TestCLIConflictsListAndResolve(t *testing.T) { + src := t.TempDir() + writeTestFile(t, filepath.Join(src, "a.txt"), "hello") + f := writeConfigFor(t, map[string]string{"src": src}) + runCLI(t, "--config", f.configPath, "index", "src") + + // Nothing frozen yet. + if out := runCLI(t, "--config", f.configPath, "conflicts"); !strings.Contains(out, "no contested paths") { + t.Fatalf("empty listing = %q, want 'no contested paths'", out) + } + + raiseTestContested(t, f.dbPath, "src", "a.txt") + + out := runCLI(t, "--config", f.configPath, "conflicts") + if !strings.Contains(out, "a.txt") || !strings.Contains(out, "1 contested path") { + t.Fatalf("listing = %q, want the frozen a.txt", out) + } + + // Resolve clears it and reports what was kept. + out = runCLI(t, "--config", f.configPath, "conflicts", "resolve", "src", "a.txt") + if !strings.Contains(out, "contested freeze cleared") { + t.Fatalf("resolve output = %q, want cleared confirmation", out) + } + + if out := runCLI(t, "--config", f.configPath, "conflicts"); !strings.Contains(out, "no contested paths") { + t.Fatalf("listing after resolve = %q, want empty", out) + } +} + +// TestCLIRunsContestedReminder covers the reminder beneath `squirrel runs`: +// it names volumes (never the internal id) and, under --volume, scopes to +// that volume rather than listing freezes on unrelated ones. +func TestCLIRunsContestedReminder(t *testing.T) { + tmp := t.TempDir() + dirA := filepath.Join(tmp, "a") + dirB := filepath.Join(tmp, "b") + for _, d := range []string{dirA, dirB} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(d, "f.txt"), "content") + } + f := writeConfigFor(t, map[string]string{"a": dirA, "b": dirB}) + runCLI(t, "--config", f.configPath, "index", "a") + runCLI(t, "--config", f.configPath, "index", "b") + raiseTestContested(t, f.dbPath, "a", "f.txt") + raiseTestContested(t, f.dbPath, "b", "f.txt") + + // Unfiltered: both volumes appear, rendered by name not numeric id. + out := runCLI(t, "--config", f.configPath, "runs") + if !strings.Contains(out, "2 contested path(s)") { + t.Fatalf("unfiltered reminder = %q, want both freezes", out) + } + if !strings.Contains(out, "volume=a ") || !strings.Contains(out, "volume=b ") { + t.Fatalf("reminder did not name both volumes: %q", out) + } + if strings.Contains(out, "volume=1") || strings.Contains(out, "volume=2") { + t.Fatalf("reminder leaked a numeric volume id: %q", out) + } + + // Filtered to volume a: only a's freeze is reminded. + out = runCLI(t, "--config", f.configPath, "runs", "--volume", "a") + if !strings.Contains(out, "1 contested path(s)") || !strings.Contains(out, "volume=a ") { + t.Fatalf("filtered reminder = %q, want only volume a", out) + } + if strings.Contains(out, "volume=b ") { + t.Fatalf("filtered reminder leaked volume b: %q", out) + } +} + +// TestCLIConflictsResolveUnknown: resolving a path that isn't contested is +// a clean error, not a silent no-op. +func TestCLIConflictsResolveUnknown(t *testing.T) { + src := t.TempDir() + writeTestFile(t, filepath.Join(src, "a.txt"), "hello") + f := writeConfigFor(t, map[string]string{"src": src}) + runCLI(t, "--config", f.configPath, "index", "src") + + _, err := runCLIExpectErr(t, "--config", f.configPath, "conflicts", "resolve", "src", "nope.txt") + if err == nil || !strings.Contains(err.Error(), "is not contested") { + t.Fatalf("error = %v, want 'is not contested'", err) + } + + // An unknown volume is a distinct, legible error. + _, err = runCLIExpectErr(t, "--config", f.configPath, "conflicts", "resolve", "ghost", "a.txt") + if err == nil || !strings.Contains(err.Error(), "no volume named") { + t.Fatalf("error = %v, want 'no volume named'", err) + } +} diff --git a/cmd/squirrel/index.go b/cmd/squirrel/index.go index 1172c60..e17e50f 100644 --- a/cmd/squirrel/index.go +++ b/cmd/squirrel/index.go @@ -2,10 +2,13 @@ package main import ( "fmt" + "time" "github.com/spf13/cobra" + "github.com/mbertschler/squirrel/config" "github.com/mbertschler/squirrel/index" + "github.com/mbertschler/squirrel/store" ) // newIndexCmd returns the `squirrel index ` cobra command. The @@ -55,6 +58,11 @@ func runIndex(cmd *cobra.Command, volumeName string, progress bool, opts index.O } defer s.Close() + // Captured before the walk (so it reflects the prior run, not this + // one): its presence tells a first index from a re-index, and its + // timestamp feeds the catch-up note. + priorNs, hadPrior := priorIndexRun(cmd, s, vol.Name) + // Progress renders to stderr so a redirected stdout still captures only // the final summary line. The line is erased right after Index returns // (both paths) so any error list and the summary print on a clean line. @@ -79,10 +87,67 @@ func runIndex(cmd *cobra.Command, volumeName string, progress bool, opts index.O if opts.DryRun { prefix = "(dry-run) " } - fmt.Fprintf(cmd.OutOrStdout(), "%sadded=%d modified=%d unchanged=%d missing=%d errors=%d\n", - prefix, rep.Added, rep.Modified, rep.Unchanged, rep.Missing, rep.Errors) + // Naming the volume keeps back-to-back index runs legible instead of + // three anonymous count lines (friction F11a). + fmt.Fprintf(cmd.OutOrStdout(), "%s%s: added=%d modified=%d unchanged=%d missing=%d errors=%d\n", + prefix, vol.Name, rep.Added, rep.Modified, rep.Unchanged, rep.Missing, rep.Errors) + printIndexAdvisories(cmd, vol, rep, priorNs, hadPrior) if rep.Errors > 0 { return fmt.Errorf("encountered %d error(s) during walk", rep.Errors) } return nil } + +// printIndexAdvisories surfaces the two affirmative-feedback lines the +// friction walk asked for: an empty-path warning on a first index (F8) +// and a catch-up note after a burst of new files following a quiet gap +// (F24). Both are advisory — neither changes the exit code. +func printIndexAdvisories(cmd *cobra.Command, vol *config.Volume, rep index.Report, priorNs int64, hadPrior bool) { + if !hadPrior && rep.Errors == 0 && !rep.SawFiles() { + fmt.Fprintf(cmd.ErrOrStderr(), + "warning: volume %q at %s: no files found — new volume, empty directory, or wrong mount?\n", + vol.Name, vol.Path) + return + } + if hadPrior && rep.SawFiles() { + if note, ok := catchUpNote(rep.Added, time.Since(time.Unix(0, priorNs))); ok { + fmt.Fprintln(cmd.OutOrStdout(), note) + } + } +} + +// priorIndexRun returns the start time of the most recent successful index +// run of the volume, and whether one exists. Best-effort: any lookup miss +// (volume never indexed, store error) reports "no prior", which is the safe +// default for both advisories. +func priorIndexRun(cmd *cobra.Command, s *store.Store, volName string) (int64, bool) { + v, err := s.GetVolumeByName(cmd.Context(), volName) + if err != nil { + return 0, false + } + run, err := s.LatestSuccessfulIndexRun(cmd.Context(), v.ID) + if err != nil { + return 0, false + } + return run.StartedAtNs, true +} + +// Catch-up heuristic thresholds: a burst of new files after a long quiet +// gap is the moment squirrel earns the most trust (a trip's worth of +// photos landing in one tick), and it should not blend into routine churn. +// The bounds are deliberately conservative so ordinary cadence runs stay +// silent. +const ( + catchUpMinFiles = 25 + catchUpMinQuiet = 7 * 24 * time.Hour +) + +// catchUpNote returns a one-line "N new files after D days" summary when a +// run added enough files after a long enough quiet gap, and whether it +// applies (friction F24). +func catchUpNote(added int, quiet time.Duration) (string, bool) { + if added < catchUpMinFiles || quiet < catchUpMinQuiet { + return "", false + } + return fmt.Sprintf("catch-up: %d new files after %d days of quiet", added, int(quiet.Hours())/24), true +} diff --git a/cmd/squirrel/main.go b/cmd/squirrel/main.go index 223fab0..bea55e0 100644 --- a/cmd/squirrel/main.go +++ b/cmd/squirrel/main.go @@ -2,16 +2,33 @@ package main import ( "context" + "errors" + "fmt" "os" "os/signal" "syscall" ) +// exitCodeError carries a specific process exit status out of a command's +// RunE. `squirrel status` uses it to report its scriptable green/amber/red +// contract (exit 0/1/2), which the generic "any error ⇒ exit 1" path below +// cannot express. errors.As recovers the code here; the command that +// returns it silences cobra's error printing so nothing but the code +// leaves the process. +type exitCodeError struct{ code int } + +func (e exitCodeError) Error() string { return fmt.Sprintf("exit status %d", e.code) } + func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - if err := newRootCmd().ExecuteContext(ctx); err != nil { + err := newRootCmd().ExecuteContext(ctx) + var ec exitCodeError + if errors.As(err, &ec) { + os.Exit(ec.code) + } + if err != nil { os.Exit(1) } } diff --git a/cmd/squirrel/offload.go b/cmd/squirrel/offload.go index 56b31e3..b90d817 100644 --- a/cmd/squirrel/offload.go +++ b/cmd/squirrel/offload.go @@ -76,7 +76,7 @@ func runOffload(cmd *cobra.Command, volumeName string, paths []string, olderThan RequireDests: requiredDestinations(cfg, vol.OffloadRequires), RelayedCaps: relayedCaps, MaxEvidenceAge: vol.OffloadMaxEvidenceAge, - VerifyCadenced: verifyCadencedTargets(cfg, vol.OffloadRequires), + VerifyCadenced: cfg.VerifyCadencedTargets(vol.OffloadRequires), DryRun: dryRun, }) printOffloadReport(cmd.OutOrStdout(), cmd.ErrOrStderr(), rep, dryRun) @@ -105,40 +105,6 @@ func requiredDestinations(cfg *config.Config, require []string) map[string]*conf return out } -// verifyCadencedTargets marks the required targets that carry an effective -// local verify cadence — a per-destination verify_every, or the [agent] -// verify_every default applied to a content-addressed/packed destination. -// The offload gate accepts a locally-advanced fingerprint-verified -// component as content-verified only for a target in this set, so the -// provider-fingerprint evidence keeps being re-confirmed for as long as -// deletions are permitted (issue #155). A required target with no local -// destination — a peer-relayed offsite this node cannot see — is omitted; -// the responder's cadence governs it, relayed and enforced at pull time. -func verifyCadencedTargets(cfg *config.Config, require []string) map[string]bool { - var agentDefault time.Duration - if cfg.Agent != nil { - agentDefault = cfg.Agent.VerifyEvery - } - out := make(map[string]bool, len(require)) - for _, name := range require { - d, ok := cfg.Destinations[name] - if !ok { - continue - } - if d.Layout != config.LayoutContentAddressed && d.Layout != config.LayoutPacked { - continue - } - eff := d.VerifyEvery - if eff == 0 { - eff = agentDefault - } - if eff > 0 { - out[name] = true - } - } - return out -} - // gatherRelayedOffloadCaps does the best-effort peer half of the offload // capability pre-check (#145): for each peer-relayed required target (an // offload_requires name that is neither a local destination nor a local diff --git a/cmd/squirrel/polish_test.go b/cmd/squirrel/polish_test.go new file mode 100644 index 0000000..627b0ae --- /dev/null +++ b/cmd/squirrel/polish_test.go @@ -0,0 +1,159 @@ +package main + +import ( + "database/sql" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/mbertschler/squirrel/store" +) + +// TestRunNeedsAttention pins the --failed predicate: only the non-clean +// terminal states qualify (F19). +func TestRunNeedsAttention(t *testing.T) { + cases := map[string]bool{ + store.RunStatusFailed: true, + store.RunStatusRefused: true, + store.RunStatusAborted: true, + store.RunStatusPartial: true, + store.RunStatusSuccess: false, + store.RunStatusRunning: false, + } + for status, want := range cases { + if got := runNeedsAttention(status); got != want { + t.Errorf("runNeedsAttention(%q) = %v, want %v", status, got, want) + } + } +} + +// TestRunIsInteresting pins the --changes predicate: clean successes that +// touched nothing fold; anything that moved files, resolved a conflict, or +// needs attention shows (F19). +func TestRunIsInteresting(t *testing.T) { + conflicts := map[int64]int{7: 2} + tests := []struct { + name string + r store.Run + want bool + }{ + {"clean no-op", store.Run{ID: 1, Status: store.RunStatusSuccess, FileCount: 0}, false}, + {"success moved files", store.Run{ID: 2, Status: store.RunStatusSuccess, FileCount: 5}, true}, + {"failed", store.Run{ID: 3, Status: store.RunStatusFailed}, true}, + {"refused", store.Run{ID: 4, Status: store.RunStatusRefused}, true}, + {"success with conflict", store.Run{ID: 7, Status: store.RunStatusSuccess, FileCount: 0}, true}, + } + for _, tc := range tests { + if got := runIsInteresting(tc.r, conflicts); got != tc.want { + t.Errorf("%s: runIsInteresting = %v, want %v", tc.name, got, tc.want) + } + } +} + +// TestFilterRuns exercises both filters end to end, including that the +// descending order is preserved. +func TestFilterRuns(t *testing.T) { + runs := []store.Run{ + {ID: 5, Status: store.RunStatusSuccess, FileCount: 0}, // no-op + {ID: 4, Status: store.RunStatusSuccess, FileCount: 3}, // change + {ID: 3, Status: store.RunStatusFailed}, // failed + {ID: 2, Status: store.RunStatusRefused}, // refused + {ID: 1, Status: store.RunStatusSuccess, FileCount: 0}, // no-op + } + conflicts := map[int64]int{} + + if ids := runIDs(filterRuns(runs, conflicts, true, false)); !sameIDs(ids, []int64{3, 2}) { + t.Fatalf("--failed ids = %v, want [3 2]", ids) + } + if ids := runIDs(filterRuns(runs, conflicts, false, true)); !sameIDs(ids, []int64{4, 3, 2}) { + t.Fatalf("--changes ids = %v, want [4 3 2]", ids) + } +} + +// TestPeerDirectionArrow / TestPeerLabelDirection cover F18: the initiator +// records runs.shallow, the receiver leaves it NULL, and the arrow follows. +func TestPeerDirectionArrow(t *testing.T) { + if got := peerDirectionArrow(store.Run{Shallow: sql.NullBool{Valid: true}}); got != "→" { + t.Errorf("initiator arrow = %q, want →", got) + } + if got := peerDirectionArrow(store.Run{Shallow: sql.NullBool{}}); got != "←" { + t.Errorf("receiver arrow = %q, want ←", got) + } +} + +func TestPeerLabelDirection(t *testing.T) { + nodes := map[int64]string{9: "htpc"} + out := peerLabel(store.Run{PeerNodeID: sql.NullInt64{Int64: 9, Valid: true}, Shallow: sql.NullBool{Valid: true}}, nodes) + if !strings.HasPrefix(out, "→ htpc") { + t.Errorf("outbound peer label = %q, want prefix %q", out, "→ htpc") + } + in := peerLabel(store.Run{PeerNodeID: sql.NullInt64{Int64: 9, Valid: true}}, nodes) + if !strings.HasPrefix(in, "← htpc") { + t.Errorf("inbound peer label = %q, want prefix %q", in, "← htpc") + } +} + +// TestCatchUpNote pins the F24 heuristic and its rendering. +func TestCatchUpNote(t *testing.T) { + if _, ok := catchUpNote(catchUpMinFiles, catchUpMinQuiet); !ok { + t.Fatal("expected catch-up at the thresholds") + } + note, ok := catchUpNote(404, 21*24*time.Hour) + if !ok || !strings.Contains(note, "404 new files") || !strings.Contains(note, "21 days") { + t.Fatalf("catch-up note = %q ok=%v", note, ok) + } + if _, ok := catchUpNote(catchUpMinFiles-1, catchUpMinQuiet); ok { + t.Error("below file threshold should not trigger") + } + if _, ok := catchUpNote(catchUpMinFiles, catchUpMinQuiet-time.Hour); ok { + t.Error("below quiet threshold should not trigger") + } +} + +// TestCLIIndexNamesVolume covers F11a: the index summary names the volume. +func TestCLIIndexNamesVolume(t *testing.T) { + src := t.TempDir() + writeTestFile(t, filepath.Join(src, "a.txt"), "hello") + f := writeConfigFor(t, map[string]string{"docs": src}) + out := runCLI(t, "--config", f.configPath, "index", "docs") + if !strings.Contains(out, "docs: added=1") { + t.Fatalf("index summary should name the volume: %q", out) + } +} + +// TestCLIIndexEmptyVolumeWarns covers F8: a first index of an empty tree +// warns, and a subsequent populated re-index does not. +func TestCLIIndexEmptyVolumeWarns(t *testing.T) { + src := t.TempDir() // empty + f := writeConfigFor(t, map[string]string{"docs": src}) + out := runCLI(t, "--config", f.configPath, "index", "docs") + if !strings.Contains(out, "no files found") { + t.Fatalf("empty first index should warn: %q", out) + } + writeTestFile(t, filepath.Join(src, "a.txt"), "hi") + out2 := runCLI(t, "--config", f.configPath, "index", "docs") + if strings.Contains(out2, "no files found") { + t.Fatalf("populated re-index should not warn: %q", out2) + } +} + +func runIDs(runs []store.Run) []int64 { + out := make([]int64, len(runs)) + for i, r := range runs { + out[i] = r.ID + } + return out +} + +func sameIDs(a, b []int64) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/cmd/squirrel/restore.go b/cmd/squirrel/restore.go index f92c1fa..9931c37 100644 --- a/cmd/squirrel/restore.go +++ b/cmd/squirrel/restore.go @@ -100,7 +100,9 @@ func runRestore(cmd *cobra.Command, volumeName, fromName string, opts sync.Resto } rep, runErr := sync.Restore(cmd.Context(), s, rcl, vol, dest, opts) - printSyncReport(out, rep, runErr) + // Restore pulls destination → local, so flip the arrow: the sync-shaped + // "volume → destination" would misreport the byte-flow direction. + printSyncReport(out, rep, runErr, true) if runErr != nil || rep.Status != "success" { return fmt.Errorf("restore did not complete cleanly") } diff --git a/cmd/squirrel/restore_test.go b/cmd/squirrel/restore_test.go index b83b67f..5d6d25a 100644 --- a/cmd/squirrel/restore_test.go +++ b/cmd/squirrel/restore_test.go @@ -43,6 +43,11 @@ func TestCLIRestoreRoundTrip(t *testing.T) { if !strings.Contains(out, "status=success") { t.Fatalf("restore did not succeed:\n%s", out) } + // The restore arrow flips to destination → volume: bytes flow that + // way, unlike a sync's volume → destination (restore-arrow friction). + if !strings.Contains(out, "scratch → pics") { + t.Fatalf("restore arrow should read destination → volume:\n%s", out) + } for _, name := range []string{"a.txt", "b.txt"} { path := filepath.Join(f.volumeDir, name) body, err := os.ReadFile(path) diff --git a/cmd/squirrel/root.go b/cmd/squirrel/root.go index b327e77..e48a89f 100644 --- a/cmd/squirrel/root.go +++ b/cmd/squirrel/root.go @@ -46,12 +46,14 @@ func newRootCmd() *cobra.Command { root.AddCommand(newRunsCmd()) root.AddCommand(newHooksCmd()) root.AddCommand(newVolumesCmd()) + root.AddCommand(newStatusCmd()) root.AddCommand(newSyncCmd()) root.AddCommand(newRestoreCmd()) root.AddCommand(newOffloadCmd()) root.AddCommand(newAgentCmd()) root.AddCommand(newAuditCmd()) root.AddCommand(newVerifyCmd()) + root.AddCommand(newConflictsCmd()) root.AddCommand(newPeerSyncCmd()) root.AddCommand(newTUICmd()) root.AddCommand(newDBCmd()) diff --git a/cmd/squirrel/runs.go b/cmd/squirrel/runs.go index 0bbf08b..c3e3f70 100644 --- a/cmd/squirrel/runs.go +++ b/cmd/squirrel/runs.go @@ -15,29 +15,35 @@ import ( // newRunsCmd returns the `squirrel runs` cobra command. It lists rows from // the runs table, most-recent first, with optional volume filtering and a -// configurable cap. Runs of every status (running, success, failed, partial) -// are surfaced so the user can see in-flight or failed indexing alongside -// successful ones. +// configurable cap. Runs of every kind (index, sync, audit, restore, +// offload) and every status (running, success, failed, partial, refused, +// aborted) are surfaced so in-flight, failed, or refused work shows up +// alongside successful runs. --failed and --changes narrow steady-state +// noise (friction F19). func newRunsCmd() *cobra.Command { var ( - volumeName string - limit int + volumeName string + limit int + onlyFailed bool + onlyChanges bool ) cmd := &cobra.Command{ Use: "runs", - Short: "List index runs (most recent first)", + Short: "List runs of every kind (most recent first)", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - return runRuns(cmd, volumeName, limit) + return runRuns(cmd, volumeName, limit, onlyFailed, onlyChanges) }, } cmd.Flags().StringVar(&volumeName, "volume", "", "filter to runs against this volume name") cmd.Flags().IntVar(&limit, "limit", 20, "maximum number of runs to show (0 for no limit)") + cmd.Flags().BoolVar(&onlyFailed, "failed", false, "show only runs that need attention (failed, refused, aborted, or partial)") + cmd.Flags().BoolVar(&onlyChanges, "changes", false, "hide clean no-op runs; show only runs that moved content or need attention") cmd.AddCommand(newRunsFailCmd()) return cmd } -func runRuns(cmd *cobra.Command, volumeName string, limit int) error { +func runRuns(cmd *cobra.Command, volumeName string, limit int, onlyFailed, onlyChanges bool) error { cfg, err := tryLoadConfig(cmd) if err != nil { return err @@ -48,18 +54,11 @@ func runRuns(cmd *cobra.Command, volumeName string, limit int) error { } defer s.Close() - opts := store.ListRunsOpts{Limit: limit, Descending: true} - if volumeName != "" { - v, err := s.GetVolumeByName(cmd.Context(), volumeName) - if err != nil { - if store.IsNotFound(err) { - return fmt.Errorf("no volume named %q", volumeName) - } - return fmt.Errorf("lookup volume %q: %w", volumeName, err) - } - opts.VolumeID = &v.ID + volumeID, err := resolveVolumeFilter(cmd, s, volumeName) + if err != nil { + return err } - runs, err := s.ListRuns(cmd.Context(), opts) + runs, conflicts, err := gatherRuns(cmd, s, volumeID, limit, onlyFailed, onlyChanges) if err != nil { return err } @@ -71,11 +70,193 @@ func runRuns(cmd *cobra.Command, volumeName string, limit int) error { if err != nil { return err } + if err := printRuns(cmd.OutOrStdout(), runs, volumes, nodes, conflicts); err != nil { + return err + } + alarms, err := s.ListDestinationAlarms(cmd.Context()) + if err != nil { + return fmt.Errorf("list destination alarms: %w", err) + } + printActiveAlarms(cmd.OutOrStdout(), alarms) + contested, err := s.ListContestedPaths(cmd.Context()) + if err != nil { + return fmt.Errorf("list contested paths: %w", err) + } + printContestedReminder(cmd.OutOrStdout(), contested, volumeID, volumes) + return nil +} + +// printContestedReminder surfaces standing contested freezes beneath the +// runs listing (#158, F27). Like the alarms reminder, a single old +// conflict run scrolls away, so the audit surface repeats the standing +// condition until a human resolves it — and points at the dedicated +// `squirrel conflicts` question-command for the detail. When the listing +// was scoped with --volume (filterVolumeID non-nil), the reminder is +// scoped to match rather than reporting freezes on unrelated volumes. +// Volumes render by name via the shared label map, never by internal id. +func printContestedReminder(w io.Writer, contested []store.ContestedPath, filterVolumeID *int64, volumes map[int64]string) { + scoped := contested + if filterVolumeID != nil { + scoped = scoped[:0:0] + for _, c := range contested { + if c.VolumeID == *filterVolumeID { + scoped = append(scoped, c) + } + } + } + if len(scoped) == 0 { + return + } + fmt.Fprintf(w, "\n%d contested path(s) frozen — inspect with `squirrel conflicts`, clear with `squirrel conflicts resolve`:\n", + len(scoped)) + for _, c := range scoped { + fmt.Fprintf(w, " CONTESTED volume=%s %s since %s\n", + volumeLabel(sql.NullInt64{Int64: c.VolumeID, Valid: true}, volumes), + c.Path, formatStarted(c.RaisedAtNs)) + } +} + +// printActiveAlarms surfaces standing per-destination alarms beneath the +// runs listing (#157, F30). A verify mismatch latches an alarm that must +// stay visible until an operator clears it — a single old run row scrolls +// away, so the audit surface repeats the standing condition on every +// `squirrel runs` until it is resolved. +func printActiveAlarms(w io.Writer, alarms []store.DestinationAlarm) { + if len(alarms) == 0 { + return + } + fmt.Fprintf(w, "\n%d destination(s) in alarm — clear with a clean `squirrel verify` or `squirrel verify ack `:\n", + len(alarms)) + for _, a := range alarms { + fmt.Fprintf(w, " ALARM %s (%s) since %s, run %d: %s\n", + a.Destination, a.Kind, + formatStarted(a.RaisedAtNs), a.RaisedRunID, a.Detail) + } +} + +// resolveVolumeFilter resolves an optional --volume name to its id, shared by +// the runs query and the standing-condition reminders printed beneath the +// listing so both are scoped the same way. An empty name means every volume +// and yields nil. +func resolveVolumeFilter(cmd *cobra.Command, s *store.Store, volumeName string) (*int64, error) { + if volumeName == "" { + return nil, nil + } + v, err := s.GetVolumeByName(cmd.Context(), volumeName) + if err != nil { + if store.IsNotFound(err) { + return nil, fmt.Errorf("no volume named %q", volumeName) + } + return nil, fmt.Errorf("lookup volume %q: %w", volumeName, err) + } + return &v.ID, nil +} + +// gatherRuns fetches runs under an already-resolved volume filter, applies the +// --failed/--changes predicates, and returns the rows to display plus the +// conflict counts for their CONFLICTS column. When a predicate is active the +// fetch is unbounded (opts.Limit=0) so the SQL LIMIT can't cap before the +// predicate runs and silently drop matches; the result is truncated to limit +// after filtering instead. +// +// Conflict counts are loaded only where they are actually needed, never +// across the whole unbounded fetch: --failed needs none for filtering, and +// the CONFLICTS column only covers the displayed rows. This keeps the id set +// handed to the count query small (and the query itself batches ids), so a +// large peer-sync history can't overflow SQLite's bound-parameter cap. +func gatherRuns(cmd *cobra.Command, s *store.Store, volumeID *int64, limit int, onlyFailed, onlyChanges bool) ([]store.Run, map[int64]int, error) { + filtering := onlyFailed || onlyChanges + opts := store.ListRunsOpts{Limit: limit, Descending: true, VolumeID: volumeID} + if filtering { + opts.Limit = 0 + } + runs, err := s.ListRuns(cmd.Context(), opts) + if err != nil { + return nil, nil, err + } + if filtering { + filterConflicts, err := conflictCountsForFilter(cmd, s, runs, onlyFailed) + if err != nil { + return nil, nil, err + } + runs = filterRuns(runs, filterConflicts, onlyFailed, onlyChanges) + if limit > 0 && len(runs) > limit { + runs = runs[:limit] + } + } + // Load the CONFLICTS-column counts for exactly the displayed rows. conflicts, err := loadConflictCounts(cmd, s, runs) if err != nil { - return err + return nil, nil, err + } + return runs, conflicts, nil +} + +// conflictCountsForFilter loads only the conflict counts the active filter +// needs. --failed consults none — an attention-worthy run is unconditionally +// kept — so it returns an empty map. --changes needs them only for the +// ambiguous rows (a clean success that touched no files), so it counts just +// those rather than the whole unbounded fetch. +func conflictCountsForFilter(cmd *cobra.Command, s *store.Store, runs []store.Run, onlyFailed bool) (map[int64]int, error) { + if onlyFailed { + return map[int64]int{}, nil } - return printRuns(cmd.OutOrStdout(), runs, volumes, nodes, conflicts) + var candidates []store.Run + for _, r := range runs { + if r.Status == store.RunStatusSuccess && r.FileCount == 0 { + candidates = append(candidates, r) + } + } + return loadConflictCounts(cmd, s, candidates) +} + +// filterRuns applies the --failed / --changes predicates in one pass, +// preserving the descending order the store returned. +func filterRuns(runs []store.Run, conflicts map[int64]int, onlyFailed, onlyChanges bool) []store.Run { + out := make([]store.Run, 0, len(runs)) + for _, r := range runs { + if onlyFailed && !runNeedsAttention(r.Status) { + continue + } + if onlyChanges && !runIsInteresting(r, conflicts) { + continue + } + out = append(out, r) + } + return out +} + +// runNeedsAttention reports whether a run ended in a state an operator +// should look at: a mid-flight failure, a preflight refusal, a reaped +// (aborted) run, or a partial completion. +func runNeedsAttention(status string) bool { + switch status { + case store.RunStatusFailed, store.RunStatusRefused, store.RunStatusAborted, store.RunStatusPartial: + return true + } + return false +} + +// runIsInteresting reports whether a run is worth showing under --changes: +// anything needing attention, still running, that touched files, or that +// resolved conflicts. A clean success that touched nothing is the routine +// no-op the filter hides. +// +// runs.file_count conflates transferred with already-correct for bucket and +// index runs (it is the total files considered, not the delta), so an +// in-sync bucket sync or an all-unchanged re-index still shows here — the +// filter deliberately errs toward showing rather than risk hiding a run +// that moved content. It reliably folds peer-sync no-ops, whose file_count +// is the receiver-verified (i.e. transferred) count and so is zero when +// nothing moved. +func runIsInteresting(r store.Run, conflicts map[int64]int) bool { + if r.Status != store.RunStatusSuccess { + return true + } + if r.FileCount > 0 { + return true + } + return conflicts[r.ID] > 0 } // loadNodeNames builds an id→name map for the peer_node_id values @@ -106,11 +287,22 @@ func loadNodeNames(cmd *cobra.Command, s *store.Store, runs []store.Run) (map[in return out, nil } -// loadConflictCounts derives the conflict count for every peer-sync -// run in the listing via one grouped query against `files`. The v6 -// schema doesn't carry the count on the run row itself; every -// conflict inserts one row under `.squirrel-conflicts/run-/`, so -// the prefix-count is the audit number we want. +// loadConflictCounts derives the conflict count for every peer-sync run in +// the listing. The schema doesn't carry the count on the run row itself, +// so it comes from two derivations merged per run: +// +// - Receiver side: every conflict inserts one row under +// `.squirrel-conflicts/run-/`, so the prefix-count is the number +// of losers this run preserved. +// - Initiator side: the conflict was frozen on a *remote* receiver, so +// no local `.squirrel-conflicts/` rows exist; instead the run raised +// local contested_paths latches (one per conflict + contested +// refusal), so the raised-by-run count is the initiator's signal +// (#158, F27). +// +// A run is one side of a sync, so at most one derivation is non-zero for +// it; taking the max yields the right figure without double-counting the +// rare case where both happen to fire on the same id. func loadConflictCounts(cmd *cobra.Command, s *store.Store, runs []store.Run) (map[int64]int, error) { var peerSyncIDs []int64 for _, r := range runs { @@ -121,10 +313,23 @@ func loadConflictCounts(cmd *cobra.Command, s *store.Store, runs []store.Run) (m if len(peerSyncIDs) == 0 { return map[int64]int{}, nil } - counts, err := s.CountFilesFirstSeenByRunWithPathPrefix(cmd.Context(), peerSyncIDs, sync.ConflictsDirName) + preserved, err := s.CountFilesFirstSeenByRunWithPathPrefix(cmd.Context(), peerSyncIDs, sync.ConflictsDirName) if err != nil { return nil, fmt.Errorf("conflict counts: %w", err) } + frozen, err := s.CountContestedRaisedByRun(cmd.Context(), peerSyncIDs) + if err != nil { + return nil, fmt.Errorf("contested counts: %w", err) + } + counts := make(map[int64]int, len(preserved)+len(frozen)) + for id, n := range preserved { + counts[id] = n + } + for id, n := range frozen { + if n > counts[id] { + counts[id] = n + } + } return counts, nil } @@ -174,10 +379,25 @@ func peerLabel(r store.Run, nodes map[int64]string) string { if !ok { name = fmt.Sprintf("id=%d", r.PeerNodeID.Int64) } + label := peerDirectionArrow(r) + " " + name if r.CorrelatedRunID.Valid { - return fmt.Sprintf("%s correlated=%d", name, r.CorrelatedRunID.Int64) + return fmt.Sprintf("%s correlated=%d", label, r.CorrelatedRunID.Int64) + } + return label +} + +// peerDirectionArrow labels which way a peer-sync ran so inbound and +// outbound are distinguishable in the audit trail (friction F18): the +// destination column holds a peer name in both directions. Only the +// initiator records runs.shallow — BeginSyncRunIfClear sets it, while the +// receiver's BeginPeerSyncRun leaves it NULL (see store.Run.Shallow) — so a +// set shallow flag marks the row as the side that pushed. "→" is outbound +// (we pushed to the peer), "←" inbound (the peer pushed to us). +func peerDirectionArrow(r store.Run) string { + if r.Shallow.Valid { + return "→" } - return name + return "←" } // conflictLabel formats the per-run conflict count for the listing. diff --git a/cmd/squirrel/status.go b/cmd/squirrel/status.go new file mode 100644 index 0000000..67f22b3 --- /dev/null +++ b/cmd/squirrel/status.go @@ -0,0 +1,113 @@ +package main + +import ( + "errors" + "fmt" + "io" + "text/tabwriter" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/status" +) + +// newStatusCmd returns the `squirrel status [volume]` cobra command: a +// read-only "am I safe?" grid answering, per (volume × target), whether +// every configured target is caught up and whether the volume's local +// content is durable where its offload policy needs it (friction-log F16, +// F17, F23). It mutates nothing. The process exit code reflects the worst +// state — 0 green, 1 amber, 2 red — so the same command scripts a health +// check without parsing the grid. +func newStatusCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "status [volume]", + Short: "Show per-target sync coverage and durability, and the offload-ready total", + Args: cobra.MaximumNArgs(1), + // The amber/red signal is carried out as an exitCodeError; silence + // cobra's own error printing so the grid is the only output and the + // signal shows up purely in the exit status. + SilenceErrors: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + var volume string + if len(args) == 1 { + volume = args[0] + } + err := runStatus(cmd, volume) + // SilenceErrors keeps the scriptable green/amber/red exit-code + // signal (an exitCodeError) from leaking a cobra "Error:" line. But + // it also silences genuine failures — an unknown volume, a missing + // config, a store error — which must still reach the user, so print + // those here. The exit-code signal stays silent. + var ec exitCodeError + if err != nil && !errors.As(err, &ec) { + fmt.Fprintln(cmd.ErrOrStderr(), cmd.ErrPrefix(), err.Error()) + } + return err + }, + } + return cmd +} + +func runStatus(cmd *cobra.Command, volume string) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + if volume != "" { + if _, ok := cfg.Volumes[volume]; !ok { + return fmt.Errorf("unknown volume %q (declare it in %s)", volume, cfg.Path) + } + } + s, err := openStore(cmd, cfg) + if err != nil { + return err + } + defer s.Close() + + rep, err := status.Build(cmd.Context(), s, cfg) + if err != nil { + return err + } + worst := renderStatus(cmd.OutOrStdout(), rep, volume) + if code := worst.ExitCode(); code != 0 { + return exitCodeError{code: code} + } + return nil +} + +// renderStatus prints the grid and returns the worst level across the +// volumes it printed. When volume is non-empty only that volume is shown, +// and only its level drives the return (so a scripted per-volume check +// isn't reddened by an unrelated volume). +func renderStatus(w io.Writer, rep status.Report, volume string) status.Level { + worst := status.LevelNeutral + for _, v := range rep.Volumes { + if volume != "" && v.Name != volume { + continue + } + renderStatusVolume(w, v) + if v.Level() > worst { + worst = v.Level() + } + } + fmt.Fprintf(w, "overall: %s\n", status.TrafficLight(worst)) + return worst +} + +// renderStatusVolume prints one volume's header, its target grid, and its +// offload-readiness line, reusing the shared status labels so the wording +// matches the TUI exactly. +func renderStatusVolume(w io.Writer, v status.VolumeStatus) { + fmt.Fprintf(w, "\n%s %s [%s]\n", v.Name, v.Path, status.TrafficLight(v.Level())) + fmt.Fprintf(w, " index: %s\n", status.IndexLabel(v)) + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, " TARGET\tROLE\tLAST SYNC\tSTATE\tDURABLE\tMETHOD\tEVIDENCE") + for _, t := range v.Targets { + fmt.Fprintf(tw, " %s\t%s\t%s\t%s\t%s\t%s\t%s\n", + t.Name, status.RoleLabel(t), status.LastSyncLabel(t), status.StateLabel(t), + status.DurableLabel(t), status.MethodLabel(t), status.EvidenceLabel(t)) + } + _ = tw.Flush() + fmt.Fprintf(w, " %s\n", status.OffloadLabel(v.Offload)) +} diff --git a/cmd/squirrel/status_test.go b/cmd/squirrel/status_test.go new file mode 100644 index 0000000..3b96ba8 --- /dev/null +++ b/cmd/squirrel/status_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// writeStatusConfig lays down a volume with a sync destination, plus its +// DB, and returns the config path. Callers index the volume themselves. +func writeStatusConfig(t *testing.T) (configPath string) { + t.Helper() + root := t.TempDir() + volumeDir := filepath.Join(root, "pics") + if err := os.MkdirAll(volumeDir, 0o755); err != nil { + t.Fatalf("mkdir volume: %v", err) + } + writeTestFile(t, filepath.Join(volumeDir, "a.txt"), "hello") + writeTestFile(t, filepath.Join(volumeDir, "b.txt"), "world") + + destDir := filepath.Join(root, "dst") + if err := os.MkdirAll(destDir, 0o755); err != nil { + t.Fatalf("mkdir dest: %v", err) + } + dbPath := filepath.Join(root, "index.db") + configPath = filepath.Join(root, "config.toml") + body := "" + + "db = \"" + dbPath + "\"\n\n" + + "[destinations.scratch]\n" + + "type = \"local\"\n" + + "root = \"" + destDir + "\"\n\n" + + "[volumes.pics]\n" + + "path = \"" + volumeDir + "\"\n" + + "sync_to = [\"scratch\"]\n" + + "sync_every = \"1h\"\n" + if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return configPath +} + +// TestCLIStatusNeverSyncedIsAmber: a configured but never-synced target +// leaves the volume amber, and the command exits with code 1 carried by an +// exitCodeError. +func TestCLIStatusNeverSyncedIsAmber(t *testing.T) { + cfg := writeStatusConfig(t) + runCLI(t, "--config", cfg, "index", "pics") + + out, err := runCLIExpectErr(t, "--config", cfg, "status") + var ec exitCodeError + if !errors.As(err, &ec) { + t.Fatalf("expected exitCodeError, got %v", err) + } + if ec.code != 1 { + t.Fatalf("exit code = %d, want 1 (amber)", ec.code) + } + if !strings.Contains(out, "pics") || !strings.Contains(out, "scratch") { + t.Fatalf("grid missing volume/target:\n%s", out) + } + if !strings.Contains(out, "never") { + t.Fatalf("never-synced target should read 'never':\n%s", out) + } + if !strings.Contains(out, "overall: amber") { + t.Fatalf("summary should be amber:\n%s", out) + } + // Cobra's own "Error:" line must be silenced — only the grid prints. + if strings.Contains(out, "Error:") { + t.Fatalf("cobra error line leaked into output:\n%s", out) + } +} + +// TestCLIStatusAfterSyncIsGreen: once a fresh sync lands, the pair is +// caught up within cadence and the command exits 0. +func TestCLIStatusAfterSyncIsGreen(t *testing.T) { + requireRcloneCLI(t) + cfg := writeStatusConfig(t) + runCLI(t, "--config", cfg, "index", "pics") + runCLI(t, "--config", cfg, "sync", "pics", "--init") + + out := runCLI(t, "--config", cfg, "status") + if !strings.Contains(out, "overall: green") { + t.Fatalf("expected green after a fresh sync:\n%s", out) + } + if !strings.Contains(out, "ok") { + t.Fatalf("expected an 'ok' target state:\n%s", out) + } +} + +// TestCLIStatusUnknownVolume errors clearly on a volume not in config. +// SilenceErrors (needed to keep the amber/red exit-code signal quiet) must +// not swallow this genuine error: it has to reach the user's stderr, not +// just the returned error value. +func TestCLIStatusUnknownVolume(t *testing.T) { + cfg := writeStatusConfig(t) + out, err := runCLIExpectErr(t, "--config", cfg, "status", "nope") + if err == nil || !strings.Contains(err.Error(), `unknown volume "nope"`) { + t.Fatalf("expected unknown-volume error, got %v", err) + } + if !strings.Contains(out, `unknown volume "nope"`) { + t.Fatalf("error must reach the user, not be silenced:\n%s", out) + } +} + +// TestCLIStatusRequiresConfig: status is a question about configured +// coverage, so a missing config is a clear error, not an empty grid — and +// the message must be printed, not silently swallowed by SilenceErrors. +func TestCLIStatusRequiresConfig(t *testing.T) { + missing := filepath.Join(t.TempDir(), "no-config.toml") + out, err := runCLIExpectErr(t, "--config", missing, "status") + if err == nil || !strings.Contains(err.Error(), "no config at") { + t.Fatalf("expected missing-config error, got %v", err) + } + if !strings.Contains(out, "no config at") { + t.Fatalf("error must reach the user, not be silenced:\n%s", out) + } +} diff --git a/cmd/squirrel/sync.go b/cmd/squirrel/sync.go index 08fcd27..8a1d29e 100644 --- a/cmd/squirrel/sync.go +++ b/cmd/squirrel/sync.go @@ -74,11 +74,17 @@ func runSync(cmd *cobra.Command, volumeName, destinationName string, progress bo if opts.Shallow { fmt.Fprintln(out, shallowSyncWarning) } - if err := sync.EnsureMinVersion(cmd.Context(), rcl, out, sync.ShallowForPairs(pairs, opts.Shallow)); err != nil { - return err - } - if err := writeRcloneConfigLogged(out, rcl, cfg); err != nil { - return err + // Skip the rclone preamble entirely when every pair targets kopia: + // kopia never invokes rclone, so a version check is pointless and + // "rclone.conf updated" is misleading noise on a kopia-only sync + // (friction F11b). + if pairsNeedRclone(pairs) { + if err := sync.EnsureMinVersion(cmd.Context(), rcl, out, sync.ShallowForPairs(pairs, opts.Shallow)); err != nil { + return err + } + if err := writeRcloneConfigLogged(out, rcl, cfg); err != nil { + return err + } } // One snapshotter shared across every pair: the VACUUM INTO snapshot // is taken once per invocation and fanned out (decision #1). Disabled @@ -104,7 +110,7 @@ func runSync(cmd *cobra.Command, volumeName, destinationName string, progress bo if pp != nil { pp.clear() } - printSyncReport(out, rep, err) + printSyncReport(out, rep, err, false) if err != nil || rep.Status != "success" { anyFailed = true } @@ -146,6 +152,20 @@ func rcloneConfigPathFor(cfg *config.Config) string { return filepath.Join(filepath.Dir(cfg.Path), "rclone.conf") } +// pairsNeedRclone reports whether any pair in the batch drives rclone. +// Peer nodes (Destination nil) transfer bytes with rclone, as do every +// non-kopia bucket destination; only a batch composed entirely of kopia +// destinations skips rclone. Used to suppress the rclone preamble on a +// kopia-only sync (F11b). +func pairsNeedRclone(pairs []sync.Pair) bool { + for _, p := range pairs { + if p.Destination == nil || p.Destination.Type != "kopia" { + return true + } + } + return false +} + // writeRcloneConfigLogged renders the rclone.conf and logs a single line // when the file actually changed. An unexpected rewrite is worth // surfacing: the config is derived deterministically from squirrel's @@ -162,7 +182,10 @@ func writeRcloneConfigLogged(out io.Writer, rcl *sync.Rclone, cfg *config.Config return nil } -func printSyncReport(w io.Writer, rep sync.Report, runErr error) { +// printSyncReport renders one pair's outcome. reverse flips the arrow to +// destination → volume for restores, whose bytes flow the other way from +// a sync (the restore-arrow friction item). +func printSyncReport(w io.Writer, rep sync.Report, runErr error, reverse bool) { r := rep.RcloneResult for _, msg := range rep.Warnings { fmt.Fprintf(w, "warning: %s\n", msg) @@ -170,37 +193,19 @@ func printSyncReport(w io.Writer, rep sync.Report, runErr error) { for _, msg := range rep.NodePendingWarnings { fmt.Fprintf(w, "warning: peer reports %s\n", msg) } - switch rep.Verification.Method { - case sync.VerifyMethodKopia: - // Kopia pushes have no rclone counters; render the snapshot's - // own numbers instead. - fmt.Fprintf(w, "%s → %s status=%s files=%d bytes=%d snapshot=%s verified=%t run=%d\n", - rep.Volume, rep.Destination, rep.Status, - rep.Verification.Files, rep.Verification.Bytes, - rep.Verification.SnapshotID, rep.Verification.Verified(), rep.RunID, - ) - case sync.VerifyMethodPresenceSize: - // Content-addressed pushes count objects, with skipped = hashes - // the destination already recorded, entries = manifest segment - // lines, and fingerprints = provider checksums captured for the - // fresh uploads. - fmt.Fprintf(w, "%s → %s status=%s objects=%d skipped=%d errors=%d bytes=%d entries=%d fingerprints=%d run=%d\n", - rep.Volume, rep.Destination, rep.Status, - r.Transferred, r.Checked, r.Errors, r.Bytes, - rep.Verification.Files, rep.Fingerprints, rep.RunID, - ) - // A packed dry-run reports its pack-side routing here; the - // object side rode the objects=/bytes= counters above. Only a - // packed dry-run sets SizeBand, so this line is preview-only. - if p := rep.PackPreview; p.SizeBand > 0 { - fmt.Fprintf(w, " would pack %d content(s) (%d uncompressed byte(s)) into ~%d pack(s), each targeting %d compressed byte(s)\n", - p.Contents, p.Bytes, p.Packs, p.SizeBand) - } - default: - fmt.Fprintf(w, "%s → %s status=%s transferred=%d checked=%d errors=%d bytes=%d run=%d\n", - rep.Volume, rep.Destination, rep.Status, - r.Transferred, r.Checked, r.Errors, r.Bytes, rep.RunID, - ) + src, dst := rep.Volume, rep.Destination + if reverse { + src, dst = rep.Destination, rep.Volume + } + // A run that never reached a terminal state — a preflight error before + // its runs row was allocated (unindexed volume, "already running") — + // carries an empty status and run=0; the counter line would read + // "status= … run=0", pure noise. Skip it and let the error below carry + // the message. Gates that mint a 'refused' row (#157) do have a status + // and run id, so they still print as their own condition (friction + // F11d). + if rep.Status != "" { + printSyncSummaryLine(w, rep, src, dst) } if rep.NodeReceiverRunID != 0 { fmt.Fprintf(w, " receiver_run=%d matched=%d mismatched=%d missing=%d conflicts=%d\n", @@ -230,6 +235,17 @@ func printSyncReport(w io.Writer, rep sync.Report, runErr error) { fmt.Fprintf(w, " preserved at %s:%s\n", rep.Destination, c.PreservedAtPath) } } + for _, c := range rep.NodeContested { + fmt.Fprintf(w, " contested %s: frozen by a prior conflict — bytes refused, not delivered\n", c.Path) + if c.PreservedAtPath != "" { + fmt.Fprintf(w, " versions live %s, preserved %s at %s:%s\n", + c.LiveBlake3Hex, c.PreservedBlake3Hex, rep.Destination, c.PreservedAtPath) + } + // Quote the args: a contested path may contain whitespace + // (validateRelPath allows it), which would otherwise split into + // several shell words when the operator copy-pastes the hint. + fmt.Fprintf(w, " resolve with: squirrel conflicts resolve %q %q\n", rep.Volume, c.Path) + } for _, ff := range r.FailedFiles { // Some rclone errors (auth, listing, fatal copy) have no Object. // Render those as a bare "error: ..." rather than "error : ...". @@ -254,3 +270,44 @@ func printSyncReport(w io.Writer, rep sync.Report, runErr error) { fmt.Fprintf(w, " %v\n", runErr) } } + +// printSyncSummaryLine renders the one-line counter summary for a pair, +// keyed off the handler's verification method. src/dst carry the already +// direction-resolved endpoints. The default (rclone bucket / peer / restore) +// line reports already_correct so an in-sync no-op is distinguishable from +// an empty one — transferred=0 alone is ambiguous (friction F7). +func printSyncSummaryLine(w io.Writer, rep sync.Report, src, dst string) { + r := rep.RcloneResult + switch rep.Verification.Method { + case sync.VerifyMethodKopia: + // Kopia pushes have no rclone counters; render the snapshot's + // own numbers instead. + fmt.Fprintf(w, "%s → %s status=%s files=%d bytes=%d snapshot=%s verified=%t run=%d\n", + src, dst, rep.Status, + rep.Verification.Files, rep.Verification.Bytes, + rep.Verification.SnapshotID, rep.Verification.Verified(), rep.RunID, + ) + case sync.VerifyMethodPresenceSize: + // Content-addressed pushes count objects, with skipped = hashes + // the destination already recorded, entries = manifest segment + // lines, and fingerprints = provider checksums captured for the + // fresh uploads. + fmt.Fprintf(w, "%s → %s status=%s objects=%d skipped=%d errors=%d bytes=%d entries=%d fingerprints=%d run=%d\n", + src, dst, rep.Status, + r.Transferred, r.Checked, r.DisplayErrors(), r.Bytes, + rep.Verification.Files, rep.Fingerprints, rep.RunID, + ) + // A packed dry-run reports its pack-side routing here; the + // object side rode the objects=/bytes= counters above. Only a + // packed dry-run sets SizeBand, so this line is preview-only. + if p := rep.PackPreview; p.SizeBand > 0 { + fmt.Fprintf(w, " would pack %d content(s) (%d uncompressed byte(s)) into ~%d pack(s), each targeting %d compressed byte(s)\n", + p.Contents, p.Bytes, p.Packs, p.SizeBand) + } + default: + fmt.Fprintf(w, "%s → %s status=%s transferred=%d already_correct=%d errors=%d bytes=%d run=%d\n", + src, dst, rep.Status, + r.Transferred, rep.AlreadyCorrect, r.DisplayErrors(), r.Bytes, rep.RunID, + ) + } +} diff --git a/cmd/squirrel/sync_test.go b/cmd/squirrel/sync_test.go index 5902cbe..dcedf01 100644 --- a/cmd/squirrel/sync_test.go +++ b/cmd/squirrel/sync_test.go @@ -42,6 +42,22 @@ func TestCLISyncHappyPath(t *testing.T) { } } +// TestCLISyncAlreadyCorrect covers F7: a second sync of an unchanged +// volume reports already_correct so an in-sync no-op is distinguishable +// from an empty one (transferred=0 alone is ambiguous). +func TestCLISyncAlreadyCorrect(t *testing.T) { + requireRcloneCLI(t) + f := writeSyncFixture(t) + writeTestFile(t, filepath.Join(f.volumeDir, "a.txt"), "alpha") + writeTestFile(t, filepath.Join(f.volumeDir, "b.txt"), "beta") + runCLI(t, "--config", f.configPath, "index", f.volumeName) + runCLI(t, "--config", f.configPath, "sync", "pics") // first sync transfers + out := runCLI(t, "--config", f.configPath, "sync", "pics") + if !strings.Contains(out, "transferred=0") || !strings.Contains(out, "already_correct=2") { + t.Fatalf("second sync should report transferred=0 already_correct=2: %q", out) + } +} + func TestCLISyncUnknownDestinationFlag(t *testing.T) { f := writeSyncFixture(t) _, err := runCLIExpectErr(t, "--config", f.configPath, "sync", "pics", "--to", "ghost") @@ -105,6 +121,11 @@ sync_to = ["mirror"] if !strings.Contains(out, "snapshot=snap123") || !strings.Contains(out, "verified=true") { t.Fatalf("output missing the kopia snapshot summary:\n%s", out) } + // F11b: a kopia-only sync never touches rclone, so it must not print + // the "rclone.conf updated" line. + if strings.Contains(out, "rclone.conf updated") { + t.Fatalf("kopia-only sync should not log an rclone.conf update:\n%s", out) + } } func TestCLISyncRequiresConfig(t *testing.T) { diff --git a/cmd/squirrel/verify.go b/cmd/squirrel/verify.go index d696903..bb27e3f 100644 --- a/cmd/squirrel/verify.go +++ b/cmd/squirrel/verify.go @@ -22,7 +22,7 @@ import ( // recorded fingerprint left exactly as found so the operator inspects the // evidence. func newVerifyCmd() *cobra.Command { - return &cobra.Command{ + cmd := &cobra.Command{ Use: "verify []", Short: "Re-check recorded offsite objects and packs against their upload fingerprints", Args: cobra.MaximumNArgs(1), @@ -34,6 +34,8 @@ func newVerifyCmd() *cobra.Command { return runVerify(cmd, destName) }, } + cmd.AddCommand(newVerifyAckCmd()) + return cmd } func runVerify(cmd *cobra.Command, destName string) error { @@ -108,7 +110,8 @@ func verifiableLayout(layout string) bool { } // printVerifyReport renders one destination's pass: a loud stderr line -// per missing or mismatched object or pack, then the summary counters. +// per missing or mismatched object or pack, then the summary counters, +// then the standing-alarm transition this pass caused (#157, F30). func printVerifyReport(out, errOut io.Writer, rep sync.RemoteVerifyReport, runErr error) { printVerifyFailures(errOut, "object", rep.Destination, rep.Missing, rep.Mismatched) printVerifyFailures(errOut, "pack", rep.Destination, rep.PacksMissing, rep.PackMismatched) @@ -125,6 +128,20 @@ func printVerifyReport(out, errOut io.Writer, rep sync.RemoteVerifyReport, runEr len(rep.Mismatched), len(rep.Missing), rep.Unrecorded, rep.Packs, rep.PacksVerified, rep.PacksPopulated, rep.PacksPending, len(rep.PackMismatched), len(rep.PacksMissing)) + printVerifyAlarmTransition(out, errOut, rep) +} + +// printVerifyAlarmTransition surfaces the standing-alarm change this pass +// made: a raised alarm shouts on stderr with how to clear it, an +// auto-cleared alarm is a quiet confidence line on stdout. +func printVerifyAlarmTransition(out, errOut io.Writer, rep sync.RemoteVerifyReport) { + if rep.AlarmRaised { + fmt.Fprintf(errOut, "ALARM %s: verification failed — this destination stays in alarm until a clean `squirrel verify %s` clears it or you run `squirrel verify ack %s`\n", + rep.Destination, rep.Destination, rep.Destination) + } + if rep.AlarmCleared { + fmt.Fprintf(out, "verify %s: standing alarm cleared by this clean pass\n", rep.Destination) + } } // printVerifyFailures writes one stderr line per missing or mismatched diff --git a/cmd/squirrel/verify_ack.go b/cmd/squirrel/verify_ack.go new file mode 100644 index 0000000..1c87ed6 --- /dev/null +++ b/cmd/squirrel/verify_ack.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "os/user" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/store" +) + +// newVerifyAckCmd returns `squirrel verify ack `: the explicit +// operator path to clear a standing verify alarm (#157, F30). It is the +// deliberate counterpart to the automatic clear a clean `squirrel verify` +// performs — for when the operator has inspected the destination and +// restored or re-uploaded the affected artifacts out of band, and wants +// the alarm gone without waiting for (or being able to run) a full +// re-verification pass. The clear is recorded against the run that raised +// the alarm and tagged with the operator, so the audit trail shows who +// cleared it and when. +func newVerifyAckCmd() *cobra.Command { + return &cobra.Command{ + Use: "ack ", + Short: "Acknowledge and clear a standing verify alarm on a destination", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runVerifyAck(cmd, args[0]) + }, + } +} + +func runVerifyAck(cmd *cobra.Command, destName string) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + s, err := openStore(cmd, cfg) + if err != nil { + return err + } + defer s.Close() + + alarm, err := s.GetDestinationAlarm(cmd.Context(), destName) + if err != nil { + if store.IsNotFound(err) { + return fmt.Errorf("destination %q has no standing alarm to acknowledge", destName) + } + return fmt.Errorf("look up standing alarm for %q: %w", destName, err) + } + cleared, err := s.ClearDestinationAlarm(cmd.Context(), destName, alarm.RaisedRunID, ackOperator()) + if err != nil { + return err + } + if !cleared { + // Raced with an auto-clear (a clean verify) between the read and + // the write — the alarm is gone either way, which is the outcome + // the operator asked for. + return fmt.Errorf("destination %q has no standing alarm to acknowledge", destName) + } + fmt.Fprintf(cmd.OutOrStdout(), "verify %s: standing alarm acknowledged and cleared\n", destName) + return nil +} + +// ackOperator names who acked for the runs_audit operator column: the OS +// user when resolvable, else a generic label. Best-effort attribution for +// the audit trail, not an authorization check. +func ackOperator() string { + if u, err := user.Current(); err == nil && u.Username != "" { + return u.Username + } + return "operator" +} diff --git a/config/destinations.go b/config/destinations.go index 573abbb..563d79a 100644 --- a/config/destinations.go +++ b/config/destinations.go @@ -775,3 +775,42 @@ func (d *Destination) CanEverGateOffload() (bool, string) { } return true, "" } + +// VerifyCadencedTargets marks the required targets that carry an effective +// local verify cadence — a per-destination verify_every, or the [agent] +// verify_every default applied to a content-addressed/packed destination. +// The offload gate accepts a locally-advanced fingerprint-verified +// component as content-verified only for a target in this set, so the +// provider-fingerprint evidence keeps being re-confirmed for as long as +// deletions are permitted (issue #155). A required target with no local +// destination — a peer-relayed offsite this node cannot see — is omitted; +// the responder's cadence governs it, relayed and enforced at pull time. +// +// It lives here, beside CanEverGateOffload, because both the offload +// command and the readiness query behind `squirrel status` must feed the +// gate the same set: readiness promises its totals match what an offload +// would actually move, and a divergence here would quietly break that. +func (c *Config) VerifyCadencedTargets(require []string) map[string]bool { + var agentDefault time.Duration + if c.Agent != nil { + agentDefault = c.Agent.VerifyEvery + } + out := make(map[string]bool, len(require)) + for _, name := range require { + d, ok := c.Destinations[name] + if !ok { + continue + } + if d.Layout != LayoutContentAddressed && d.Layout != LayoutPacked { + continue + } + eff := d.VerifyEvery + if eff == 0 { + eff = agentDefault + } + if eff > 0 { + out[name] = true + } + } + return out +} diff --git a/design/friction-log.md b/design/friction-log.md index b69f8f6..732afee 100644 --- a/design/friction-log.md +++ b/design/friction-log.md @@ -319,6 +319,22 @@ re-supersede a path whose live row lost a conflict since your last delivery without operator action), and a `squirrel conflicts` question-command to list unresolved ones. +*Resolved (#158, contested-freeze, option 2a).* A conflict now raises a +`contested_paths` latch on the receiver (schema v26); while it stands the +`/plan` classifier answers a divergent re-assertion from any peer with a +new `contested` disposition (version-gated for older peers) instead of +minting another `.squirrel-conflicts/run-N/` copy — the flip-flop stops at +the first conflict, preserved once. Both versions stay reachable (winner +live, loser under `.squirrel-conflicts/`). Conflict/contested counts flow +into `SyncRunReport`, the initiators' run rows (the CONFLICTS column), a +`scheduler.conflict` warn line, and a "Contested paths" TUI badge — each +node mirrors the freeze into its own latch, so the losing edge sees it too, +not only the hub. `squirrel conflicts` lists the frozen paths and +`squirrel conflicts resolve ` clears the latch (the explicit +human act that unfreezes; squirrel never resolves on its own). Adopting the +non-live version stays a deliberate `restore`, not something resolve does +silently. + **F29 · S1 — relayed offload against a cold archive is structurally unreachable, so no machine in the reference household can offload anything.** The endgame of the whole design — htpc/laptop dropping diff --git a/index/index.go b/index/index.go index b9c43df..d219c5f 100644 --- a/index/index.go +++ b/index/index.go @@ -66,6 +66,14 @@ type Report struct { RunID int64 } +// SawFiles reports whether the walk observed any live file (added, +// modified, or unchanged). Zero across all three means the tree held no +// regular files this run — an empty or wrong-mounted volume path, which +// callers surface as a warning on first index since it is otherwise +// indistinguishable from a healthy no-op (friction F8). This package never +// writes to stderr, so the decision is returned rather than printed. +func (r Report) SawFiles() bool { return r.Added+r.Modified+r.Unchanged > 0 } + // ErrAlreadyRunning is returned from Index when store.BeginIndexRunIfClear // refuses to start because another index- or audit-kind run is already in // flight against the same volume. Callers can errors.As against this type diff --git a/offload/readiness.go b/offload/readiness.go new file mode 100644 index 0000000..aa226f2 --- /dev/null +++ b/offload/readiness.go @@ -0,0 +1,96 @@ +package offload + +import ( + "context" + "time" + + "github.com/mbertschler/squirrel/store" +) + +// ReadinessOptions selects the volume and durability policy for a +// Readiness evaluation. It mirrors the durability-relevant subset of +// Options: no path/age selectors (readiness is always whole-volume) and no +// DryRun flag (readiness never touches disk or opens a run). +type ReadinessOptions struct { + // VolumeID is the local volumes row the gate is evaluated against. + VolumeID int64 + // Require is the volume's offload_requires policy. Empty means the + // volume has no offload policy, and Readiness reports Applicable=false + // without evaluating anything. + Require []string + // MaxEvidenceAge is the volume's offload_max_evidence_age; it feeds the + // same time-based staleness gate an offload would apply. Zero disables + // the staleness policy. + MaxEvidenceAge time.Duration + // VerifyCadenced is Options.VerifyCadenced: the required targets under + // an effective local verify cadence, from + // config.Config.VerifyCadencedTargets. It must be the same set the real + // offload passes — it widens the gate for locally-advanced + // fingerprint-verified components, so a readiness call that omitted it + // would report fewer bytes than an offload would actually move. Nil is + // a valid empty set. + VerifyCadenced map[string]bool +} + +// ReadinessReport totals what an offload of the whole volume would move +// right now. OffloadableFiles/Bytes are the subset of the present set that +// passes the durability gate; PresentFiles/Bytes are the whole gated +// candidate set, so a UI can render "N of M". +type ReadinessReport struct { + // Applicable is false when the volume declares no offload policy — the + // caller can then distinguish "nothing offloadable" from "offload not + // configured here" without inspecting the totals. + Applicable bool + OffloadableFiles int + OffloadableBytes int64 + PresentFiles int + PresentBytes int64 +} + +// Readiness reports how many present files, and how many bytes, currently +// pass the offload gate for the volume — the "N GB offloadable now" +// decision-support number (friction log F17) without the side effects of +// `offload --dry-run`. It reads only the local index (durability vectors, +// scan-back fingerprints), opens no run, reads no disk bytes, validates no +// volume marker, and deletes nothing, so it is safe to call from a +// read-only introspection surface at any time. +// +// The gate is evaluated exactly as Offload's dry-run path evaluates it — +// same loadGate, same present-and-not-reserved candidate predicate, same +// per-file check — so the totals match what an offload would actually move. +// Totals are order-independent, so this walks the index map in a single +// pass rather than building and sorting an intermediate candidate slice. +func Readiness(ctx context.Context, s *store.Store, opts ReadinessOptions) (ReadinessReport, error) { + if len(opts.Require) == 0 { + return ReadinessReport{Applicable: false}, nil + } + now := time.Now() + g, err := loadGate(ctx, s, opts.VolumeID, opts.Require, now.UnixNano(), opts.MaxEvidenceAge, opts.VerifyCadenced) + if err != nil { + return ReadinessReport{}, err + } + rows, err := s.LoadVolumeIndex(ctx, opts.VolumeID) + if err != nil { + return ReadinessReport{}, err + } + rep := ReadinessReport{Applicable: true} + for p, row := range rows { + if err := ctx.Err(); err != nil { + return ReadinessReport{}, err + } + if row.Status != store.StatusPresent || underReservedSubtree(p) { + continue + } + rep.PresentFiles++ + rep.PresentBytes += row.SizeBytes + failures, err := g.check(ctx, row) + if err != nil { + return ReadinessReport{}, err + } + if len(failures) == 0 { + rep.OffloadableFiles++ + rep.OffloadableBytes += row.SizeBytes + } + } + return rep, nil +} diff --git a/status/build.go b/status/build.go new file mode 100644 index 0000000..d48a3cc --- /dev/null +++ b/status/build.go @@ -0,0 +1,279 @@ +package status + +import ( + "context" + "database/sql" + "fmt" + "sort" + "strings" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/offload" + "github.com/mbertschler/squirrel/store" +) + +// lateFactor is how many cadence periods a pair may fall behind before its +// staleness reads red rather than amber. Within one cadence is on-time +// (green); up to lateFactor cadences is late (amber); beyond is a +// destination that has effectively stopped (red). +const lateFactor = 3 + +// Options tunes Build. +type Options struct { + // SkipOffloadReadiness omits the per-volume offload-readiness tally. + // offload.Readiness loads the whole volume index and runs the gate over + // every present file, so on large volumes it is the report's dominant + // cost. The TUI dashboard sets it on its one-second tick (and refreshes + // the tally on a slower cadence into its own cache) so the coverage and + // durability grid stays responsive; the CLI `squirrel status` leaves it + // off so the "N offloadable now" figure is always current. When set, a + // volume's Offload still reports Applicable from its config policy, just + // with zero counts. + SkipOffloadReadiness bool +} + +// Build produces the status Report for this node with readiness computed — +// the CLI's view and the default. See BuildWithOptions for the tunable +// form the TUI uses. +func Build(ctx context.Context, s *store.Store, cfg *config.Config) (Report, error) { + return BuildWithOptions(ctx, s, cfg, Options{}) +} + +// BuildWithOptions produces the status Report for this node: every +// config-declared volume with its per-target coverage and durability. It is +// read-only — no run rows, no disk reads, no mutation — so it is safe to +// call from any introspection surface at any time. cfg is the source of +// truth for which volumes and targets should exist; the store supplies the +// observations. +func BuildWithOptions(ctx context.Context, s *store.Store, cfg *config.Config, opts Options) (Report, error) { + rep := Report{Now: time.Now()} + self, err := s.GetSelfNode(ctx) + if err != nil { + return Report{}, fmt.Errorf("look up self node: %w", err) + } + alarms, err := s.ListDestinationAlarms(ctx) + if err != nil { + return Report{}, fmt.Errorf("list destination alarms: %w", err) + } + alarmByDest := indexAlarms(alarms) + for _, name := range sortedVolumeNames(cfg) { + vs, err := buildVolume(ctx, s, cfg, self, alarmByDest, cfg.Volumes[name], rep.Now, opts) + if err != nil { + return Report{}, err + } + rep.Volumes = append(rep.Volumes, vs) + } + return rep, nil +} + +// indexAlarms keys the active alarms by destination name for O(1) lookup +// per target. Alarms latch per destination, not per volume, so one entry +// applies to every volume that names the destination. +func indexAlarms(alarms []store.DestinationAlarm) map[string]store.DestinationAlarm { + out := make(map[string]store.DestinationAlarm, len(alarms)) + for _, a := range alarms { + out[a.Destination] = a + } + return out +} + +// sortedVolumeNames returns the config's volume names in deterministic +// order so the report (and the CLI grid) is stable across runs. +func sortedVolumeNames(cfg *config.Config) []string { + names := make([]string, 0, len(cfg.Volumes)) + for name := range cfg.Volumes { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// buildVolume assembles one volume's status. A volume with no index row +// (declared but never indexed) still lists its planned targets so the +// grid shows the configured coverage rather than a blank line. +func buildVolume(ctx context.Context, s *store.Store, cfg *config.Config, self store.Node, alarmByDest map[string]store.DestinationAlarm, vol *config.Volume, now time.Time, opts Options) (VolumeStatus, error) { + vs := VolumeStatus{Name: vol.Name, Path: vol.Path} + // Applicable reflects the config policy, not whether a tally ran, so a + // never-indexed or skip-readiness volume still reports "has a policy". + vs.Offload.Applicable = len(vol.OffloadRequires) > 0 + dbVol, err := s.GetVolumeByName(ctx, vol.Name) + if store.IsNotFound(err) { + vs.Targets, err = buildTargets(ctx, s, cfg, self, alarmByDest, vol, 0, false, 0, now) + return vs, err + } + if err != nil { + return VolumeStatus{}, fmt.Errorf("look up volume %q: %w", vol.Name, err) + } + vs.Indexed = true + vs.LastIndexAgo, err = lastIndexAge(ctx, s, dbVol.ID, now) + if err != nil { + return VolumeStatus{}, err + } + selfMax, err := selfOriginMax(ctx, s, dbVol.ID, self.ID) + if err != nil { + return VolumeStatus{}, err + } + vs.Targets, err = buildTargets(ctx, s, cfg, self, alarmByDest, vol, dbVol.ID, true, selfMax, now) + if err != nil { + return VolumeStatus{}, err + } + if opts.SkipOffloadReadiness { + return vs, nil + } + readiness, err := offload.Readiness(ctx, s, offload.ReadinessOptions{ + VolumeID: dbVol.ID, + Require: vol.OffloadRequires, + MaxEvidenceAge: vol.OffloadMaxEvidenceAge, + VerifyCadenced: cfg.VerifyCadencedTargets(vol.OffloadRequires), + }) + if err != nil { + return VolumeStatus{}, fmt.Errorf("offload readiness for %q: %w", vol.Name, err) + } + vs.Offload = OffloadReadiness(readiness) + return vs, nil +} + +// buildTargets builds a TargetStatus for every target in the union of the +// volume's sync_to and offload_requires, sync targets first in config +// order then relayed offload targets, deduplicated. +func buildTargets(ctx context.Context, s *store.Store, cfg *config.Config, self store.Node, alarmByDest map[string]store.DestinationAlarm, vol *config.Volume, volID int64, indexed bool, selfMax int64, now time.Time) ([]TargetStatus, error) { + var out []TargetStatus + for _, name := range orderedTargets(vol) { + ts, err := buildTarget(ctx, s, cfg, self, alarmByDest, vol, name, volID, indexed, selfMax, now) + if err != nil { + return nil, err + } + out = append(out, ts) + } + return out, nil +} + +// orderedTargets returns the union of sync_to and offload_requires, +// sync_to first in declared order, then required targets not already +// listed, deduplicated. +func orderedTargets(vol *config.Volume) []string { + seen := make(map[string]struct{}) + var out []string + for _, group := range [][]string{vol.SyncTo, vol.OffloadRequires} { + for _, name := range group { + if _, dup := seen[name]; dup { + continue + } + seen[name] = struct{}{} + out = append(out, name) + } + } + return out +} + +// buildTarget assembles one (volume × target) cell: coverage freshness, +// standing state, and durability. +func buildTarget(ctx context.Context, s *store.Store, cfg *config.Config, self store.Node, alarmByDest map[string]store.DestinationAlarm, vol *config.Volume, name string, volID int64, indexed bool, selfMax int64, now time.Time) (TargetStatus, error) { + ts := TargetStatus{ + Name: name, + Kind: targetKind(cfg, name), + SyncTarget: contains(vol.SyncTo, name), + Required: contains(vol.OffloadRequires, name), + Cadence: vol.SyncEvery, + } + var lastTerminal *store.Run + if indexed { + lastSync, err := latestSuccessfulSync(ctx, s, volID, name) + if err != nil { + return TargetStatus{}, err + } + ts.LastSyncAgo = ageOf(lastSync, now) + if lastTerminal, err = latestTerminalSync(ctx, s, volID, name); err != nil { + return TargetStatus{}, err + } + if lastTerminal != nil { + ts.LastOutcome = lastTerminal.Status + } + } + if alarm, ok := alarmByDest[name]; ok { + ts.Standing = StandingAlarm + ts.AlarmDetail = alarm.Detail + } else { + ts.Standing = classifyStanding(lastTerminal, ts.LastSyncAgo != nil) + } + ts.SyncLevel = syncLevel(ts, lastTerminal) + if indexed { + dur, err := buildDurability(ctx, s, volID, self, selfMax, vol, name, now) + if err != nil { + return TargetStatus{}, err + } + ts.Durability = dur + } + return ts, nil +} + +// classifyStanding derives the standing state from the pair's latest +// terminal sync run. A refusal that regressed from a prior success is a +// broken destination (red); a bootstrap-style refusal on a pair that has +// never succeeded is a fresh destination awaiting one-time `--init` (amber). +func classifyStanding(lastTerminal *store.Run, hadSuccess bool) Standing { + if lastTerminal == nil || lastTerminal.Status != store.RunStatusRefused { + return StandingNone + } + if !hadSuccess && isBootstrapRefusal(lastTerminal.Error) { + return StandingNeedsBootstrap + } + return StandingRefused +} + +// isBootstrapRefusal reports whether a refused run's error is a first-use +// bootstrap refusal (a missing destination marker or an unconnected kopia +// repository), both of which tell the operator to "re-run with --init". +// The layout guard and the init-over-a-different-volume refusal carry no +// "--init" hint, so they stay classified as plain refusals. +func isBootstrapRefusal(errVal sql.NullString) bool { + return errVal.Valid && strings.Contains(errVal.String, "--init") +} + +// syncLevel scores the coverage dimension: standing states first, then the +// latest terminal outcome (a failed run is red, a partial run at least +// amber — both are more recent than any success, so they must not be +// masked by freshness), then freshness relative to the pair's own cadence. +// A relayed target (not pushed to by this node) has no coverage dimension — +// its safety is entirely its durability — so it reads neutral. +func syncLevel(ts TargetStatus, lastTerminal *store.Run) Level { + switch ts.Standing { + case StandingAlarm, StandingRefused: + return LevelCritical + case StandingNeedsBootstrap: + return LevelWarn + } + if lastTerminal != nil { + switch lastTerminal.Status { + case store.RunStatusFailed: + return LevelCritical + case store.RunStatusPartial: + return LevelWarn + } + } + if !ts.SyncTarget { + return LevelNeutral + } + if ts.LastSyncAgo == nil { + return LevelWarn + } + return cadenceLevel(*ts.LastSyncAgo, ts.Cadence) +} + +// cadenceLevel colours a successful sync's age against the pair's cadence: +// within cadence is green, up to lateFactor cadences amber, beyond red. A +// zero cadence (no scheduled sync) cannot be late, so any success is green. +func cadenceLevel(age, cadence time.Duration) Level { + if cadence <= 0 { + return LevelOK + } + switch { + case age <= cadence: + return LevelOK + case age <= lateFactor*cadence: + return LevelWarn + default: + return LevelCritical + } +} diff --git a/status/build_test.go b/status/build_test.go new file mode 100644 index 0000000..798cca8 --- /dev/null +++ b/status/build_test.go @@ -0,0 +1,303 @@ +package status + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/index" + "github.com/mbertschler/squirrel/store" +) + +func setupStore(t *testing.T) *store.Store { + t.Helper() + s, err := store.Open(filepath.Join(t.TempDir(), "status.db")) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +// indexTree writes three files into a fresh directory and indexes it as +// volume name, returning the root and the index run. +func indexTree(t *testing.T, s *store.Store, name string) (string, index.Report) { + t.Helper() + root := t.TempDir() + for _, n := range []string{"a.txt", "b.txt", "c.txt"} { + if err := os.WriteFile(filepath.Join(root, n), []byte("content-"+n), 0o644); err != nil { + t.Fatalf("write %s: %v", n, err) + } + } + rep, err := index.Index(context.Background(), s, root, index.Options{Name: name, Workers: 2}) + if err != nil || rep.Errors > 0 { + t.Fatalf("index %s: err=%v errors=%v", name, err, rep.ErrorList) + } + return root, rep +} + +// cfgFor builds a minimal config with one volume and its targets. Every +// volume gets a one-hour sync cadence so a fresh sync reads as on-time. +func cfgFor(volName, root string, syncTo, requires []string, dests, nodes []string) *config.Config { + cfg := &config.Config{ + Volumes: map[string]*config.Volume{}, + Destinations: map[string]*config.Destination{}, + Nodes: map[string]*config.Node{}, + } + cfg.Volumes[volName] = &config.Volume{ + Name: volName, + Path: root, + SyncTo: syncTo, + OffloadRequires: requires, + SyncEvery: time.Hour, + } + for _, d := range dests { + cfg.Destinations[d] = &config.Destination{Name: d, Type: "s3", Layout: config.LayoutContentAddressed} + } + for _, n := range nodes { + cfg.Nodes[n] = &config.Node{Name: n} + } + return cfg +} + +func recordSuccessfulSync(t *testing.T, s *store.Store, volID int64, dest string) { + t.Helper() + id, blocker, err := s.BeginSyncRunIfClear(context.Background(), store.SyncRunSpec{VolumeID: volID, Destination: dest}) + if err != nil || blocker != nil { + t.Fatalf("BeginSyncRunIfClear(%s): err=%v blocker=%+v", dest, err, blocker) + } + if err := s.FinishRun(context.Background(), id, store.RunStatusSuccess, "", 3); err != nil { + t.Fatalf("FinishRun(%s): %v", dest, err) + } +} + +// TestBuildFullyDurable exercises the happy path end to end: an indexed +// volume, a fresh successful sync, a content-verified durability vector +// covering the local origin, and an offload policy — everything green and +// every file offloadable. +func TestBuildFullyDurable(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + root, idx := indexTree(t, s, "photos") + v, err := s.GetVolumeByName(ctx, "photos") + if err != nil { + t.Fatalf("GetVolumeByName: %v", err) + } + self, err := s.GetSelfNode(ctx) + if err != nil { + t.Fatalf("GetSelfNode: %v", err) + } + if err := s.UpsertDestinationRunIDVerified(ctx, v.ID, "dest", self.ID, idx.RunID, store.VerifyMethodBlake3, false); err != nil { + t.Fatalf("seed vector: %v", err) + } + recordSuccessfulSync(t, s, v.ID, "dest") + + cfg := cfgFor("photos", root, []string{"dest"}, []string{"dest"}, []string{"dest"}, nil) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + if len(rep.Volumes) != 1 { + t.Fatalf("volumes = %d, want 1", len(rep.Volumes)) + } + vs := rep.Volumes[0] + if !vs.Indexed || vs.LastIndexAgo == nil { + t.Fatalf("volume not indexed/aged: %+v", vs) + } + if len(vs.Targets) != 1 { + t.Fatalf("targets = %d, want 1", len(vs.Targets)) + } + tg := vs.Targets[0] + if tg.Kind != KindDestination || !tg.SyncTarget || !tg.Required { + t.Errorf("target role wrong: %+v", tg) + } + if tg.Standing != StandingNone || tg.SyncLevel != LevelOK { + t.Errorf("target coverage = standing %v level %v, want none/ok", tg.Standing, tg.SyncLevel) + } + if tg.Durability == nil || !tg.Durability.LocalContent || !tg.Durability.Covered || tg.Durability.Method != store.VerifyMethodBlake3 { + t.Errorf("durability wrong: %+v", tg.Durability) + } + if !vs.Offload.Applicable || vs.Offload.OffloadableFiles != 3 || vs.Offload.PresentFiles != 3 { + t.Errorf("offload readiness wrong: %+v", vs.Offload) + } + if vs.Offload.OffloadableBytes != vs.Offload.PresentBytes || vs.Offload.OffloadableBytes == 0 { + t.Errorf("offload bytes wrong: %+v", vs.Offload) + } + if rep.Level() != LevelOK { + t.Errorf("report level = %v, want ok", rep.Level()) + } +} + +// TestBuildNeedsBootstrap: a refused sync whose reason names --init, with +// no prior success, is the amber one-time-bootstrap state — not red. +func TestBuildNeedsBootstrap(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + root, _ := indexTree(t, s, "docs") + v, _ := s.GetVolumeByName(ctx, "docs") + id, err := s.BeginRun(ctx, store.RunKindSync, v.ID, "dest", false) + if err != nil { + t.Fatalf("BeginRun: %v", err) + } + if err := s.FinishRun(ctx, id, store.RunStatusRefused, "destination \"dest\" has no marker — re-run with --init to bootstrap", 0); err != nil { + t.Fatalf("FinishRun: %v", err) + } + cfg := cfgFor("docs", root, []string{"dest"}, nil, []string{"dest"}, nil) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + tg := rep.Volumes[0].Targets[0] + if tg.Standing != StandingNeedsBootstrap { + t.Errorf("standing = %v, want needs-bootstrap", tg.Standing) + } + if rep.Level() != LevelWarn { + t.Errorf("report level = %v, want amber", rep.Level()) + } +} + +// TestBuildAlarm: a latched destination alarm reddens its target. +func TestBuildAlarm(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + root, idx := indexTree(t, s, "media") + if _, err := s.RaiseDestinationAlarm(ctx, "dest", store.AlarmKindVerifyMismatch, "checksum mismatch on object", idx.RunID); err != nil { + t.Fatalf("RaiseDestinationAlarm: %v", err) + } + cfg := cfgFor("media", root, []string{"dest"}, nil, []string{"dest"}, nil) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + tg := rep.Volumes[0].Targets[0] + if tg.Standing != StandingAlarm || tg.AlarmDetail == "" { + t.Errorf("standing = %v detail=%q, want alarm", tg.Standing, tg.AlarmDetail) + } + if rep.Level() != LevelCritical { + t.Errorf("report level = %v, want red", rep.Level()) + } +} + +// TestBuildNeverIndexed: a configured-but-unindexed volume is amber and +// lists its planned targets rather than rendering blank (F4). +func TestBuildNeverIndexed(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + cfg := cfgFor("photos", "/does/not/matter", []string{"nas"}, nil, nil, []string{"nas"}) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + vs := rep.Volumes[0] + if vs.Indexed { + t.Errorf("volume should not be indexed") + } + if len(vs.Targets) != 1 || vs.Targets[0].Kind != KindNode { + t.Errorf("planned targets wrong: %+v", vs.Targets) + } + if vs.Level() != LevelWarn { + t.Errorf("volume level = %v, want amber", vs.Level()) + } + // No offload policy configured here, so it must not claim one. + if vs.Offload.Applicable { + t.Errorf("offload should not be applicable without a policy") + } +} + +// TestBuildNeverIndexedReportsPolicy: a never-indexed volume that DOES +// declare offload_requires must report Applicable so the surface shows its +// policy state, not "no policy" (Copilot review on #177). +func TestBuildNeverIndexedReportsPolicy(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + cfg := cfgFor("photos", "/does/not/matter", []string{"nas"}, []string{"nas", "s3archive"}, nil, []string{"nas"}) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + vs := rep.Volumes[0] + if vs.Indexed { + t.Errorf("volume should not be indexed") + } + if !vs.Offload.Applicable { + t.Errorf("offload should be applicable — the config declares offload_requires") + } + if vs.Offload.OffloadableFiles != 0 || vs.Offload.PresentFiles != 0 { + t.Errorf("never-indexed volume should report zero counts: %+v", vs.Offload) + } +} + +// TestBuildSkipOffloadReadiness: the TUI tick path skips the whole-index +// gate pass but still reports the policy flag; the CLI path computes the +// counts. Both see the same coverage/durability grid. +func TestBuildSkipOffloadReadiness(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + root, idx := indexTree(t, s, "photos") + v, _ := s.GetVolumeByName(ctx, "photos") + self, _ := s.GetSelfNode(ctx) + if err := s.UpsertDestinationRunIDVerified(ctx, v.ID, "dest", self.ID, idx.RunID, store.VerifyMethodBlake3, false); err != nil { + t.Fatalf("seed vector: %v", err) + } + recordSuccessfulSync(t, s, v.ID, "dest") + cfg := cfgFor("photos", root, []string{"dest"}, []string{"dest"}, []string{"dest"}, nil) + + skipped, err := BuildWithOptions(ctx, s, cfg, Options{SkipOffloadReadiness: true}) + if err != nil { + t.Fatalf("BuildWithOptions(skip): %v", err) + } + so := skipped.Volumes[0].Offload + if !so.Applicable || so.OffloadableFiles != 0 || so.PresentFiles != 0 { + t.Errorf("skip-readiness offload = %+v, want applicable with zero counts", so) + } + full, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build(full): %v", err) + } + fo := full.Volumes[0].Offload + if !fo.Applicable || fo.OffloadableFiles != 3 || fo.PresentFiles != 3 { + t.Errorf("full offload = %+v, want 3 offloadable of 3 present", fo) + } + // The coverage grid is identical regardless of the readiness option. + if len(skipped.Volumes[0].Targets) != len(full.Volumes[0].Targets) { + t.Errorf("target count differs between skip and full builds") + } +} + +// TestBuildRelayedRequiredTarget: a target named only in offload_requires +// (never in sync_to) reads as relayed for coverage but is scored on its +// durability; with no evidence yet it is amber, not red. +func TestBuildRelayedRequiredTarget(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + root, _ := indexTree(t, s, "photos") + cfg := cfgFor("photos", root, []string{"nas"}, []string{"nas", "s3archive"}, nil, []string{"nas"}) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + var relayed *TargetStatus + for i := range rep.Volumes[0].Targets { + if rep.Volumes[0].Targets[i].Name == "s3archive" { + relayed = &rep.Volumes[0].Targets[i] + } + } + if relayed == nil { + t.Fatalf("relayed target s3archive missing from grid") + } + if relayed.SyncTarget || !relayed.Required || relayed.Kind != KindRelayed { + t.Errorf("relayed target classification wrong: %+v", relayed) + } + if relayed.SyncLevel != LevelNeutral { + t.Errorf("relayed sync level = %v, want neutral", relayed.SyncLevel) + } + if relayed.Durability == nil || relayed.Durability.Covered { + t.Errorf("relayed durability should be uncovered: %+v", relayed.Durability) + } + if relayed.Level() != LevelWarn { + t.Errorf("relayed target level = %v, want amber (evidence not yet arrived)", relayed.Level()) + } +} diff --git a/status/durability.go b/status/durability.go new file mode 100644 index 0000000..e814a1a --- /dev/null +++ b/status/durability.go @@ -0,0 +1,106 @@ +package status + +import ( + "context" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" +) + +// buildDurability computes the local-content durability of a volume on one +// target. Durability is tracked when the target is offload-required or when +// a vector component for it already exists; a non-required target with no +// recorded evidence returns nil (nothing to report). +func buildDurability(ctx context.Context, s *store.Store, volID int64, self store.Node, selfMax int64, vol *config.Volume, name string, now time.Time) (*Durability, error) { + comp, hasComp, err := selfComponent(ctx, s, volID, name, self.ID) + if err != nil { + return nil, err + } + required := contains(vol.OffloadRequires, name) + if !required && !hasComp { + return nil, nil + } + d := &Durability{ + LocalContent: selfMax > 0, + MaxEvidenceAge: vol.OffloadMaxEvidenceAge, + } + if hasComp { + d.Method = comp.VerifyMethod + d.Covered = comp.OriginRunID >= selfMax + d.EvidenceAge = evidenceAge(comp.VerifiedAtNs, now) + } + d.Stale = d.LocalContent && evidenceStale(d) + d.Level = durabilityLevel(d, required) + return d, nil +} + +// evidenceStale reports whether a max-evidence-age policy is set and the +// covering component's verification is older than it. It is fail-closed: +// an unknown evidence age (no component, or one that never carried a +// verification instant) counts as stale. +func evidenceStale(d *Durability) bool { + if d.MaxEvidenceAge <= 0 { + return false + } + if d.EvidenceAge == nil { + return true + } + return *d.EvidenceAge > d.MaxEvidenceAge +} + +// durabilityLevel scores the durability dimension. Content that does not +// originate locally is vacuously safe (neutral). A required target +// escalates to amber when local content is not yet covered or the evidence +// has aged out; a non-required target never escalates "am I safe?" — it +// shows green when healthy and neutral otherwise, since offload does not +// gate on it. +func durabilityLevel(d *Durability, required bool) Level { + if !d.LocalContent { + return LevelNeutral + } + if !required { + if d.Covered && !d.Stale { + return LevelOK + } + return LevelNeutral + } + if !d.Covered || d.Stale { + return LevelWarn + } + return LevelOK +} + +// selfComponent returns the target's durability-vector component for this +// node's own origin — the coverage of locally-introduced content, which is +// what the offload gate checks on an edge machine. The bool is false when +// the target has no component for the self node. +func selfComponent(ctx context.Context, s *store.Store, volID int64, name string, selfID int64) (store.DestinationRunID, bool, error) { + comps, err := s.ListDestinationRunIDs(ctx, volID, name) + if err != nil { + return store.DestinationRunID{}, false, err + } + for _, c := range comps { + if c.OriginNodeID == selfID { + return c, true, nil + } + } + return store.DestinationRunID{}, false, nil +} + +// selfOriginMax returns the highest origin run this node contributes to the +// volume's present set — the coverage floor the durability vector must +// reach for local content to count as durable. Zero means this node +// originates no present content in the volume. +func selfOriginMax(ctx context.Context, s *store.Store, volID, selfID int64) (int64, error) { + comps, err := s.PresentOriginMaxima(ctx, volID, selfID) + if err != nil { + return 0, err + } + for _, c := range comps { + if c.OriginNodeID == selfID { + return c.OriginRunID, nil + } + } + return 0, nil +} diff --git a/status/labels.go b/status/labels.go new file mode 100644 index 0000000..790d4d9 --- /dev/null +++ b/status/labels.go @@ -0,0 +1,170 @@ +package status + +import ( + "fmt" + "time" +) + +// This file holds the neutral text labels every surface renders from a +// Report. They live in the query layer so the CLI grid and the TUI +// dashboard show identical words for identical facts (the "build the +// shared layer once, consume it from both" contract); each surface adds +// only its own colour and framing. The functions return strings and never +// write to any stream, keeping the library free of I/O. + +// HumanAge renders a duration in the largest useful whole unit (5s, 3m, +// 2h, 4d), the compact form both the CLI grid and the TUI use for ages. +func HumanAge(d time.Duration) string { + switch { + case d < time.Minute: + return fmt.Sprintf("%ds", int(d.Seconds())) + case d < time.Hour: + return fmt.Sprintf("%dm", int(d.Minutes())) + case d < 24*time.Hour: + return fmt.Sprintf("%dh", int(d.Hours())) + default: + return fmt.Sprintf("%dd", int(d.Hours()/24)) + } +} + +// IndexLabel renders a volume's index freshness or the never-indexed gap +// (friction-log F4 — a configured-but-unindexed volume must say so, not +// render blank). +func IndexLabel(v VolumeStatus) string { + if !v.Indexed || v.LastIndexAgo == nil { + return "never indexed" + } + return HumanAge(*v.LastIndexAgo) + " ago" +} + +// RoleLabel names why a target is in the grid: a sync destination, an +// offload-gating target, or both. +func RoleLabel(t TargetStatus) string { + switch { + case t.SyncTarget && t.Required: + return "sync+offload" + case t.SyncTarget: + return "sync" + case t.Required: + return "offload" + default: + return "—" + } +} + +// LastSyncLabel renders the last successful sync's age, "never" for a +// configured-but-unsynced target, or "relayed" for a target this node +// never pushes to (its evidence arrives via a peer durability pull). +func LastSyncLabel(t TargetStatus) string { + if !t.SyncTarget { + return "relayed" + } + if t.LastSyncAgo == nil { + return "never" + } + return HumanAge(*t.LastSyncAgo) + " ago" +} + +// StateLabel names the coverage state, most-specific first: standing states +// outrank the latest terminal outcome (a failed or partial run — both more +// recent than any success), which outranks freshness. +func StateLabel(t TargetStatus) string { + switch t.Standing { + case StandingAlarm: + return "alarm" + case StandingRefused: + return "refused" + case StandingNeedsBootstrap: + return "needs-init" + } + switch t.LastOutcome { + case "failed": + return "failed" + case "partial": + return "partial" + } + if !t.SyncTarget { + return "—" + } + if t.LastSyncAgo == nil { + return "never-synced" + } + switch t.SyncLevel { + case LevelWarn: + return "late" + case LevelCritical: + return "stalled" + default: + return "ok" + } +} + +// DurableLabel answers "is my local content durable here": yes/no for a +// tracked target, n/a when this node originates no content in the volume, +// and — when nothing is tracked at all. +func DurableLabel(t TargetStatus) string { + if t.Durability == nil { + return "—" + } + if !t.Durability.LocalContent { + return "n/a" + } + if t.Durability.Covered { + return "yes" + } + return "no" +} + +// MethodLabel renders the verify method behind the covering component. +func MethodLabel(t TargetStatus) string { + if t.Durability == nil || t.Durability.Method == "" { + return "—" + } + return t.Durability.Method +} + +// EvidenceLabel renders the evidence age, and — when the volume sets +// offload_max_evidence_age — the age against that budget, flagging stale +// evidence loudly. +func EvidenceLabel(t TargetStatus) string { + d := t.Durability + if d == nil || !d.LocalContent { + return "—" + } + age := "unknown" + if d.EvidenceAge != nil { + age = HumanAge(*d.EvidenceAge) + } + if d.MaxEvidenceAge > 0 { + age = fmt.Sprintf("%s / %s", age, HumanAge(d.MaxEvidenceAge)) + } + if d.Stale { + age += " STALE" + } + return age +} + +// OffloadLabel renders the "N offloadable now" decision-support line +// (friction-log F17) or a note that the volume has no offload policy. +func OffloadLabel(o OffloadReadiness) string { + if !o.Applicable { + return "offload: no policy" + } + return fmt.Sprintf("offloadable now: %s (%d files) of %s present (%d files)", + HumanBytes(o.OffloadableBytes), o.OffloadableFiles, + HumanBytes(o.PresentBytes), o.PresentFiles) +} + +// TrafficLight maps a level to the green/amber/red word the CLI summary +// line and the surfaces' headers use. Neutral reads as green — no problem +// to report. +func TrafficLight(l Level) string { + switch l { + case LevelWarn: + return "amber" + case LevelCritical: + return "red" + default: + return "green" + } +} diff --git a/status/logic_test.go b/status/logic_test.go new file mode 100644 index 0000000..09c3ab2 --- /dev/null +++ b/status/logic_test.go @@ -0,0 +1,184 @@ +package status + +import ( + "database/sql" + "testing" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" +) + +func TestCadenceLevel(t *testing.T) { + h := time.Hour + cases := []struct { + age time.Duration + cadence time.Duration + want Level + }{ + {30 * time.Minute, h, LevelOK}, // within cadence + {h, h, LevelOK}, // exactly at cadence + {2 * h, h, LevelWarn}, // late, within lateFactor + {lateFactor * h, h, LevelWarn}, // at the boundary + {lateFactor*h + time.Minute, h, LevelCritical}, // beyond lateFactor + {1000 * h, 0, LevelOK}, // no cadence: any age is fine + } + for _, c := range cases { + if got := cadenceLevel(c.age, c.cadence); got != c.want { + t.Errorf("cadenceLevel(%s, %s) = %v, want %v", c.age, c.cadence, got, c.want) + } + } +} + +func refusedRun(errMsg string) *store.Run { + return &store.Run{Status: store.RunStatusRefused, Error: sql.NullString{String: errMsg, Valid: errMsg != ""}} +} + +func TestClassifyStanding(t *testing.T) { + bootstrapErr := "destination has no marker — re-run with --init to bootstrap" + cases := []struct { + name string + lastTerm *store.Run + hadSuccess bool + want Standing + }{ + {"no runs", nil, false, StandingNone}, + {"fresh bootstrap refusal", refusedRun(bootstrapErr), false, StandingNeedsBootstrap}, + {"refusal after prior success", refusedRun(bootstrapErr), true, StandingRefused}, + {"non-bootstrap refusal", refusedRun("layout guard: history is not packed"), false, StandingRefused}, + {"success", &store.Run{Status: store.RunStatusSuccess}, true, StandingNone}, + } + for _, c := range cases { + if got := classifyStanding(c.lastTerm, c.hadSuccess); got != c.want { + t.Errorf("%s: classifyStanding = %v, want %v", c.name, got, c.want) + } + } +} + +func TestDurabilityLevel(t *testing.T) { + dur := func(local, covered, stale bool) *Durability { + return &Durability{LocalContent: local, Covered: covered, Stale: stale} + } + cases := []struct { + name string + d *Durability + required bool + want Level + }{ + {"no local content", dur(false, false, false), true, LevelNeutral}, + {"required covered fresh", dur(true, true, false), true, LevelOK}, + {"required uncovered", dur(true, false, false), true, LevelWarn}, + {"required covered stale", dur(true, true, true), true, LevelWarn}, + {"non-required covered", dur(true, true, false), false, LevelOK}, + {"non-required uncovered", dur(true, false, false), false, LevelNeutral}, + } + for _, c := range cases { + if got := durabilityLevel(c.d, c.required); got != c.want { + t.Errorf("%s: durabilityLevel = %v, want %v", c.name, got, c.want) + } + } +} + +func TestEvidenceStale(t *testing.T) { + age := func(d time.Duration) *time.Duration { return &d } + cases := []struct { + name string + d *Durability + want bool + }{ + {"no policy", &Durability{MaxEvidenceAge: 0, EvidenceAge: age(time.Hour)}, false}, + {"unknown age fail-closed", &Durability{MaxEvidenceAge: time.Hour, EvidenceAge: nil}, true}, + {"fresh", &Durability{MaxEvidenceAge: time.Hour, EvidenceAge: age(30 * time.Minute)}, false}, + {"aged out", &Durability{MaxEvidenceAge: time.Hour, EvidenceAge: age(2 * time.Hour)}, true}, + } + for _, c := range cases { + if got := evidenceStale(c.d); got != c.want { + t.Errorf("%s: evidenceStale = %v, want %v", c.name, got, c.want) + } + } +} + +func TestSyncLevelTerminalOutcome(t *testing.T) { + age := func(d time.Duration) *time.Duration { return &d } + // A fresh success would be OK, but a more-recent failed/partial terminal + // run must not be masked by that freshness. + fresh := TargetStatus{SyncTarget: true, LastSyncAgo: age(time.Minute), Cadence: time.Hour} + cases := []struct { + name string + terminal *store.Run + want Level + }{ + {"failed outranks fresh success", &store.Run{Status: store.RunStatusFailed}, LevelCritical}, + {"partial is at least amber", &store.Run{Status: store.RunStatusPartial}, LevelWarn}, + {"success uses freshness", &store.Run{Status: store.RunStatusSuccess}, LevelOK}, + } + for _, c := range cases { + if got := syncLevel(fresh, c.terminal); got != c.want { + t.Errorf("%s: syncLevel = %v, want %v", c.name, got, c.want) + } + } +} + +func TestLevelExitCode(t *testing.T) { + cases := map[Level]int{LevelNeutral: 0, LevelOK: 0, LevelWarn: 1, LevelCritical: 2} + for l, want := range cases { + if got := l.ExitCode(); got != want { + t.Errorf("%v.ExitCode() = %d, want %d", l, got, want) + } + } +} + +func TestHumanBytes(t *testing.T) { + cases := map[int64]string{ + 0: "0 B", + 512: "512 B", + 1024: "1.0 KiB", + 1536: "1.5 KiB", + 1024 * 1024: "1.0 MiB", + 3 * 1024 * 1024 * 1024: "3.0 GiB", + } + for n, want := range cases { + if got := HumanBytes(n); got != want { + t.Errorf("HumanBytes(%d) = %q, want %q", n, got, want) + } + } +} + +func TestOrderedTargets(t *testing.T) { + // sync_to first in declared order, then offload_requires entries not + // already listed, deduplicated ("nas" appears in both). + vol := &config.Volume{SyncTo: []string{"nas"}, OffloadRequires: []string{"nas", "s3archive"}} + got := orderedTargets(vol) + want := []string{"nas", "s3archive"} + if len(got) != len(want) { + t.Fatalf("orderedTargets = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("orderedTargets = %v, want %v", got, want) + } + } +} + +func TestStateLabel(t *testing.T) { + age := func(d time.Duration) *time.Duration { return &d } + cases := []struct { + name string + t TargetStatus + want string + }{ + {"alarm", TargetStatus{Standing: StandingAlarm}, "alarm"}, + {"needs-init", TargetStatus{Standing: StandingNeedsBootstrap}, "needs-init"}, + {"failed", TargetStatus{LastOutcome: "failed", SyncTarget: true}, "failed"}, + {"partial", TargetStatus{LastOutcome: "partial", SyncTarget: true, LastSyncAgo: age(time.Minute), SyncLevel: LevelWarn}, "partial"}, + {"relayed", TargetStatus{SyncTarget: false, Required: true}, "—"}, + {"never", TargetStatus{SyncTarget: true}, "never-synced"}, + {"ok", TargetStatus{SyncTarget: true, LastSyncAgo: age(time.Minute), SyncLevel: LevelOK}, "ok"}, + {"late", TargetStatus{SyncTarget: true, LastSyncAgo: age(time.Hour), SyncLevel: LevelWarn}, "late"}, + } + for _, c := range cases { + if got := StateLabel(c.t); got != c.want { + t.Errorf("%s: StateLabel = %q, want %q", c.name, got, c.want) + } + } +} diff --git a/status/runs.go b/status/runs.go new file mode 100644 index 0000000..2883637 --- /dev/null +++ b/status/runs.go @@ -0,0 +1,101 @@ +package status + +import ( + "context" + "database/sql" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" +) + +// latestSuccessfulSync returns the most recent fully-landed sync of the +// pair, or nil when there is none. sql.ErrNoRows is the expected "never +// synced" signal and maps to nil; any other error propagates. +func latestSuccessfulSync(ctx context.Context, s *store.Store, volID int64, name string) (*store.Run, error) { + run, err := s.LatestSuccessfulSyncRun(ctx, volID, name) + if store.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, err + } + return &run, nil +} + +// latestTerminalSync returns the pair's most recent terminal sync run of +// any outcome (success, partial, failed, refused), or nil when there is +// none. It is the basis for the standing-state classification: a refused +// or failed latest run outranks an older success. +func latestTerminalSync(ctx context.Context, s *store.Store, volID int64, name string) (*store.Run, error) { + run, err := s.LatestFinishedRun(ctx, store.RunKindSync, volID, name) + if store.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, err + } + return &run, nil +} + +// lastIndexAge returns the age of the volume's most recent successful (or +// partial) index run, or nil when it has never been indexed. +func lastIndexAge(ctx context.Context, s *store.Store, volID int64, now time.Time) (*time.Duration, error) { + run, err := s.LatestSuccessfulIndexRun(ctx, volID) + if store.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, err + } + return ageOf(&run, now), nil +} + +// ageOf returns how long ago a run ended, or nil for a nil run or one that +// never recorded an end. Negative ages (clock skew) clamp to zero so a +// surface never renders "-3s ago". +func ageOf(run *store.Run, now time.Time) *time.Duration { + if run == nil || !run.EndedAtNs.Valid { + return nil + } + d := now.Sub(time.Unix(0, run.EndedAtNs.Int64)) + if d < 0 { + d = 0 + } + return &d +} + +// evidenceAge returns how long ago a durability component was last +// verified, or nil when the instant is unknown (NULL verified_at_ns). +// Clamped at zero for clock skew, matching ageOf. +func evidenceAge(verifiedAtNs sql.NullInt64, now time.Time) *time.Duration { + if !verifiedAtNs.Valid { + return nil + } + d := now.Sub(time.Unix(0, verifiedAtNs.Int64)) + if d < 0 { + d = 0 + } + return &d +} + +// targetKind classifies a target name against this node's config. +func targetKind(cfg *config.Config, name string) TargetKind { + if _, ok := cfg.Nodes[name]; ok { + return KindNode + } + if _, ok := cfg.Destinations[name]; ok { + return KindDestination + } + return KindRelayed +} + +// contains reports whether name appears in names. +func contains(names []string, name string) bool { + for _, n := range names { + if n == name { + return true + } + } + return false +} diff --git a/status/status.go b/status/status.go new file mode 100644 index 0000000..d6b7f83 --- /dev/null +++ b/status/status.go @@ -0,0 +1,285 @@ +// Package status is squirrel's shared "am I safe?" query layer. It reads +// the local index and config and produces, per configured volume and per +// (volume × target), the coverage and durability facts both the CLI +// `squirrel status` command and the TUI dashboard render — friction-log +// F16 (per-target sync coverage), F17 and F23 (durability and offload +// readiness). It is read-only introspection (ux-principle 2): it opens no +// run, reads no disk bytes, and mutates nothing. +// +// The package returns data and a severity per cell; it does not format for +// any particular surface beyond the neutral HumanBytes helper. Each +// consumer renders the Report its own way while sharing one source of +// truth for the facts and their severities. +package status + +import ( + "fmt" + "time" +) + +// Level is the "am I safe?" severity of a cell, a volume, or the whole +// report. It is ordered so the worst of a set is its maximum, and it maps +// directly onto the CLI's scriptable exit code via ExitCode. +type Level int + +const ( + // LevelNeutral is an informational state that says nothing about + // safety: a relayed target this node never pushes to, a durability + // figure for a volume with no offload policy, content that does not + // originate locally. It never escalates the worst-of aggregation and + // contributes exit code 0. + LevelNeutral Level = iota + // LevelOK is an affirmatively-healthy state — caught up within cadence, + // durable and freshly verified. "You may close the laptop." + LevelOK + // LevelWarn is amber: not caught up yet, needs a one-time bootstrap, + // local content not yet durable on a required target, evidence aged + // past the policy. Recoverable and expected, not a failure. + LevelWarn + // LevelCritical is red: a latched alarm, a refused sync that regressed + // from a prior success, a failed transfer, or a pair far past its + // cadence. Something is wrong and needs attention. + LevelCritical +) + +// String renders a Level as a short lowercase word for CLI output and +// tests. It is deliberately stable — scripts and golden tests key on it. +func (l Level) String() string { + switch l { + case LevelNeutral: + return "neutral" + case LevelOK: + return "ok" + case LevelWarn: + return "amber" + case LevelCritical: + return "red" + default: + return "unknown" + } +} + +// ExitCode maps a Level onto the process exit status `squirrel status` +// returns: 0 for green/neutral, 1 for amber, 2 for red. Neutral and OK +// share 0 because neither reports a problem to a script. +func (l Level) ExitCode() int { + switch l { + case LevelWarn: + return 1 + case LevelCritical: + return 2 + default: + return 0 + } +} + +// worst returns the higher-severity of two levels. +func worst(a, b Level) Level { + if b > a { + return b + } + return a +} + +// Standing is a per-target standing state carried over from #157: an +// abnormal condition that persists across cadence ticks until resolved, +// rather than a single run's outcome. +type Standing int + +const ( + // StandingNone means no standing condition — the target's health is + // whatever its latest sync freshness says. + StandingNone Standing = iota + // StandingNeedsBootstrap is a fresh destination awaiting its one-time + // `sync --init`: the latest run refused with a bootstrap reason and the + // pair has never had a successful sync. Amber — an expected human step, + // not a fault (F10). + StandingNeedsBootstrap + // StandingRefused is a preflight safety gate declining after the pair + // had previously succeeded (a marker gone missing, a layout guard) — + // the destination broke. Red (F26). + StandingRefused + // StandingAlarm is a latched verify mismatch on the destination (F30). + // Red until an operator clears it. + StandingAlarm +) + +// String renders a Standing as a short word for display and tests. +func (s Standing) String() string { + switch s { + case StandingNeedsBootstrap: + return "needs-init" + case StandingRefused: + return "refused" + case StandingAlarm: + return "alarm" + default: + return "ok" + } +} + +// TargetKind classifies a target name against this node's config: a local +// destination, a peer node, or a relayed target named only in +// offload_requires (evidence for it arrives via a peer durability pull; +// this node has no local config for it). +type TargetKind string + +const ( + KindDestination TargetKind = "destination" + KindNode TargetKind = "node" + KindRelayed TargetKind = "relayed" +) + +// HumanBytes renders a byte count in binary units with one decimal place +// above KiB, for the "N GB offloadable now" figure both surfaces show. It +// is the one formatting helper the library exposes, because the two +// consumers must render identical numbers. +func HumanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for x := n / unit; x >= unit; x /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} + +// Report is the whole snapshot for one node: every configured volume with +// its per-target coverage and durability. Now is the instant every age and +// staleness judgement in the report was taken against, so a consumer +// renders one coherent moment. +type Report struct { + Now time.Time + Volumes []VolumeStatus +} + +// Level is the worst level across every volume — the report's single +// "am I safe?" answer and the basis for the CLI exit code. +func (r Report) Level() Level { + l := LevelNeutral + for _, v := range r.Volumes { + l = worst(l, v.Level()) + } + return l +} + +// VolumeStatus is one configured volume's coverage and durability. +type VolumeStatus struct { + Name string + Path string + // Indexed is true when the volume has a row in the local index (it has + // been indexed at least once). A configured-but-never-indexed volume is + // Indexed=false with no targets carrying run history — the F4 "a freshly + // configured machine reports nothing" gap, surfaced as amber rather than + // a blank screen. + Indexed bool + // LastIndexAgo is the age of the most recent successful index run, or + // nil when the volume has never been indexed. + LastIndexAgo *time.Duration + Targets []TargetStatus + // Offload is the volume's offload-readiness tally ("N GB offloadable + // now"). Offload.Applicable is false for a volume with no offload + // policy. + Offload OffloadReadiness +} + +// Level is the worst level across the volume's targets, escalated to amber +// when the volume has never been indexed. +func (v VolumeStatus) Level() Level { + l := LevelNeutral + if !v.Indexed { + l = LevelWarn + } + for _, t := range v.Targets { + l = worst(l, t.Level()) + } + return l +} + +// TargetStatus is one (volume × target) cell: is this target caught up, +// and is the volume's local content durable on it? +type TargetStatus struct { + Name string + Kind TargetKind + // SyncTarget is true when the target is in the volume's sync_to (this + // node pushes to it). Required is true when it is in offload_requires + // (its durability gates offload). A relayed target is Required but not + // a SyncTarget. + SyncTarget bool + Required bool + // Cadence is the volume's sync_every; lateness of the last sync is + // judged relative to it, not a global constant. Zero means no scheduled + // cadence, so a successful sync of any age reads as OK. + Cadence time.Duration + // LastSyncAgo is the age of the most recent successful sync to this + // target, or nil when this node has never successfully synced it (or + // never pushes to it — a relayed target). + LastSyncAgo *time.Duration + // Standing is the target's standing state (alarm / refused / + // needs-bootstrap / none). + Standing Standing + // AlarmDetail is the latched alarm's detail text when Standing is + // StandingAlarm, for the surface to show what tripped. + AlarmDetail string + // LastOutcome is the status of the pair's most recent terminal sync run + // (success / partial / failed / refused / aborted), or "" when this + // node has never run a sync against the target. It lets a surface name + // a failed transfer precisely, distinct from an aged-out success. + LastOutcome string + // SyncLevel is the severity of the coverage dimension alone (freshness + // and standing state); durability is scored separately in Durability. + SyncLevel Level + // Durability is the local-content durability on this target, or nil when + // durability is not tracked for the pair (a non-required target with no + // recorded vector). + Durability *Durability +} + +// Level is the worst of the target's coverage and durability dimensions. +func (t TargetStatus) Level() Level { + l := t.SyncLevel + if t.Durability != nil { + l = worst(l, t.Durability.Level) + } + return l +} + +// Durability is the local-content durability of a volume on one target. +type Durability struct { + // LocalContent is true when this node originates present content in the + // volume. When false, "is my local content durable here" is vacuously + // satisfied and Level is neutral — the hub, whose photos originate on + // the edges, sees this for its inbound volumes. + LocalContent bool + // Covered is true when the target's durability vector covers this + // node's highest local origin run — every locally-originated present + // content is recorded durable there. + Covered bool + // Method is the verify method behind the covering component (blake3, + // peer-blake3, kopia-verify, presence+size, size+mtime), or "" when + // there is no component. + Method string + // EvidenceAge is how long ago the covering component was last verified, + // or nil when unknown (no component, or a component that never carried + // a verification instant). + EvidenceAge *time.Duration + // MaxEvidenceAge is the volume's offload_max_evidence_age; zero means no + // staleness policy. Stale is true when the policy is set and the + // evidence is older than it (or its age is unknown). + MaxEvidenceAge time.Duration + Stale bool + Level Level +} + +// OffloadReadiness is a volume's "what could I offload right now" tally. +type OffloadReadiness struct { + // Applicable is false when the volume declares no offload policy. + Applicable bool + OffloadableFiles int + OffloadableBytes int64 + PresentFiles int + PresentBytes int64 +} diff --git a/store/abort_runs_test.go b/store/abort_runs_test.go new file mode 100644 index 0000000..5a37aa0 --- /dev/null +++ b/store/abort_runs_test.go @@ -0,0 +1,113 @@ +package store + +import ( + "context" + "testing" +) + +// TestAbortRunningRuns reaps every in-flight run to 'aborted', leaves +// terminal rows untouched, is idempotent, and records an 'abort' +// transition per reaped row. +func TestAbortRunningRuns(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + vol, err := s.CreateVolume(ctx, "v", "/v") + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + + idx, err := s.BeginIndexRun(ctx, RunKindIndex, vol.ID, true) + if err != nil { + t.Fatalf("BeginIndexRun: %v", err) + } + syncing, err := s.BeginRun(ctx, RunKindSync, vol.ID, "offsite", false) + if err != nil { + t.Fatalf("BeginRun sync: %v", err) + } + done, err := s.BeginRun(ctx, RunKindSync, vol.ID, "other", false) + if err != nil { + t.Fatalf("BeginRun done: %v", err) + } + if err := s.FinishRun(ctx, done, RunStatusSuccess, "", 5); err != nil { + t.Fatalf("FinishRun done: %v", err) + } + + ids, err := s.AbortRunningRuns(ctx, "killed") + if err != nil { + t.Fatalf("AbortRunningRuns: %v", err) + } + if len(ids) != 2 || ids[0] != idx || ids[1] != syncing { + t.Fatalf("reaped ids = %v, want [%d %d]", ids, idx, syncing) + } + + for _, id := range []int64{idx, syncing} { + run, err := s.GetRun(ctx, id) + if err != nil { + t.Fatalf("GetRun(%d): %v", id, err) + } + if run.Status != RunStatusAborted { + t.Fatalf("run %d status = %q, want aborted", id, run.Status) + } + if !run.EndedAtNs.Valid { + t.Fatalf("aborted run %d has no ended_at_ns", id) + } + if run.Error.String != "killed" { + t.Fatalf("aborted run %d error = %q, want the reap reason", id, run.Error.String) + } + if n := countTransition(t, s, id, TransitionAbort); n != 1 { + t.Fatalf("run %d abort-transition count = %d, want 1", id, n) + } + } + + // The already-terminal success is untouched. + if run, _ := s.GetRun(ctx, done); run.Status != RunStatusSuccess { + t.Fatalf("finished run status = %q, want success", run.Status) + } + + // Idempotent: a second reap finds nothing. + again, err := s.AbortRunningRuns(ctx, "killed") + if err != nil { + t.Fatalf("second AbortRunningRuns: %v", err) + } + if len(again) != 0 { + t.Fatalf("second reap returned %v, want none", again) + } +} + +// TestLatestFinishedRunStatusScope pins the cadence-watermark policy: a +// 'refused' run consumes the cadence window (so the scheduler backs off a +// dead destination) while an 'aborted' run does not (so a crashed run is +// re-attempted). +func TestLatestFinishedRunStatusScope(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + vol, err := s.CreateVolume(ctx, "v", "/v") + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + + refused, err := s.BeginRun(ctx, RunKindSync, vol.ID, "dead", false) + if err != nil { + t.Fatalf("BeginRun refused: %v", err) + } + if err := s.FinishRun(ctx, refused, RunStatusRefused, "no marker", 0); err != nil { + t.Fatalf("FinishRun refused: %v", err) + } + got, err := s.LatestFinishedRun(ctx, RunKindSync, vol.ID, "dead") + if err != nil { + t.Fatalf("refused run should be a finished run: %v", err) + } + if got.ID != refused || got.Status != RunStatusRefused { + t.Fatalf("LatestFinishedRun = %+v, want the refused run", got) + } + + if _, err := s.BeginRun(ctx, RunKindSync, vol.ID, "crashed", false); err != nil { + t.Fatalf("BeginRun crashed: %v", err) + } + if _, err := s.AbortRunningRuns(ctx, "killed"); err != nil { + t.Fatalf("AbortRunningRuns: %v", err) + } + if _, err := s.LatestFinishedRun(ctx, RunKindSync, vol.ID, "crashed"); !IsNotFound(err) { + t.Fatalf("aborted run must not count as a finished run, got %v", err) + } +} diff --git a/store/contested_paths.go b/store/contested_paths.go new file mode 100644 index 0000000..e5956a8 --- /dev/null +++ b/store/contested_paths.go @@ -0,0 +1,238 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// ContestedPath is one row of the contested_paths latch: a path frozen by +// a peer-sync conflict so divergent edits stop ping-ponging (#158, F27). +// While the row stands the receiver refuses to re-supersede the path (the +// `contested` disposition) and every node that knows about it surfaces a +// badge; both versions stay preserved — the winner live at Path, the loser +// under .squirrel-conflicts/ at PreservedAtPath. It is derived standing +// state, like DestinationAlarm: the permanent record of every conflict +// lives in runs/runs_audit and the preserved bytes on disk, so clearing +// the latch loses no history. +// +// LiveBlake3 is the frozen winner (nil when a node recorded the freeze +// without knowing the winning digest); PreservedBlake3 is the loser. +// PeerNodeID is the node that caused the freeze — the initiator peer on +// the receiver, or the destination peer on an initiator — and is 0 when +// unattributed. RaisedRunID references the run that first raised the +// latch; RaisedAtNs is stable across repeated conflicts so a surface can +// show "contested since". +type ContestedPath struct { + VolumeID int64 + Path string + LiveBlake3 []byte + PreservedBlake3 []byte + PreservedAtPath string + PeerNodeID int64 + RaisedRunID int64 + RaisedAtNs int64 +} + +// RaiseContested latches a contested freeze on (volume, path) when none is +// already active. The raise is idempotent: an existing latch keeps its +// original digests, timestamp, and run so a repeated conflict does not +// reset "contested since" or append a duplicate audit row — the freeze is +// recorded once, not every cadence tick (the F27 fix). A +// 'contested-raise' runs_audit entry is written against RaisedRunID in the +// same transaction as the insert, only when a fresh latch is created. +func (s *Store) RaiseContested(ctx context.Context, c ContestedPath) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin raise contested %q: %w", c.Path, err) + } + defer func() { _ = tx.Rollback() }() + + atNs := NowNs() + res, err := tx.ExecContext(ctx, ` + INSERT INTO contested_paths ( + volume_id, path, live_blake3, preserved_blake3, preserved_at_path, + peer_node_id, raised_run_id, raised_at_ns + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(volume_id, path) DO NOTHING + `, c.VolumeID, c.Path, nullBlob(c.LiveBlake3), nullBlob(c.PreservedBlake3), + nullString(c.PreservedAtPath), nullPeerID(c.PeerNodeID), c.RaisedRunID, atNs) + if err != nil { + return fmt.Errorf("raise contested %q: %w", c.Path, err) + } + inserted, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("raise contested %q rows affected: %w", c.Path, err) + } + if inserted == 0 { + // Already frozen: keep the original latch untouched. + return tx.Commit() + } + if err := appendRunAuditTx(ctx, tx, RunAuditEntry{ + RunID: c.RaisedRunID, Transition: TransitionContestedRaise, Note: c.Path, + }, atNs); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit raise contested %q: %w", c.Path, err) + } + return nil +} + +// ClearContested clears the active freeze on (volume, path), if any, and +// appends a 'contested-clear' runs_audit entry (tagged with operator) +// against the run that raised it. Returns whether a latch was active. +// Deleting the latch loses no history: the raise and this clear both +// survive in runs_audit, every conflict run survives in runs, and the +// preserved bytes stay on disk under .squirrel-conflicts/. +func (s *Store) ClearContested(ctx context.Context, volumeID int64, path, operator string) (bool, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, fmt.Errorf("begin clear contested %q: %w", path, err) + } + defer func() { _ = tx.Rollback() }() + + var raisedRunID int64 + err = tx.QueryRowContext(ctx, + `SELECT raised_run_id FROM contested_paths WHERE volume_id = ? AND path = ?`, + volumeID, path).Scan(&raisedRunID) + if IsNotFound(err) { + return false, tx.Commit() + } + if err != nil { + return false, fmt.Errorf("read contested %q: %w", path, err) + } + if _, err := tx.ExecContext(ctx, + `DELETE FROM contested_paths WHERE volume_id = ? AND path = ?`, + volumeID, path); err != nil { + return false, fmt.Errorf("clear contested %q: %w", path, err) + } + if err := appendRunAuditTx(ctx, tx, RunAuditEntry{ + RunID: raisedRunID, Transition: TransitionContestedClear, Operator: operator, Note: path, + }, NowNs()); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("commit clear contested %q: %w", path, err) + } + return true, nil +} + +// IsPathContested reports whether (volume, path) is frozen and, if so, +// returns the frozen winner's digest so the classifier can distinguish the +// winner re-asserting (digest matches — allowed) from a divergent +// re-assertion (digest differs — refused). liveBlake3 is nil when the +// freeze was recorded without a known winner digest. +func (s *Store) IsPathContested(ctx context.Context, volumeID int64, path string) (liveBlake3 []byte, contested bool, err error) { + var digest []byte + err = s.db.QueryRowContext(ctx, + `SELECT live_blake3 FROM contested_paths WHERE volume_id = ? AND path = ?`, + volumeID, path).Scan(&digest) + if IsNotFound(err) { + return nil, false, nil + } + if err != nil { + return nil, false, fmt.Errorf("check contested %q: %w", path, err) + } + return digest, true, nil +} + +// GetContestedPath returns the active freeze on (volume, path), or +// sql.ErrNoRows (IsNotFound) when the path is not contested. +func (s *Store) GetContestedPath(ctx context.Context, volumeID int64, path string) (ContestedPath, error) { + row := s.db.QueryRowContext(ctx, + `SELECT `+contestedColumns+` FROM contested_paths WHERE volume_id = ? AND path = ?`, + volumeID, path) + return scanContestedPath(row.Scan) +} + +// ListContestedPaths returns every active freeze, ordered by (volume, path) +// so the listing is deterministic. An empty slice means nothing is frozen. +func (s *Store) ListContestedPaths(ctx context.Context) ([]ContestedPath, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT `+contestedColumns+` FROM contested_paths ORDER BY volume_id, path`) + if err != nil { + return nil, fmt.Errorf("list contested paths: %w", err) + } + defer rows.Close() + var out []ContestedPath + for rows.Next() { + c, err := scanContestedPath(rows.Scan) + if err != nil { + return nil, err + } + out = append(out, c) + } + return out, rows.Err() +} + +// CountContestedRaisedByRun returns a map from run id to the number of +// contested_paths latches that run raised, for the given run ids. Only +// runs with a non-zero count appear as keys. `squirrel runs` uses it to +// fill the CONFLICTS column for an initiator's own sync runs — where the +// conflict was frozen on a remote receiver, so the receiver-side +// `.squirrel-conflicts/run-N/` prefix count is zero on this node. +func (s *Store) CountContestedRaisedByRun(ctx context.Context, runIDs []int64) (map[int64]int, error) { + out := make(map[int64]int, len(runIDs)) + if len(runIDs) == 0 { + return out, nil + } + placeholders := strings.Repeat("?,", len(runIDs)-1) + "?" + args := make([]any, 0, len(runIDs)) + for _, id := range runIDs { + args = append(args, id) + } + rows, err := s.db.QueryContext(ctx, + `SELECT raised_run_id, COUNT(*) FROM contested_paths + WHERE raised_run_id IN (`+placeholders+`) + GROUP BY raised_run_id`, args...) + if err != nil { + return nil, fmt.Errorf("count contested by run: %w", err) + } + defer rows.Close() + for rows.Next() { + var id int64 + var n int + if err := rows.Scan(&id, &n); err != nil { + return nil, fmt.Errorf("scan contested count row: %w", err) + } + out[id] = n + } + return out, rows.Err() +} + +const contestedColumns = `volume_id, path, live_blake3, preserved_blake3, preserved_at_path, peer_node_id, raised_run_id, raised_at_ns` + +func scanContestedPath(scan func(...any) error) (ContestedPath, error) { + var ( + c ContestedPath + prsvd sql.NullString + peerID sql.NullInt64 + ) + if err := scan(&c.VolumeID, &c.Path, &c.LiveBlake3, &c.PreservedBlake3, + &prsvd, &peerID, &c.RaisedRunID, &c.RaisedAtNs); err != nil { + return ContestedPath{}, err + } + c.PreservedAtPath = prsvd.String + c.PeerNodeID = peerID.Int64 + return c, nil +} + +// nullBlob renders a digest for an INSERT bind: an empty slice becomes a +// NULL BLOB (the "digest unknown" convention), any other value binds as-is. +func nullBlob(b []byte) any { + if len(b) == 0 { + return nil + } + return b +} + +// nullPeerID maps a zero node id to a NULL peer_node_id (unattributed +// freeze) and any positive id to a valid one. +func nullPeerID(id int64) sql.NullInt64 { + if id == 0 { + return sql.NullInt64{} + } + return sql.NullInt64{Int64: id, Valid: true} +} diff --git a/store/contested_paths_test.go b/store/contested_paths_test.go new file mode 100644 index 0000000..672a8dd --- /dev/null +++ b/store/contested_paths_test.go @@ -0,0 +1,143 @@ +package store + +import ( + "bytes" + "context" + "testing" +) + +// TestContestedPathLifecycle exercises raise → idempotent re-raise → +// query → clear and the runs_audit trail each transition leaves. +func TestContestedPathLifecycle(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + volID := makeVolume(t, s, "/v") + first := makeRun(t, s, volID) + live := digest(0xAA) + loser := digest(0xBB) + + c := ContestedPath{ + VolumeID: volID, + Path: "doc.md", + LiveBlake3: live, + PreservedBlake3: loser, + PreservedAtPath: ".squirrel-conflicts/run-1/doc.md", + RaisedRunID: first, + } + if err := s.RaiseContested(ctx, c); err != nil { + t.Fatalf("RaiseContested: %v", err) + } + + got, err := s.GetContestedPath(ctx, volID, "doc.md") + if err != nil { + t.Fatalf("GetContestedPath: %v", err) + } + if !bytes.Equal(got.LiveBlake3, live) || !bytes.Equal(got.PreservedBlake3, loser) { + t.Fatalf("digests = live %x preserved %x, want %x / %x", got.LiveBlake3, got.PreservedBlake3, live, loser) + } + if got.PreservedAtPath != c.PreservedAtPath || got.RaisedRunID != first || got.RaisedAtNs == 0 { + t.Fatalf("latch = %+v, want preserved-path/run/timestamp set", got) + } + firstAt := got.RaisedAtNs + + // IsPathContested returns the frozen winner so the classifier can tell + // the winner re-asserting from a divergent re-assertion. + winner, contested, err := s.IsPathContested(ctx, volID, "doc.md") + if err != nil || !contested { + t.Fatalf("IsPathContested = (%x, %v, %v), want frozen", winner, contested, err) + } + if !bytes.Equal(winner, live) { + t.Fatalf("IsPathContested winner = %x, want %x", winner, live) + } + + // A second conflict on the same path keeps the original latch: same + // run, same timestamp — "contested since" never resets, and no second + // copy is minted (the F27 fix is preserved-once). + second := makeRun(t, s, volID) + if err := s.RaiseContested(ctx, ContestedPath{ + VolumeID: volID, Path: "doc.md", LiveBlake3: digest(0xCC), + PreservedBlake3: digest(0xDD), RaisedRunID: second, + }); err != nil { + t.Fatalf("re-raise: %v", err) + } + got, err = s.GetContestedPath(ctx, volID, "doc.md") + if err != nil { + t.Fatalf("GetContestedPath after re-raise: %v", err) + } + if got.RaisedRunID != first || got.RaisedAtNs != firstAt || !bytes.Equal(got.LiveBlake3, live) { + t.Fatalf("re-raise mutated the latch: %+v", got) + } + + // Exactly one contested-raise audit row against the first run; none + // against the second (the re-raise was a no-op). + if n := countTransition(t, s, first, TransitionContestedRaise); n != 1 { + t.Fatalf("first run contested-raise count = %d, want 1", n) + } + if n := countTransition(t, s, second, TransitionContestedRaise); n != 0 { + t.Fatalf("second run contested-raise count = %d, want 0 (idempotent)", n) + } + + // The run-row conflict column derivation counts one freeze against the + // raising run. + counts, err := s.CountContestedRaisedByRun(ctx, []int64{first, second}) + if err != nil { + t.Fatalf("CountContestedRaisedByRun: %v", err) + } + if counts[first] != 1 || counts[second] != 0 { + t.Fatalf("raised-by-run counts = %v, want {first:1}", counts) + } + + // Clear via resolve: recorded against the raising run, tagged operator. + cleared, err := s.ClearContested(ctx, volID, "doc.md", "alice") + if err != nil { + t.Fatalf("ClearContested: %v", err) + } + if !cleared { + t.Fatal("ClearContested reported no active freeze") + } + if _, err := s.GetContestedPath(ctx, volID, "doc.md"); !IsNotFound(err) { + t.Fatalf("latch still present after clear: %v", err) + } + if _, contested, _ := s.IsPathContested(ctx, volID, "doc.md"); contested { + t.Fatal("IsPathContested still reports frozen after clear") + } + if n := countTransition(t, s, first, TransitionContestedClear); n != 1 { + t.Fatalf("contested-clear count against raising run = %d, want 1", n) + } + + // Clearing an already-cleared path is a benign no-op, not an error. + cleared, err = s.ClearContested(ctx, volID, "doc.md", "alice") + if err != nil || cleared { + t.Fatalf("second ClearContested = (%v, %v), want (false, nil)", cleared, err) + } +} + +// TestContestedPathUnknownDigest covers a freeze recorded without a known +// winner digest (an initiator mirror that lost the hex): the NULL columns +// round-trip as nil, and IsPathContested still reports frozen. +func TestContestedPathUnknownDigest(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + volID := makeVolume(t, s, "/v") + run := makeRun(t, s, volID) + + if err := s.RaiseContested(ctx, ContestedPath{ + VolumeID: volID, Path: "a/b.txt", RaisedRunID: run, + }); err != nil { + t.Fatalf("RaiseContested: %v", err) + } + winner, contested, err := s.IsPathContested(ctx, volID, "a/b.txt") + if err != nil || !contested { + t.Fatalf("IsPathContested = (%x, %v, %v), want frozen", winner, contested, err) + } + if winner != nil { + t.Fatalf("winner = %x, want nil (unknown digest)", winner) + } + list, err := s.ListContestedPaths(ctx) + if err != nil { + t.Fatalf("ListContestedPaths: %v", err) + } + if len(list) != 1 || list[0].Path != "a/b.txt" || list[0].PreservedAtPath != "" { + t.Fatalf("list = %+v, want one row with empty preserved path", list) + } +} diff --git a/store/destination_alarms.go b/store/destination_alarms.go new file mode 100644 index 0000000..3b031a8 --- /dev/null +++ b/store/destination_alarms.go @@ -0,0 +1,157 @@ +package store + +import ( + "context" + "fmt" +) + +// AlarmKindVerifyMismatch is the destination_alarms.kind for a standing +// alarm raised when `squirrel verify` finds a recorded object or pack whose +// provider checksum no longer matches its upload fingerprint, or that has +// gone missing from the remote (#157, F30). It is the only alarm kind +// today; the column is free text so a future standing condition can share +// the table without a migration. +const AlarmKindVerifyMismatch = "verify-mismatch" + +// DestinationAlarm is one row of the destination_alarms latch: an abnormal +// condition on a destination that persists ("latches") until a subsequent +// clean check auto-clears it or an operator acks it. It is derived standing +// state — the permanent forensic record of every raise and clear lives in +// the runs and runs_audit tables — so the live latch can be cleared in +// place without losing history. RaisedRunID references the run that first +// detected the condition; RaisedAtNs is when that first detection landed +// (stable across repeated detections, so the surface can show "in alarm +// since"). +type DestinationAlarm struct { + Destination string + Kind string + Detail string + RaisedRunID int64 + RaisedAtNs int64 +} + +// RaiseDestinationAlarm latches an alarm on destination when none is +// already active and reports whether this call actually created the latch +// (raised == true). The raise is idempotent: an existing alarm keeps its +// original raised-at timestamp and run so a repeated detection does not +// reset "in alarm since" or append a duplicate audit row, and raised comes +// back false. An 'alarm-raise' runs_audit entry is written against +// raisedRunID (in the same transaction as the latch insert) only on the +// creating call, keeping the append-only trail to one row per alarm episode. +// +// raised is derived from the INSERT's RowsAffected inside the same atomic +// ON CONFLICT DO NOTHING statement, so it is race-free: concurrent verify +// runs contend on the destination PRIMARY KEY and exactly one sees +// RowsAffected == 1. Callers must key their "newly raised" UX off this +// return, never off a separate pre-read. +func (s *Store) RaiseDestinationAlarm(ctx context.Context, destination, kind, detail string, raisedRunID int64) (raised bool, err error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, fmt.Errorf("begin raise alarm %q: %w", destination, err) + } + defer func() { _ = tx.Rollback() }() + + atNs := NowNs() + res, err := tx.ExecContext(ctx, ` + INSERT INTO destination_alarms (destination, kind, detail, raised_run_id, raised_at_ns) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(destination) DO NOTHING + `, destination, kind, detail, raisedRunID, atNs) + if err != nil { + return false, fmt.Errorf("raise alarm %q: %w", destination, err) + } + inserted, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("raise alarm %q rows affected: %w", destination, err) + } + if inserted == 0 { + // Already in alarm: keep the original latch untouched. + return false, tx.Commit() + } + note := fmt.Sprintf("destination=%s %s", destination, detail) + if err := appendRunAuditTx(ctx, tx, + RunAuditEntry{RunID: raisedRunID, Transition: TransitionAlarmRaise, Note: note}, atNs); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("commit raise alarm %q: %w", destination, err) + } + return true, nil +} + +// ClearDestinationAlarm clears the active alarm on destination, if any, +// and appends an 'alarm-clear' runs_audit entry against auditRunID. +// operator is "" for an automatic clear by a clean verify pass (the +// clearing run is that pass) and the operator name for an explicit ack +// (the clearing run is the one that raised the alarm). Returns whether an +// alarm was active. Deleting the latch loses no history: the raise and +// this clear both survive in runs_audit, and every verify run that +// touched the destination survives in runs. +func (s *Store) ClearDestinationAlarm(ctx context.Context, destination string, auditRunID int64, operator string) (bool, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, fmt.Errorf("begin clear alarm %q: %w", destination, err) + } + defer func() { _ = tx.Rollback() }() + + res, err := tx.ExecContext(ctx, `DELETE FROM destination_alarms WHERE destination = ?`, destination) + if err != nil { + return false, fmt.Errorf("clear alarm %q: %w", destination, err) + } + deleted, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("clear alarm %q rows affected: %w", destination, err) + } + if deleted == 0 { + return false, tx.Commit() + } + if err := appendRunAuditTx(ctx, tx, RunAuditEntry{ + RunID: auditRunID, Transition: TransitionAlarmClear, Operator: operator, + Note: "destination=" + destination, + }, NowNs()); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("commit clear alarm %q: %w", destination, err) + } + return true, nil +} + +// GetDestinationAlarm returns the active alarm on destination, or +// sql.ErrNoRows (IsNotFound) when the destination is not in alarm. +func (s *Store) GetDestinationAlarm(ctx context.Context, destination string) (DestinationAlarm, error) { + row := s.db.QueryRowContext(ctx, + `SELECT destination, kind, detail, raised_run_id, raised_at_ns + FROM destination_alarms WHERE destination = ?`, destination) + return scanDestinationAlarm(row.Scan) +} + +// ListDestinationAlarms returns every active alarm, destination-sorted so +// the listing is deterministic. An empty slice means nothing is in alarm. +func (s *Store) ListDestinationAlarms(ctx context.Context) ([]DestinationAlarm, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT destination, kind, detail, raised_run_id, raised_at_ns + FROM destination_alarms ORDER BY destination`) + if err != nil { + return nil, fmt.Errorf("list destination alarms: %w", err) + } + defer rows.Close() + var out []DestinationAlarm + for rows.Next() { + a, err := scanDestinationAlarm(rows.Scan) + if err != nil { + return nil, err + } + out = append(out, a) + } + return out, rows.Err() +} + +func scanDestinationAlarm(scan func(...any) error) (DestinationAlarm, error) { + var a DestinationAlarm + err := scan(&a.Destination, &a.Kind, &a.Detail, &a.RaisedRunID, &a.RaisedAtNs) + if err != nil { + return DestinationAlarm{}, err + } + return a, nil +} diff --git a/store/destination_alarms_test.go b/store/destination_alarms_test.go new file mode 100644 index 0000000..2f8d90d --- /dev/null +++ b/store/destination_alarms_test.go @@ -0,0 +1,123 @@ +package store + +import ( + "context" + "testing" +) + +// TestDestinationAlarmLifecycle exercises the raise → idempotent re-raise → +// clear cycle and the runs_audit trail each transition leaves. +func TestDestinationAlarmLifecycle(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + + first, err := s.BeginRemoteVerifyRun(ctx) + if err != nil { + t.Fatalf("BeginRemoteVerifyRun: %v", err) + } + raised, err := s.RaiseDestinationAlarm(ctx, "offsite", AlarmKindVerifyMismatch, "mismatched=1 missing=0", first) + if err != nil { + t.Fatalf("RaiseDestinationAlarm: %v", err) + } + if !raised { + t.Fatal("first raise reported not-raised, want a fresh latch") + } + got, err := s.GetDestinationAlarm(ctx, "offsite") + if err != nil { + t.Fatalf("GetDestinationAlarm: %v", err) + } + if got.Kind != AlarmKindVerifyMismatch || got.RaisedRunID != first || got.RaisedAtNs == 0 { + t.Fatalf("alarm = %+v, want kind/verify-run/timestamp set", got) + } + firstAt := got.RaisedAtNs + + // A second detection keeps the original latch: same run, same + // timestamp, same detail — so "in alarm since" never resets. + second, err := s.BeginRemoteVerifyRun(ctx) + if err != nil { + t.Fatalf("BeginRemoteVerifyRun: %v", err) + } + reraised, err := s.RaiseDestinationAlarm(ctx, "offsite", AlarmKindVerifyMismatch, "mismatched=2 missing=1", second) + if err != nil { + t.Fatalf("re-raise: %v", err) + } + if reraised { + t.Fatal("second raise reported a fresh latch, want not-raised (idempotent)") + } + got, err = s.GetDestinationAlarm(ctx, "offsite") + if err != nil { + t.Fatalf("GetDestinationAlarm after re-raise: %v", err) + } + if got.RaisedRunID != first || got.RaisedAtNs != firstAt || got.Detail != "mismatched=1 missing=0" { + t.Fatalf("re-raise mutated the latch: %+v", got) + } + + all, err := s.ListDestinationAlarms(ctx) + if err != nil { + t.Fatalf("ListDestinationAlarms: %v", err) + } + if len(all) != 1 || all[0].Destination != "offsite" { + t.Fatalf("active alarms = %+v, want exactly offsite", all) + } + + // Exactly one alarm-raise audit row against the first run; none against + // the second (the re-raise was a no-op). + if n := countTransition(t, s, first, TransitionAlarmRaise); n != 1 { + t.Fatalf("first run alarm-raise count = %d, want 1", n) + } + if n := countTransition(t, s, second, TransitionAlarmRaise); n != 0 { + t.Fatalf("second run alarm-raise count = %d, want 0 (idempotent re-raise)", n) + } + + // Clear via ack: recorded against the raising run, tagged operator. + cleared, err := s.ClearDestinationAlarm(ctx, "offsite", first, "alice") + if err != nil { + t.Fatalf("ClearDestinationAlarm: %v", err) + } + if !cleared { + t.Fatal("ClearDestinationAlarm reported no active alarm") + } + if _, err := s.GetDestinationAlarm(ctx, "offsite"); !IsNotFound(err) { + t.Fatalf("alarm still present after clear: %v", err) + } + audits, err := s.ListRunAudit(ctx, first) + if err != nil { + t.Fatalf("ListRunAudit: %v", err) + } + var sawClear bool + for _, a := range audits { + if a.Transition == TransitionAlarmClear { + sawClear = true + if a.Operator.String != "alice" { + t.Fatalf("alarm-clear operator = %q, want alice", a.Operator.String) + } + } + } + if !sawClear { + t.Fatal("no alarm-clear audit row against the raising run") + } + + // Clearing an already-clear destination reports false and writes nothing. + cleared, err = s.ClearDestinationAlarm(ctx, "offsite", first, "alice") + if err != nil { + t.Fatalf("second clear: %v", err) + } + if cleared { + t.Fatal("second clear reported an active alarm") + } +} + +func countTransition(t *testing.T, s *Store, runID int64, transition string) int { + t.Helper() + audits, err := s.ListRunAudit(context.Background(), runID) + if err != nil { + t.Fatalf("ListRunAudit(%d): %v", runID, err) + } + n := 0 + for _, a := range audits { + if a.Transition == transition { + n++ + } + } + return n +} diff --git a/store/destination_run_ids_test.go b/store/destination_run_ids_test.go index 78fed54..b1bde23 100644 --- a/store/destination_run_ids_test.go +++ b/store/destination_run_ids_test.go @@ -9,6 +9,30 @@ import ( "time" ) +// minimalRunsDDL is the runs table at its v15-through-v24 shape. The +// destination_run_ids migration fixtures below carry it because the +// v24→v25 migration rebuilds runs (widening its status CHECK), which reads +// from the existing table — a real database at any of these versions +// always has runs, so the minimal fixtures must too. +const minimalRunsDDL = `CREATE TABLE runs ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL CHECK (kind IN ('index','sync','restore','audit','offload')), + volume_id INTEGER REFERENCES volumes(id), + destination TEXT, + started_at_ns INTEGER NOT NULL, + ended_at_ns INTEGER, + status TEXT NOT NULL CHECK (status IN ('running','success','failed','partial')), + error TEXT, + file_count INTEGER NOT NULL DEFAULT 0, + peer_node_id INTEGER REFERENCES nodes(id), + correlated_run_id INTEGER, + shallow INTEGER CHECK (shallow IS NULL OR shallow IN (0, 1)), + CHECK ( + (kind IN ('index','audit','offload') AND destination IS NULL) OR + (kind IN ('sync','restore') AND destination IS NOT NULL AND destination != '') + ) +)` + // TestMigrateV18ToV19AddsVerifyMethod builds a minimal v18 database with // a pre-existing durability component and confirms the migration adds // verify_method (NULL on the carried-over row) without disturbing the @@ -23,6 +47,7 @@ func TestMigrateV18ToV19AddsVerifyMethod(t *testing.T) { `CREATE TABLE schema_version (version INTEGER NOT NULL PRIMARY KEY)`, `CREATE TABLE volumes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, path TEXT NOT NULL)`, `CREATE TABLE nodes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, endpoint TEXT, public_key_fingerprint TEXT)`, + minimalRunsDDL, // contents exists from v14 on; a real v18 DB carries it, and the // v20→v21 triggers attach to it, so the minimal fixture must too. `CREATE TABLE contents ( @@ -97,6 +122,7 @@ func TestMigrateV22ToV23AddsVerifiedAt(t *testing.T) { `CREATE TABLE schema_version (version INTEGER NOT NULL PRIMARY KEY)`, `CREATE TABLE volumes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, path TEXT NOT NULL)`, `CREATE TABLE nodes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, endpoint TEXT, public_key_fingerprint TEXT)`, + minimalRunsDDL, `CREATE TABLE contents ( id INTEGER PRIMARY KEY, blake3 BLOB NOT NULL UNIQUE CHECK (length(blake3) = 32), @@ -824,6 +850,7 @@ func TestMigrateV21ToV22AddsSourceNodeID(t *testing.T) { `CREATE TABLE schema_version (version INTEGER NOT NULL PRIMARY KEY)`, `CREATE TABLE volumes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, path TEXT NOT NULL)`, `CREATE TABLE nodes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, endpoint TEXT, public_key_fingerprint TEXT)`, + minimalRunsDDL, `CREATE TABLE contents ( id INTEGER PRIMARY KEY, blake3 BLOB NOT NULL UNIQUE CHECK (length(blake3) = 32), diff --git a/store/files.go b/store/files.go index 107a2c6..9b355e9 100644 --- a/store/files.go +++ b/store/files.go @@ -922,6 +922,23 @@ func (s *Store) ListPresentPathsUnder(ctx context.Context, volumeID int64) (map[ return out, rows.Err() } +// CountPresentFilesInVolume returns the number of live (status='present') +// file rows in the volume. The peer-sync summary uses it to report the +// already-correct count: under the Merkle walk only differing folders +// reach /plan, so already-correct is derived as present-total minus the +// paths the sync acted on rather than counted from the (partial) +// disposition list. +func (s *Store) CountPresentFilesInVolume(ctx context.Context, volumeID int64) (int64, error) { + var n int64 + err := s.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM `+fileFromJoin+` + WHERE f.status = 'present' AND fo.volume_id = ?`, volumeID).Scan(&n) + if err != nil { + return 0, fmt.Errorf("count present files in volume %d: %w", volumeID, err) + } + return n, nil +} + // ListPresentFilesInFolder returns every present file row in one // folder, ordered by name ascending so the caller's downstream // processing is deterministic. Used by the Merkle walk's initiator diff --git a/store/migrate_v25_test.go b/store/migrate_v25_test.go new file mode 100644 index 0000000..fdf689e --- /dev/null +++ b/store/migrate_v25_test.go @@ -0,0 +1,118 @@ +package store + +import ( + "context" + "database/sql" + "path/filepath" + "testing" +) + +// TestMigrateV24ToV25PreservesRunsAndWidensStatus builds a minimal v24 +// database carrying runs rows in every pre-v25 terminal state plus an +// in-flight 'running' row, and a child row with a foreign key into runs, +// then migrates to v25. It pins the highest-risk part of the migration — +// the FK-off runs-table rebuild — proving it copies every row verbatim +// (id and status preserved), keeps a referencing FK resolving to the +// rebuilt table (the property the migration's own foreign_key_check +// guards on a full DB), and widens the status CHECK to admit 'refused' +// and 'aborted' without admitting a bogus status. +func TestMigrateV24ToV25PreservesRunsAndWidensStatus(t *testing.T) { + dsn := filepath.Join(t.TempDir(), "test.db") + rawDB, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatalf("raw sql.Open: %v", err) + } + v24DDL := []string{ + `CREATE TABLE schema_version (version INTEGER NOT NULL PRIMARY KEY)`, + `CREATE TABLE volumes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, path TEXT NOT NULL)`, + `CREATE TABLE nodes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, endpoint TEXT, public_key_fingerprint TEXT)`, + minimalRunsDDL, + // A child table with a real FK into runs proves the id-preserving + // rebuild keeps referencing rows resolving after runs is dropped + // and recreated under FK-off. + `CREATE TABLE run_child (id INTEGER PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES runs(id))`, + `INSERT INTO schema_version (version) VALUES (24)`, + `INSERT INTO volumes (id, name, path) VALUES (1, 'v', '/v')`, + `INSERT INTO nodes (id, name) VALUES (1, 'self')`, + `INSERT INTO runs (id, kind, volume_id, destination, started_at_ns, ended_at_ns, status, error, file_count) + VALUES (1, 'index', 1, NULL, 100, 200, 'success', NULL, 5)`, + `INSERT INTO runs (id, kind, volume_id, destination, started_at_ns, ended_at_ns, status, error, file_count) + VALUES (2, 'sync', 1, 'bucket', 110, 210, 'failed', 'boom', 0)`, + `INSERT INTO runs (id, kind, volume_id, destination, started_at_ns, ended_at_ns, status, error, file_count) + VALUES (3, 'sync', 1, 'bucket', 120, 220, 'partial', NULL, 3)`, + `INSERT INTO runs (id, kind, volume_id, destination, started_at_ns, status, file_count) + VALUES (4, 'index', 1, NULL, 130, 'running', 0)`, + `INSERT INTO run_child (id, run_id) VALUES (1, 2)`, + } + for _, q := range v24DDL { + if _, err := rawDB.Exec(q); err != nil { + t.Fatalf("v24 DDL %q: %v", q, err) + } + } + rawDB.Close() + + s, err := Open(dsn) + if err != nil { + t.Fatalf("Open (migrates v24→v25): %v", err) + } + defer s.Close() + ctx := context.Background() + if v, _ := s.CurrentSchemaVersion(ctx); v != SchemaVersion { + t.Fatalf("schema_version = %d, want %d", v, SchemaVersion) + } + + // Every runs row carried over with its id and status intact. + wantStatus := map[int64]string{ + 1: RunStatusSuccess, 2: RunStatusFailed, 3: RunStatusPartial, 4: RunStatusRunning, + } + runs, err := s.ListRuns(ctx, ListRunsOpts{}) + if err != nil { + t.Fatalf("ListRuns: %v", err) + } + if len(runs) != len(wantStatus) { + t.Fatalf("runs after migration = %d, want %d", len(runs), len(wantStatus)) + } + for _, r := range runs { + if want := wantStatus[r.ID]; r.Status != want { + t.Fatalf("run %d status = %q, want %q", r.ID, r.Status, want) + } + } + + // The child FK still resolves to the rebuilt runs table, and no + // dangling reference slipped through the FK-off rebuild. + var refs int + if err := s.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM run_child c JOIN runs r ON r.id = c.run_id`).Scan(&refs); err != nil { + t.Fatalf("join child→runs: %v", err) + } + if refs != 1 { + t.Fatalf("child rows resolving to runs = %d, want 1", refs) + } + fkRows, err := s.db.QueryContext(ctx, `PRAGMA foreign_key_check`) + if err != nil { + t.Fatalf("foreign_key_check: %v", err) + } + violations := 0 + for fkRows.Next() { + violations++ + } + fkRows.Close() + if violations != 0 { + t.Fatalf("foreign_key_check reported %d violations after migration", violations) + } + + // The widened CHECK admits the two new terminal states... + for _, status := range []string{RunStatusRefused, RunStatusAborted} { + if _, err := s.db.ExecContext(ctx, + `INSERT INTO runs (kind, volume_id, destination, started_at_ns, status, file_count) + VALUES ('sync', 1, 'bucket', 300, ?, 0)`, status); err != nil { + t.Fatalf("insert %q run rejected by CHECK after migration: %v", status, err) + } + } + // ...but the CHECK still rejects an unknown status. + if _, err := s.db.ExecContext(ctx, + `INSERT INTO runs (kind, volume_id, destination, started_at_ns, status, file_count) + VALUES ('sync', 1, 'bucket', 320, 'bogus', 0)`); err == nil { + t.Fatalf("CHECK accepted a bogus status, want rejection") + } +} diff --git a/store/migrations.go b/store/migrations.go index 9550b85..1526724 100644 --- a/store/migrations.go +++ b/store/migrations.go @@ -10,7 +10,7 @@ import ( ) // SchemaVersion is the schema version this binary writes and reads. -const SchemaVersion = 24 +const SchemaVersion = 26 // freshSchemaBaseline is the version applied to a brand-new database. The // chain in `migrations` continues from here. v1 is no longer reachable from @@ -66,6 +66,8 @@ func buildMigrations(mctx migrationCtx) []migration { {version: 22, up: migrateV21ToV22}, {version: 23, up: migrateV22ToV23}, {version: 24, up: migrateV23ToV24}, + {version: 25, up: migrateV24ToV25}, + {version: 26, up: migrateV25ToV26}, } } @@ -2065,3 +2067,180 @@ func migrateV23ToV24(ctx context.Context, db *sql.DB) error { } return tx.Commit() } + +// --- v24 → v25 --- + +// migrateV24ToV25 lays the audit-trail foundation for issue #157 — every +// failure and refusal becomes a run row, every abnormal condition a +// standing state: +// +// - The runs.status CHECK widens to admit two terminal states. +// 'refused' records a preflight safety gate declining before the +// transfer began (a missing/mismatched volume marker, a kopia connect +// that found no repository without --init, a layout guard tripping) — +// distinct from 'failed', which is a mid-flight failure. 'aborted' +// records a 'running' row reaped at agent startup because the process +// that owned it was killed mid-run (F14). Both are terminal; neither +// is a success. Reuses the FK-off rebuild recipe (v6→v7) because runs +// is referenced by files, hook_runs, packs, remote_objects, +// remote_packs, and runs_audit. +// - destination_alarms is the per-destination standing-alarm latch +// (F30): a verify mismatch raises a row that stays until a clean +// verify auto-clears it or an operator acks it. It is derived standing +// state — like peer_sync_state, the live row is maintained in place +// while the permanent forensic record of every raise and clear lives +// in runs/runs_audit — so clearing the latch loses no history. STRICT +// per the new-table convention. +func migrateV24ToV25(ctx context.Context, db *sql.DB) error { + conn, restore, err := disableForeignKeys(ctx, db) + if err != nil { + return err + } + defer restore() + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + if err := rebuildRunsTableV25(ctx, tx); err != nil { + return err + } + if err := createDestinationAlarmsV25(ctx, tx); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO schema_version (version) VALUES (25)`); err != nil { + return fmt.Errorf("record schema v25: %w", err) + } + if err := verifyForeignKeysClean(ctx, tx, "v24→v25"); err != nil { + return err + } + return tx.Commit() +} + +func rebuildRunsTableV25(ctx context.Context, tx *sql.Tx) error { + stmts := []string{ + `CREATE TABLE runs_v25 ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL CHECK (kind IN ('index','sync','restore','audit','offload')), + volume_id INTEGER REFERENCES volumes(id), + destination TEXT, + started_at_ns INTEGER NOT NULL, + ended_at_ns INTEGER, + status TEXT NOT NULL CHECK (status IN ('running','success','failed','partial','refused','aborted')), + error TEXT, + file_count INTEGER NOT NULL DEFAULT 0, + peer_node_id INTEGER REFERENCES nodes(id), + correlated_run_id INTEGER, + shallow INTEGER CHECK (shallow IS NULL OR shallow IN (0, 1)), + CHECK ( + (kind IN ('index','audit','offload') AND destination IS NULL) OR + (kind IN ('sync','restore') AND destination IS NOT NULL AND destination != '') + ) + )`, + `INSERT INTO runs_v25 ( + id, kind, volume_id, destination, started_at_ns, ended_at_ns, + status, error, file_count, peer_node_id, correlated_run_id, shallow + ) + SELECT id, kind, volume_id, destination, started_at_ns, ended_at_ns, + status, error, file_count, peer_node_id, correlated_run_id, shallow + FROM runs`, + `DROP TABLE runs`, + `ALTER TABLE runs_v25 RENAME TO runs`, + `CREATE INDEX idx_runs_volume_started ON runs(volume_id, started_at_ns)`, + `CREATE INDEX idx_runs_destination ON runs(destination) WHERE destination IS NOT NULL`, + } + for _, q := range stmts { + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rebuild runs: %w", err) + } + } + return nil +} + +// createDestinationAlarmsV25 creates the per-destination standing-alarm +// latch. destination is the PRIMARY KEY: at most one active alarm per +// destination, which the raise path keeps idempotent so "in alarm since" +// stays stable across repeated mismatches. raised_run_id is a real FK to +// the verify run that raised it. No secondary index: the table holds one +// row per destination in alarm (usually zero), so every read is a PK +// lookup or a full scan of a handful of rows — an index would never be +// used and would violate the low-cardinality-index guidance. +func createDestinationAlarmsV25(ctx context.Context, tx *sql.Tx) error { + const ddl = `CREATE TABLE destination_alarms ( + destination TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + detail TEXT NOT NULL, + raised_run_id INTEGER NOT NULL REFERENCES runs(id), + raised_at_ns INTEGER NOT NULL + ) STRICT` + if _, err := tx.ExecContext(ctx, ddl); err != nil { + return fmt.Errorf("create destination_alarms: %w", err) + } + return nil +} + +// --- v25 → v26 --- + +// migrateV25ToV26 adds the contested_paths standing-state latch (#158, +// F27): the per-(volume, path) freeze that stops divergent edits from +// ping-ponging forever. A conflict raises a row; while it stands, the +// receiver refuses to re-supersede the path (the `contested` disposition) +// and the initiators surface a badge, so both versions stay preserved +// exactly once instead of a fresh `.squirrel-conflicts/run-N/` copy every +// cadence tick. Like destination_alarms (v25) and peer_sync_state, it is +// derived standing state: the permanent forensic record of every conflict +// lives in runs/runs_audit and the preserved bytes on disk, so clearing +// the latch (an explicit `squirrel conflicts resolve`) loses no history. +// +// The table is written on every node. On the receiver a row records the +// authoritative freeze; on an initiator a row mirrors what the receiver +// reported so its own `squirrel conflicts` and TUI badge answer "am I +// safe?" from its local index (no fleet view needed). STRICT per the +// new-table convention; additive, so no rebuild of any existing table. +func migrateV25ToV26(ctx context.Context, db *sql.DB) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + if err := createContestedPathsV26(ctx, tx); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO schema_version (version) VALUES (26)`); err != nil { + return fmt.Errorf("record schema v26: %w", err) + } + return tx.Commit() +} + +// createContestedPathsV26 creates the contested-freeze latch. The PK is +// (volume_id, path): at most one active freeze per path, which the raise +// path keeps idempotent so "contested since" stays stable across repeated +// conflicts. live_blake3 is the frozen winner and preserved_blake3 the +// loser; the classifier compares an incoming digest against live_blake3 to +// tell the winner re-asserting (allowed) from a divergent re-assertion +// (refused). peer_node_id is nullable — the initiator that caused the +// freeze on the receiver, or the destination peer on an initiator — and +// raised_run_id is a real FK to the run that raised it. No secondary +// index: reads are PK lookups or a full scan of the handful of rows in +// alarm, so an index would go unused and violate the +// low-cardinality-index guidance. +func createContestedPathsV26(ctx context.Context, tx *sql.Tx) error { + const ddl = `CREATE TABLE contested_paths ( + volume_id INTEGER NOT NULL REFERENCES volumes(id), + path TEXT NOT NULL, + live_blake3 BLOB CHECK (live_blake3 IS NULL OR length(live_blake3) = 32), + preserved_blake3 BLOB CHECK (preserved_blake3 IS NULL OR length(preserved_blake3) = 32), + preserved_at_path TEXT, + peer_node_id INTEGER REFERENCES nodes(id), + raised_run_id INTEGER NOT NULL REFERENCES runs(id), + raised_at_ns INTEGER NOT NULL, + PRIMARY KEY (volume_id, path) + ) STRICT` + if _, err := tx.ExecContext(ctx, ddl); err != nil { + return fmt.Errorf("create contested_paths: %w", err) + } + return nil +} diff --git a/store/runs.go b/store/runs.go index 49ac6a2..de9f38c 100644 --- a/store/runs.go +++ b/store/runs.go @@ -29,29 +29,38 @@ const ( // Run statuses. A run begins in 'running' and is moved to a terminal state by // FinishRun. 'partial' means the walk completed but some files errored. +// 'refused' means a preflight safety gate declined before the operation +// began — a missing/mismatched volume marker, a kopia connect that found no +// repository without --init, a layout guard — so the refusal reaches the +// audit trail instead of living only in a caller's error (#157, F26). It is +// distinct from 'failed', which is a mid-flight failure. 'aborted' means a +// 'running' row was reaped because the process that owned it died mid-run +// (the agent reaps its own orphans at startup — #157, F14); it is terminal +// but records that the run never completed, not that it failed. const ( RunStatusRunning = "running" RunStatusSuccess = "success" RunStatusFailed = "failed" RunStatusPartial = "partial" + RunStatusRefused = "refused" + RunStatusAborted = "aborted" ) // ErrAlreadyFinished is returned by FinishRun when the target run is -// already in a terminal status (success/partial/failed). The first -// terminal write wins: its status, error, and ended_at_ns are the audit -// record, and a second FinishRun would silently rewrite them. Callers -// that may legitimately race a double-finish (e.g. agent/sync.go's +// already in a terminal status (success/partial/failed/refused/aborted). +// The first terminal write wins: its status, error, and ended_at_ns are +// the audit record, and a second FinishRun would silently rewrite them. +// Callers that may legitimately race a double-finish (e.g. agent/sync.go's // handleClose firing after closeSession already finalised) detect this // with errors.Is and fall back to a log-only path rather than treating // it as a hard error. var ErrAlreadyFinished = errors.New("run is already in a terminal status") -// isTerminalStatus reports whether status is one of the three terminal -// run states. A row in any of these must not be re-finalised by -// FinishRun. +// isTerminalStatus reports whether status is one of the terminal run +// states. A row in any of these must not be re-finalised by FinishRun. func isTerminalStatus(status string) bool { switch status { - case RunStatusSuccess, RunStatusPartial, RunStatusFailed: + case RunStatusSuccess, RunStatusPartial, RunStatusFailed, RunStatusRefused, RunStatusAborted: return true } return false @@ -412,11 +421,31 @@ func (s *Store) GetRun(ctx context.Context, id int64) (Run, error) { // from the sync-package directory naming convention. func (s *Store) CountFilesFirstSeenByRunWithPathPrefix(ctx context.Context, runIDs []int64, pathPrefix string) (map[int64]int, error) { out := make(map[int64]int, len(runIDs)) + // Batch the id set: each query binds len(chunk)+2 parameters, so an + // unbatched call over a large peer-sync history would overflow SQLite's + // bound-parameter cap (SQLITE_MAX_VARIABLE_NUMBER) and fail with "too + // many SQL variables". 400 keeps every query well under any cap. + const idsPerQuery = 400 + for start := 0; start < len(runIDs); start += idsPerQuery { + end := start + idsPerQuery + if end > len(runIDs) { + end = len(runIDs) + } + if err := s.countFilesFirstSeenChunk(ctx, runIDs[start:end], pathPrefix, out); err != nil { + return nil, err + } + } + return out, nil +} + +// countFilesFirstSeenChunk runs the grouped count for one id batch and folds +// the results into out. Non-zero counts only; absent keys mean zero. +func (s *Store) countFilesFirstSeenChunk(ctx context.Context, runIDs []int64, pathPrefix string, out map[int64]int) error { if len(runIDs) == 0 { - return out, nil + return nil } placeholders := strings.Repeat("?,", len(runIDs)-1) + "?" - args := make([]any, 0, len(runIDs)+1) + args := make([]any, 0, len(runIDs)+2) for _, id := range runIDs { args = append(args, id) } @@ -428,18 +457,18 @@ func (s *Store) CountFilesFirstSeenByRunWithPathPrefix(ctx context.Context, runI AND (fo.path = ? OR fo.path LIKE ? ESCAPE '\') GROUP BY f.first_seen_run_id`, args...) if err != nil { - return nil, fmt.Errorf("count files by run: %w", err) + return fmt.Errorf("count files by run: %w", err) } defer rows.Close() for rows.Next() { var id int64 var n int if err := rows.Scan(&id, &n); err != nil { - return nil, fmt.Errorf("scan count row: %w", err) + return fmt.Errorf("scan count row: %w", err) } out[id] = n } - return out, rows.Err() + return rows.Err() } // escapeLikePrefix escapes %, _ and \ in s so it can be embedded into a @@ -736,18 +765,22 @@ func (s *Store) LatestSuccessfulIndexRun(ctx context.Context, volumeID int64) (R return scanRun(row.Scan) } -// LatestFinishedRun returns the most recent terminal-status (success, -// partial, or failed) run of the given kind for (volumeID, destination). -// destination must be "" for index/audit kinds (the schema CHECK ensures -// those carry no destination) and the rclone destination or peer node -// name for sync/restore kinds. Returns sql.ErrNoRows when no matching -// run exists. +// LatestFinishedRun returns the most recent terminal-status run of the +// given kind for (volumeID, destination). destination must be "" for +// index/audit kinds (the schema CHECK ensures those carry no destination) +// and the rclone destination or peer node name for sync/restore kinds. +// Returns sql.ErrNoRows when no matching run exists. // // The scheduler (#39) computes `now - last_finished` from this row to -// decide whether a cadence-driven run is due. Failed terminal states -// count: per the issue's failure-policy (no special retry, the next -// tick re-evaluates), a failed run consumes the cadence window like -// any other. +// decide whether a cadence-driven run is due. success, partial, failed, +// and refused all count: per the issue's failure-policy (no special +// retry, the next tick re-evaluates), each consumes the cadence window +// like any other — a 'refused' run in particular backs the scheduler off +// a dead destination (a missing marker, an un-bootstrapped repository) +// instead of re-minting an identical refusal every tick. 'aborted' is +// deliberately excluded: a run reaped mid-flight never completed, so the +// cadence should re-attempt rather than treat the crash as a finished +// window. func (s *Store) LatestFinishedRun(ctx context.Context, kind string, volumeID int64, destination string) (Run, error) { var row *sql.Row if destination == "" { @@ -755,7 +788,7 @@ func (s *Store) LatestFinishedRun(ctx context.Context, kind string, volumeID int `SELECT `+runColumns+` FROM runs WHERE kind = ? AND volume_id = ? AND destination IS NULL - AND status IN ('success','partial','failed') + AND status IN ('success','partial','failed','refused') ORDER BY id DESC LIMIT 1`, kind, volumeID) } else { @@ -763,7 +796,7 @@ func (s *Store) LatestFinishedRun(ctx context.Context, kind string, volumeID int `SELECT `+runColumns+` FROM runs WHERE kind = ? AND volume_id = ? AND destination = ? - AND status IN ('success','partial','failed') + AND status IN ('success','partial','failed','refused') ORDER BY id DESC LIMIT 1`, kind, volumeID, destination) } @@ -902,3 +935,69 @@ func (s *Store) HasRunningRun(ctx context.Context, kind string, volumeID int64, } return found, nil } + +// AbortRunningRuns transitions every run currently in 'running' to +// 'aborted', stamping ended_at_ns, recording reason as the row's error, +// and appending an 'abort' runs_audit transition per reaped row so the +// append-only trail records who moved the row and why. It returns the ids +// reaped, oldest first, so the caller can log them. +// +// Called once at agent startup to clear phantom 'running' rows left by a +// prior agent that was killed mid-run (#157, F14): those rows otherwise +// block BeginSyncRunIfClear/BeginIndexRunIfClear forever and render as a +// live, elapsed-ticking banner in the TUI hours later. A freshly started +// agent has kicked nothing yet, so at startup every 'running' row +// necessarily predates it — the reap assumes the agent is the single +// writer of runs on this node (the reference-setup model), and a +// concurrent CLI run started against the same DB is out of scope. +func (s *Store) AbortRunningRuns(ctx context.Context, reason string) ([]int64, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("begin abort running runs: %w", err) + } + defer func() { _ = tx.Rollback() }() + + ids, err := runningRunIDsTx(ctx, tx) + if err != nil { + return nil, err + } + if len(ids) == 0 { + return nil, nil + } + atNs := NowNs() + for _, id := range ids { + if _, err := tx.ExecContext(ctx, ` + UPDATE runs SET ended_at_ns = ?, status = ?, error = ? + WHERE id = ? AND status = 'running' + `, atNs, RunStatusAborted, reason, id); err != nil { + return nil, fmt.Errorf("abort run %d: %w", id, err) + } + if err := appendRunAuditTx(ctx, tx, + RunAuditEntry{RunID: id, Transition: TransitionAbort, Note: reason}, atNs); err != nil { + return nil, err + } + } + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("commit abort running runs: %w", err) + } + return ids, nil +} + +// runningRunIDsTx returns the ids of every 'running' run inside tx, +// ascending, so the reap order and the returned slice are deterministic. +func runningRunIDsTx(ctx context.Context, tx *sql.Tx) ([]int64, error) { + rows, err := tx.QueryContext(ctx, `SELECT id FROM runs WHERE status = 'running' ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("list running runs: %w", err) + } + defer rows.Close() + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("scan running run id: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() +} diff --git a/store/runs_audit.go b/store/runs_audit.go index 35b86d8..7eaf406 100644 --- a/store/runs_audit.go +++ b/store/runs_audit.go @@ -33,6 +33,30 @@ const ( // CHECK keeps destination NULL on audit rows, so this entry is where // the audit trail names the verified destination. TransitionVerifyDestination = "verify-destination" + // TransitionAbort records a 'running' row reaped to 'aborted' by + // AbortRunningRuns at agent startup (#157, F14). The note carries the + // reap reason so a forensic reader can tell a crash-reap apart from a + // real failure. + TransitionAbort = "abort" + // TransitionAlarmRaise records a destination_alarms latch being raised + // against the verify run that detected the mismatch (#157, F30). The + // note carries the destination and a bounded detail summary. + TransitionAlarmRaise = "alarm-raise" + // TransitionAlarmClear records a destination_alarms latch being + // cleared: against the clean verify run that auto-cleared it (operator + // NULL) or against the run that raised it when an operator acks + // (operator set). The note carries the destination. + TransitionAlarmClear = "alarm-clear" + // TransitionContestedRaise records a contested_paths latch being + // raised against the sync run that hit the conflict (#158, F27): the + // freeze that stops the divergent-edit ping-pong. The note carries the + // volume-relative path. Written once per distinct freeze episode. + TransitionContestedRaise = "contested-raise" + // TransitionContestedClear records a contested_paths latch being + // cleared by an explicit `squirrel conflicts resolve` (operator set), + // against the run that raised it. The note carries the path. This is + // the deliberate human act that unfreezes the path. + TransitionContestedClear = "contested-clear" // TransitionPullDurability records an agent-scheduled durability pull // against its kind='audit' run (see BeginDurabilityPullRun). The note // carries the pulled volume, the peer, and the fetched/applied/dropped/ diff --git a/store/schema.sql b/store/schema.sql index 5bfab8b..0713a6a 100644 --- a/store/schema.sql +++ b/store/schema.sql @@ -1,6 +1,6 @@ -- Generated by `go test ./store -update-schema` — DO NOT EDIT. -- --- Flattened snapshot of the squirrel index schema at version 24, for humans +-- Flattened snapshot of the squirrel index schema at version 26, for humans -- and agents who want the current shape without replaying the migration -- chain in migrations.go. It is NOT used to create or migrate databases — -- a fresh DB is built by applyV5 plus the migration registry. The golden @@ -27,6 +27,26 @@ CREATE TRIGGER contents_no_update BEFORE UPDATE ON contents SELECT RAISE(ABORT, 'contents is append-only; supersede the files row and insert new content instead of updating'); END; +CREATE TABLE contested_paths ( + volume_id INTEGER NOT NULL REFERENCES volumes(id), + path TEXT NOT NULL, + live_blake3 BLOB CHECK (live_blake3 IS NULL OR length(live_blake3) = 32), + preserved_blake3 BLOB CHECK (preserved_blake3 IS NULL OR length(preserved_blake3) = 32), + preserved_at_path TEXT, + peer_node_id INTEGER REFERENCES nodes(id), + raised_run_id INTEGER NOT NULL REFERENCES runs(id), + raised_at_ns INTEGER NOT NULL, + PRIMARY KEY (volume_id, path) + ) STRICT; + +CREATE TABLE destination_alarms ( + destination TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + detail TEXT NOT NULL, + raised_run_id INTEGER NOT NULL REFERENCES runs(id), + raised_at_ns INTEGER NOT NULL + ) STRICT; + CREATE TABLE destination_push_freshness ( volume_id INTEGER NOT NULL REFERENCES volumes(id), destination TEXT NOT NULL, @@ -181,7 +201,7 @@ CREATE TABLE "runs" ( destination TEXT, started_at_ns INTEGER NOT NULL, ended_at_ns INTEGER, - status TEXT NOT NULL CHECK (status IN ('running','success','failed','partial')), + status TEXT NOT NULL CHECK (status IN ('running','success','failed','partial','refused','aborted')), error TEXT, file_count INTEGER NOT NULL DEFAULT 0, peer_node_id INTEGER REFERENCES nodes(id), diff --git a/store/store_test.go b/store/store_test.go index 6eefdd0..477f997 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -2523,6 +2523,42 @@ func TestCountFilesFirstSeenByRunWithPathPrefix(t *testing.T) { } } +// TestCountFilesFirstSeenBatchesLargeIDSet guards the Copilot #179 fix: an +// id set far larger than SQLite's bound-parameter cap must be batched, not +// fail with "too many SQL variables". A real matching row buried among the +// synthetic ids confirms the per-batch results still merge correctly. +func TestCountFilesFirstSeenBatchesLargeIDSet(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer s.Close() + ctx := context.Background() + volID := makeVolume(t, s, "/v") + run := makeRun(t, s, volID) + if err := s.Upsert(ctx, FileRow{ + VolumeID: volID, Path: ".squirrel-conflicts/run-1/a", Blake3: digest(0x42), + SizeBytes: 1, MtimeNs: 1, Status: StatusPresent, + FirstSeenRunID: run, LastSeenRunID: run, IndexedAtNs: 1, + }, nil); err != nil { + t.Fatalf("upsert: %v", err) + } + // Far more ids than any SQLITE_MAX_VARIABLE_NUMBER a single query could + // bind; the real run id is appended after 5000 synthetic non-matches. + ids := make([]int64, 0, 5001) + for i := int64(1); i <= 5000; i++ { + ids = append(ids, i*1000) + } + ids = append(ids, run) + counts, err := s.CountFilesFirstSeenByRunWithPathPrefix(ctx, ids, ".squirrel-conflicts") + if err != nil { + t.Fatalf("large id set errored (batching regressed?): %v", err) + } + if counts[run] != 1 { + t.Fatalf("counts[run] = %d, want 1", counts[run]) + } +} + // TestListPresentByOrigin pins the two filter modes: valid nodeID // returns rows attributed to that peer, NULL nodeID returns rows // without provenance (local writes). Superseded and missing rows diff --git a/sync/content_addressed.go b/sync/content_addressed.go index b5c6637..7131140 100644 --- a/sync/content_addressed.go +++ b/sync/content_addressed.go @@ -283,7 +283,7 @@ func (h *contentAddressedHandler) watermark(ctx context.Context, volID int64) (i if freshStartOnEmptyRoot(ctx, h.rcl, h.dest) { return 0, nil } - return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no manifest segment at %s — its history does not look content-addressed; point the layout at a fresh destination or root, or (after wiping the remote root) run `squirrel destination reset %s`, instead of switching an existing one: %w", h.dest.Name, last.ID, segURI, h.dest.Name, err) + return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no manifest segment at %s — its history does not look content-addressed; point the layout at a fresh destination or root, or (after wiping the remote root) run `squirrel destination reset %s`, instead of switching an existing one: %w: %w", h.dest.Name, last.ID, segURI, h.dest.Name, err, ErrRefused) } return last.ID, nil } diff --git a/sync/content_addressed_test.go b/sync/content_addressed_test.go index d6013df..0ddb920 100644 --- a/sync/content_addressed_test.go +++ b/sync/content_addressed_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -744,8 +745,11 @@ func TestContentAddressedWatermarkGuard(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "does not look content-addressed") { t.Fatalf("expected layout-flip refusal, got %v", err) } - if rep.Status != store.RunStatusFailed { - t.Fatalf("Status = %q, want failed", rep.Status) + if !errors.Is(err, ErrRefused) { + t.Fatalf("layout guard should be a refusal (ErrRefused), got %v", err) + } + if rep.Status != store.RunStatusRefused { + t.Fatalf("Status = %q, want refused", rep.Status) } } diff --git a/sync/handler.go b/sync/handler.go index 5ee15f3..833a585 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -175,15 +175,22 @@ func (h *peerHandler) Push(ctx context.Context, opts Options) (Report, error) { func (h *peerHandler) sealed() {} -// finishHandlerRun writes a handler-driven run's terminal state from -// rep.Status, mirroring the rclone scaffold's finishRun contract: a -// FinishRun failure lands on rep.FinishErr so the caller surfaces it -// next to the push outcome. The kopia and content-addressed handlers -// share it; their file counts ride on rep.Verification.Files. +// finishHandlerRun writes a handler-driven run's terminal state, +// mirroring the rclone scaffold's finishRun contract: a FinishRun failure +// lands on rep.FinishErr so the caller surfaces it next to the push +// outcome. The kopia, content-addressed, and packed handlers share it; +// their file counts ride on rep.Verification.Files. +// +// A preflight refusal (runErr wrapping ErrRefused — a kopia connect +// without --init, a layout guard) overrides the derived status to +// 'refused' via terminalStatus, and rep.Status is updated in place so the +// CLI summary and TUI show the refusal as its own condition rather than a +// generic failure (#157, F26). func finishHandlerRun(ctx context.Context, s *store.Store, rep *Report, runErr error) { if rep.RunID == 0 { return } + rep.Status = terminalStatus(rep.Status, runErr) errMsg := "" if runErr != nil { errMsg = runErr.Error() diff --git a/sync/kopia.go b/sync/kopia.go index 26fe2e4..15d4d64 100644 --- a/sync/kopia.go +++ b/sync/kopia.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strconv" "strings" @@ -15,6 +16,16 @@ import ( "github.com/mbertschler/squirrel/store" ) +// ansiEscapeRE matches ANSI CSI escape sequences (colour among them). +// kopia paints its progress and error output for a terminal; folding that +// raw into an error would leak escape sequences into squirrel's structured +// agent log and the runs.error column, so the stderr tail is stripped +// first (friction F11c). +var ansiEscapeRE = regexp.MustCompile("\x1b\\[[0-9;?]*[ -/]*[@-~]") + +// stripANSI removes ANSI escape sequences from s. +func stripANSI(s string) string { return ansiEscapeRE.ReplaceAllString(s, "") } + // Kopia is a configured kopia wrapper. Like rclone, kopia is treated as // an opaque child process: squirrel owns the argv, points every // invocation at a destination-scoped config file under ConfigDir @@ -71,7 +82,7 @@ func (k *Kopia) run(ctx context.Context, cfgFile, password string, args ...strin cmd.Stderr = &stderr if err := cmd.Run(); err != nil { verb := strings.Join(args[:min(2, len(args))], " ") - if msg := strings.TrimSpace(stderr.String()); msg != "" { + if msg := stripANSI(strings.TrimSpace(stderr.String())); msg != "" { return stdout.Bytes(), fmt.Errorf("kopia %s: %w: %s", verb, err, msg) } return stdout.Bytes(), fmt.Errorf("kopia %s: %w", verb, err) @@ -100,7 +111,7 @@ func (k *Kopia) ensureRepository(ctx context.Context, cfgFile, password, repoPat return nil } if !init { - return fmt.Errorf("kopia repository at %s: connect failed (%w) — re-run with --init to create a new repository (refusing to auto-create in case the path is wrong or the destination is temporarily unreachable)", repoPath, connectErr) + return fmt.Errorf("kopia repository at %s: connect failed (%w) — re-run with --init to create a new repository (refusing to auto-create in case the path is wrong or the destination is temporarily unreachable): %w", repoPath, connectErr, ErrRefused) } if _, createErr := k.run(ctx, cfgFile, password, "repository", "create", "filesystem", "--path", repoPath, "--no-persist-credentials"); createErr != nil { return fmt.Errorf("kopia repository at %s: connect failed (%w); create failed: %w", repoPath, connectErr, createErr) diff --git a/sync/kopia_test.go b/sync/kopia_test.go index e8e838d..46b9070 100644 --- a/sync/kopia_test.go +++ b/sync/kopia_test.go @@ -248,8 +248,11 @@ func TestKopiaConnectFailWithoutInitRefuses(t *testing.T) { if !strings.Contains(err.Error(), "--init") { t.Fatalf("error should point at --init, got %v", err) } - if rep.Status != store.RunStatusFailed { - t.Fatalf("Status = %q, want failed", rep.Status) + if !errors.Is(err, ErrRefused) { + t.Fatalf("connect-without-init should be a refusal (ErrRefused), got %v", err) + } + if rep.Status != store.RunStatusRefused { + t.Fatalf("Status = %q, want refused", rep.Status) } argv, _ := readCallLog(t, logPath) for _, line := range argv { @@ -608,3 +611,20 @@ func TestKopiaAdvanceScopedToCapturedPresentSet(t *testing.T) { t.Fatalf("verify method = %q, want %q", vector[0].VerifyMethod, store.VerifyMethodKopia) } } + +// TestStripANSI covers F11c: kopia paints its output for a terminal, and +// those escape sequences must not leak into the error folded into +// squirrel's structured log / runs.error column. +func TestStripANSI(t *testing.T) { + cases := map[string]string{ + "\x1b[31merror:\x1b[0m disk full": "error: disk full", + "plain message": "plain message", + "\x1b[1;33mwarn\x1b[0m \x1b[2Kcleared": "warn cleared", + "": "", + } + for in, want := range cases { + if got := stripANSI(in); got != want { + t.Errorf("stripANSI(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/sync/node.go b/sync/node.go index ba92d79..1fbbc90 100644 --- a/sync/node.go +++ b/sync/node.go @@ -114,6 +114,11 @@ type nodeSyncDriver struct { // receiverRunID is filled in after the begin handshake; rclone + // verify + close reference it. receiverRunID int64 + // peerNodeID is the local nodes-row id of the destination peer, + // resolved at /begin. Recorded on any contested_paths latch this run + // raises locally so the initiator's `squirrel conflicts` can name the + // peer the freeze lives on. + peerNodeID int64 // protocolVersion is the plan-exchange version negotiated at // /begin. Defaults to ProtocolVersionFlat so a missing field in // the receiver's response (older agent) keeps today's behaviour. @@ -215,8 +220,19 @@ func (d *nodeSyncDriver) run() error { // Conflicts flow through the transfer path: the receiver has // already pre-staged each loser under .squirrel-conflicts/ and // the original path is empty, so rclone treats the entry like a - // fresh transfer. + // fresh transfer. Contested paths are frozen: the receiver refused + // them, no bytes move, and the initiator only surfaces the freeze. d.report.NodeConflicts = plan.Conflicts + d.report.NodeContested = plan.Contested + d.recordAlreadyCorrect(plan) + // Mirror the freeze into this node's own contested_paths latch as + // soon as /plan has classified it, deferred so it still runs when a + // later phase (transfer, verify, close) fails. The receiver already + // pre-staged the loser and raised its own latch during /plan, so the + // conflict is real regardless of whether our bytes land — recording + // only after a successful close would hide the badge on the losing + // edge exactly when a sync failed mid-flight (#158, F27). + defer d.recordContestedLocally() if !d.opts.DryRun { advance, err := captureDurabilityAdvance(d.ctx, d.store, d.volID) if err != nil { @@ -236,6 +252,87 @@ func (d *nodeSyncDriver) run() error { return nil } +// recordContestedLocally mirrors the receiver-reported conflicts and +// contested refusals into this node's own contested_paths latch, so the +// losing/edge machine answers "am I safe?" from its local index — a badge +// in its TUI and a row in its `squirrel conflicts`, not just a line on the +// hub (#158, F27). Both a conflict (this node's bytes won live, the prior +// version was preserved) and a contested refusal (a frozen path declined +// this node's divergent bytes) leave the household master contested until +// a human resolves it, so both raise a marker. Raises are idempotent and +// best-effort: a failure here is logged into the report's warnings and +// never flips the sync's status — the authoritative freeze already lives +// on the receiver. +func (d *nodeSyncDriver) recordContestedLocally() { + if d.opts.DryRun { + return + } + for _, c := range d.report.NodeConflicts { + d.raiseContestedMarker(c.Path, c.InitiatorBlake3Hex, c.ReceiverBlake3Hex, c.PreservedAtPath) + } + for _, c := range d.report.NodeContested { + d.raiseContestedMarker(c.Path, c.LiveBlake3Hex, c.PreservedBlake3Hex, c.PreservedAtPath) + } +} + +// raiseContestedMarker records one local freeze. liveHex is the winning +// version (this node's own bytes for a conflict, the frozen winner for a +// contested refusal) and preservedHex the loser; a bad hex digest is +// dropped to nil so a marker is still recorded rather than lost. Errors +// are surfaced as a report warning, never fatal. +func (d *nodeSyncDriver) raiseContestedMarker(path, liveHex, preservedHex, preservedAt string) { + err := d.store.RaiseContested(d.ctx, store.ContestedPath{ + VolumeID: d.volID, + Path: path, + LiveBlake3: decodeHexOrNil(liveHex), + PreservedBlake3: decodeHexOrNil(preservedHex), + PreservedAtPath: preservedAt, + PeerNodeID: d.peerNodeID, + RaisedRunID: d.report.RunID, + }) + if err != nil { + d.report.Warnings = append(d.report.Warnings, + fmt.Sprintf("record contested %s locally: %v", path, err)) + } +} + +// decodeHexOrNil decodes a 64-char lowercase-hex digest to raw bytes, +// returning nil for an empty or malformed value so the caller records a +// marker with an unknown-digest field rather than dropping the freeze. +func decodeHexOrNil(hexStr string) []byte { + if hexStr == "" { + return nil + } + b, err := hex.DecodeString(hexStr) + if err != nil || len(b) != 32 { + return nil + } + return b +} + +// recordAlreadyCorrect derives the count of paths the receiver already +// held correctly for the summary (F7). Under the Merkle walk only +// differing folders reach /plan, so the identical-folder files never +// appear as dispositions; already-correct is therefore present-total +// minus the paths the sync acted on (every non-already-correct +// disposition). Best-effort: a count error leaves the field zero rather +// than failing the sync over a cosmetic number. +func (d *nodeSyncDriver) recordAlreadyCorrect(plan syncproto.PlanResponse) { + actionable := 0 + for _, disp := range plan.Dispositions { + if disp.Disposition != syncproto.DispositionAlreadyCorrect { + actionable++ + } + } + present, err := d.store.CountPresentFilesInVolume(d.ctx, d.volID) + if err != nil { + return + } + if ac := present - int64(actionable); ac > 0 { + d.report.AlreadyCorrect = ac + } +} + // phaseBegin opens a session with the receiver. The initiator's own // runs row is inserted first so the (peer_node_id, correlated_run_id) // pair lands the right way around: the receiver's id becomes our @@ -248,6 +345,7 @@ func (d *nodeSyncDriver) phaseBegin() error { if err != nil { return fmt.Errorf("record peer node: %w", err) } + d.peerNodeID = peer.ID self, err := d.store.GetSelfNode(d.ctx) if err != nil { return fmt.Errorf("look up self node: %w", err) @@ -276,7 +374,7 @@ func (d *nodeSyncDriver) phaseBegin() error { InitiatorNodeName: self.Name, InitiatorRunID: runID, DedupStrategy: d.node.DedupStrategy, - ProtocolVersion: syncproto.ProtocolVersionMerkleWalk, + ProtocolVersion: syncproto.ProtocolVersionContested, }) if err != nil { return err diff --git a/sync/node_test.go b/sync/node_test.go index 429350d..176f62c 100644 --- a/sync/node_test.go +++ b/sync/node_test.go @@ -451,6 +451,175 @@ func TestNodeSyncResolvesConflictOnLocalWriteOnReceiver(t *testing.T) { } } +// countRecvConflictDirs returns how many run-/ subdirectories exist +// under the receiver volume's .squirrel-conflicts — the F27 regression +// (unbounded copies) shows up as this climbing past 1. +func countRecvConflictDirs(t *testing.T, f *nodeFixture) int { + t.Helper() + entries, err := os.ReadDir(filepath.Join(f.recvVol.Path, ConflictsDirName)) + if err != nil { + if os.IsNotExist(err) { + return 0 + } + t.Fatalf("read conflicts dir: %v", err) + } + return len(entries) +} + +// TestNodeSyncContestedFreezeEndToEnd is the #158 acceptance across two +// real nodes: a first conflict freezes the path and raises a badge on the +// initiator too (not just the hub); a later divergent push is refused with +// `contested` and mints no second copy; and an explicit resolve on the +// receiver unfreezes so the next push flows. +func TestNodeSyncContestedFreezeEndToEnd(t *testing.T) { + f := setupNodeFixture(t) + ctx := context.Background() + + // Receiver holds a local-write doc.md; the initiator diverges. + v, err := f.recvStore.CreateVolume(ctx, f.recvVol.Name, f.recvVol.Path) + if err != nil { + t.Fatalf("seed receiver volume: %v", err) + } + runID, err := f.recvStore.BeginIndexRun(ctx, store.RunKindIndex, v.ID, false) + if err != nil { + t.Fatalf("BeginIndexRun: %v", err) + } + _ = f.recvStore.FinishRun(ctx, runID, store.RunStatusSuccess, "", 1) + if err := os.WriteFile(filepath.Join(f.recvVol.Path, "doc.md"), []byte("recvr"), 0o644); err != nil { + t.Fatal(err) + } + receiverDigest := hashFile(t, filepath.Join(f.recvVol.Path, "doc.md")) + if err := f.recvStore.Upsert(ctx, store.FileRow{ + VolumeID: v.ID, Path: "doc.md", Blake3: receiverDigest, SizeBytes: 5, MtimeNs: 50, + Status: store.StatusPresent, FirstSeenRunID: runID, LastSeenRunID: runID, IndexedAtNs: 50, + }, nil); err != nil { + t.Fatalf("seed receiver file row: %v", err) + } + if err := os.WriteFile(filepath.Join(f.initVol.Path, "doc.md"), []byte("initr"), 0o644); err != nil { + t.Fatal(err) + } + f.indexInitiator(t) + + // Round 1: conflict. Initiator wins live; the loser is preserved once. + rep, err := SyncNode(ctx, f.initStore, f.rcl, f.initVol, f.node, Options{Shallow: true}) + if err != nil || len(rep.NodeConflicts) != 1 { + t.Fatalf("round 1: err=%v conflicts=%+v, want one conflict", err, rep.NodeConflicts) + } + if got := countRecvConflictDirs(t, f); got != 1 { + t.Fatalf("conflict dirs after round 1 = %d, want 1", got) + } + // The receiver froze the path... + if _, contested, _ := f.recvStore.IsPathContested(ctx, v.ID, "doc.md"); !contested { + t.Fatal("receiver did not freeze doc.md after the conflict") + } + // ...and the initiator raised its own local badge (F27: not only the hub). + initVolRow, err := f.initStore.GetVolumeByName(ctx, "pics") + if err != nil { + t.Fatalf("initiator GetVolumeByName: %v", err) + } + if _, contested, _ := f.initStore.IsPathContested(ctx, initVolRow.ID, "doc.md"); !contested { + t.Fatal("initiator did not raise a local contested marker") + } + + // Round 2: the initiator pushes yet another divergent version. The + // freeze refuses it — NodeContested, no bytes delivered, no new copy. + if err := os.WriteFile(filepath.Join(f.initVol.Path, "doc.md"), []byte("third"), 0o644); err != nil { + t.Fatal(err) + } + f.indexInitiator(t) + rep2, err := SyncNode(ctx, f.initStore, f.rcl, f.initVol, f.node, Options{Shallow: true}) + if err != nil { + t.Fatalf("round 2 SyncNode: %v", err) + } + if len(rep2.NodeContested) != 1 || rep2.NodeContested[0].Path != "doc.md" { + t.Fatalf("round 2 NodeContested = %+v, want one doc.md entry", rep2.NodeContested) + } + if len(rep2.NodeConflicts) != 0 { + t.Fatalf("round 2 NodeConflicts = %+v, want none (frozen, not re-conflicted)", rep2.NodeConflicts) + } + if got := countRecvConflictDirs(t, f); got != 1 { + t.Fatalf("conflict dirs after round 2 = %d, want 1 (no second copy)", got) + } + // The receiver's live bytes are unchanged — the divergent push was refused. + if live, _ := os.ReadFile(filepath.Join(f.recvVol.Path, "doc.md")); string(live) != "initr" { + t.Fatalf("live content after refusal = %q, want unchanged 'initr'", live) + } + + // Resolve on the receiver clears the freeze (the explicit human act). + cleared, err := f.recvStore.ClearContested(ctx, v.ID, "doc.md", "operator") + if err != nil || !cleared { + t.Fatalf("ClearContested = (%v, %v), want cleared", cleared, err) + } + + // Round 3: with the freeze lifted, the initiator's pending edit flows + // through as an ordinary supersede. + rep3, err := SyncNode(ctx, f.initStore, f.rcl, f.initVol, f.node, Options{Shallow: true}) + if err != nil { + t.Fatalf("round 3 SyncNode: %v", err) + } + if len(rep3.NodeContested) != 0 || rep3.Status != store.RunStatusSuccess { + t.Fatalf("round 3 status=%q contested=%+v, want clean success", rep3.Status, rep3.NodeContested) + } + if live, _ := os.ReadFile(filepath.Join(f.recvVol.Path, "doc.md")); string(live) != "third" { + t.Fatalf("live content after resolve+sync = %q, want 'third'", live) + } +} + +// TestNodeSyncContestedMirroredOnTransferFailure guards the observability +// fix: the initiator must mirror a freeze into its own contested_paths +// latch even when the sync fails *after* /plan. The receiver already +// pre-staged the loser and froze the path during /plan, so recording the +// badge only on a successful close would hide it on the losing edge +// exactly when a sync broke mid-flight (#158, F27). A bogus rclone binary +// fails the transfer deterministically without needing rclone installed — +// begin + plan run over the in-process HTTP receiver. +func TestNodeSyncContestedMirroredOnTransferFailure(t *testing.T) { + f, root := buildNodeFixture(t) + f.rcl = &Rclone{Binary: filepath.Join(root, "no-such-rclone-binary")} + ctx := context.Background() + + // Receiver holds a local-write doc.md; the initiator diverges → conflict. + v, err := f.recvStore.CreateVolume(ctx, f.recvVol.Name, f.recvVol.Path) + if err != nil { + t.Fatalf("seed receiver volume: %v", err) + } + runID, err := f.recvStore.BeginIndexRun(ctx, store.RunKindIndex, v.ID, false) + if err != nil { + t.Fatalf("BeginIndexRun: %v", err) + } + _ = f.recvStore.FinishRun(ctx, runID, store.RunStatusSuccess, "", 1) + if err := os.WriteFile(filepath.Join(f.recvVol.Path, "doc.md"), []byte("recvr"), 0o644); err != nil { + t.Fatal(err) + } + receiverDigest := hashFile(t, filepath.Join(f.recvVol.Path, "doc.md")) + if err := f.recvStore.Upsert(ctx, store.FileRow{ + VolumeID: v.ID, Path: "doc.md", Blake3: receiverDigest, SizeBytes: 5, MtimeNs: 50, + Status: store.StatusPresent, FirstSeenRunID: runID, LastSeenRunID: runID, IndexedAtNs: 50, + }, nil); err != nil { + t.Fatalf("seed receiver file row: %v", err) + } + if err := os.WriteFile(filepath.Join(f.initVol.Path, "doc.md"), []byte("initr"), 0o644); err != nil { + t.Fatal(err) + } + f.indexInitiator(t) + + // The transfer must fail (bogus binary), so the sync returns an error. + _, err = SyncNode(ctx, f.initStore, f.rcl, f.initVol, f.node, Options{Shallow: true}) + if err == nil { + t.Fatal("SyncNode succeeded, want a transfer failure") + } + + // Despite the failure, the initiator mirrored the freeze locally — the + // losing edge's badge / `squirrel conflicts` signal is present. + initVolRow, err := f.initStore.GetVolumeByName(ctx, "pics") + if err != nil { + t.Fatalf("initiator GetVolumeByName: %v", err) + } + if _, contested, err := f.initStore.IsPathContested(ctx, initVolRow.ID, "doc.md"); err != nil || !contested { + t.Fatalf("initiator IsPathContested = (%v, %v), want frozen after a post-plan failure", contested, err) + } +} + // TestNodeSyncVerifyMismatchPartialStatus simulates rclone "succeeding" // but the on-disk content being wrong (the agent's re-hash will // catch it). We inject the mismatch by writing different content diff --git a/sync/packed.go b/sync/packed.go index 1a117a5..d43bbb9 100644 --- a/sync/packed.go +++ b/sync/packed.go @@ -369,7 +369,7 @@ func (h *packedHandler) watermark(ctx context.Context, volID int64) (int64, erro if freshStartOnEmptyRoot(ctx, h.rcl, h.dest) { return 0, nil } - return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no pack placement map at %s — its history is not packed (a mirror or content-addressed root); point the layout at a fresh destination or root, or (after wiping the remote root) run `squirrel destination reset %s`, instead of switching an existing one: %w", h.dest.Name, last.ID, mapURI, h.dest.Name, err) + return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no pack placement map at %s — its history is not packed (a mirror or content-addressed root); point the layout at a fresh destination or root, or (after wiping the remote root) run `squirrel destination reset %s`, instead of switching an existing one: %w: %w", h.dest.Name, last.ID, mapURI, h.dest.Name, err, ErrRefused) } return last.ID, nil } diff --git a/sync/packed_test.go b/sync/packed_test.go index c586284..a35315f 100644 --- a/sync/packed_test.go +++ b/sync/packed_test.go @@ -504,8 +504,11 @@ func TestPackedWatermarkGuardRefusesContentAddressed(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "not packed") { t.Fatalf("expected packed-history refusal, got %v", err) } - if rep.Status != store.RunStatusFailed { - t.Fatalf("Status = %q, want failed", rep.Status) + if !errors.Is(err, ErrRefused) { + t.Fatalf("layout guard should be a refusal (ErrRefused), got %v", err) + } + if rep.Status != store.RunStatusRefused { + t.Fatalf("Status = %q, want refused", rep.Status) } } @@ -531,8 +534,11 @@ func TestPackedWatermarkGuardRefusesMirror(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "not packed") { t.Fatalf("expected packed-history refusal, got %v", err) } - if rep.Status != store.RunStatusFailed { - t.Fatalf("Status = %q, want failed", rep.Status) + if !errors.Is(err, ErrRefused) { + t.Fatalf("layout guard should be a refusal (ErrRefused), got %v", err) + } + if rep.Status != store.RunStatusRefused { + t.Fatalf("Status = %q, want refused", rep.Status) } } diff --git a/sync/rclone.go b/sync/rclone.go index 64caa7c..27e328e 100644 --- a/sync/rclone.go +++ b/sync/rclone.go @@ -22,6 +22,8 @@ import ( "strconv" "strings" "sync" + "sync/atomic" + "time" "github.com/mbertschler/squirrel/config" "github.com/mbertschler/squirrel/runevents" @@ -33,6 +35,26 @@ import ( // --hash blake3 flag used by sync would be rejected by rclone. var MinRcloneVersion = Version{Major: 1, Minor: 66} +// DefaultStallTimeout is the no-progress bound the agent scheduler applies +// to every automatic rclone transfer (see Rclone.StallTimeout). Ten +// minutes is comfortably longer than the longest legitimate gap between +// progress reports — hashing one very large file for a --checksum +// comparison surfaces no transfer/check advance until it completes — while +// bounding a wedged endpoint far below the "indefinite" hang F25 recorded. +// A dark endpoint usually fails sooner via rclone's own --contimeout / +// --timeout; the guard covers the live-but-stuck case those don't. +const DefaultStallTimeout = 10 * time.Minute + +// rcloneConnectTimeout and rcloneIOTimeout are passed as --contimeout and +// --timeout on every streaming transfer so a dead or unresponsive endpoint +// self-terminates instead of hanging. They match rclone's own defaults but +// are set explicitly so the bound is guaranteed applied and lives in one +// tunable place (#160, F25). +const ( + rcloneConnectTimeout = "1m" + rcloneIOTimeout = "5m" +) + // Rclone is a configured rclone wrapper. Binary is the resolved executable // path; Config is the path to the squirrel-managed rclone.conf written by // WriteRcloneConfig. All Run invocations pass --config Config so the user's @@ -40,6 +62,17 @@ var MinRcloneVersion = Version{Major: 1, Minor: 66} type Rclone struct { Binary string Config string + // StallTimeout, when > 0, bounds a streaming transfer (RunWithProgress) + // by progress rather than by total wall-clock: if rclone reports no + // advance in transferred+checked+bytes for this long, the child is + // killed and the run fails with a diagnosable "stalled" error. It is the + // squirrel-side backstop for a connection that stays alive but stops + // making progress — the storage-full S3 hang of F25 that tripped neither + // --contimeout nor --timeout. Zero (the default, used by the foreground + // CLI where a human can interrupt) disables the guard; the agent + // scheduler sets it to DefaultStallTimeout so an unattended cadence can + // never wedge forever. + StallTimeout time.Duration } // Find locates the rclone binary on PATH. The returned Rclone has Config @@ -240,6 +273,33 @@ type RunResult struct { // for BLAKE3 verification but hit this path was not content-verified, // however rclone exited, so the caller must not record it as verified. HashFallback bool + // Stderr is a bounded tail of rclone's non-JSON stderr — the + // human-readable diagnostics it prints before (or instead of) the + // structured JSON log: backend construction, config, auth, and host-key + // failures. Bounded to maxStderrCapture bytes so a pathological run + // can't balloon memory. It is the evidence that made a failing + // destination undiagnosable when it was dropped on the floor (#157, + // F6/F15); callers fold it into the run row's error and the scheduler + // log. Empty when rclone emitted only structured events. + Stderr string +} + +// DisplayErrors is the error count for summary lines and status +// reconciliation. It returns rclone's per-file Errors, except that a fatal +// invocation failure that produced no per-file count reports the number of +// captured diagnostics (at least one) — so a run that is status=failed +// never simultaneously claims errors=0, the contradiction F6 flagged. +func (r RunResult) DisplayErrors() int64 { + if r.Errors > 0 { + return r.Errors + } + if r.FatalError { + if n := int64(len(r.FailedFiles)); n > 0 { + return n + } + return 1 + } + return 0 } // FailedFile is one per-object error from the JSON log. Object may be @@ -254,6 +314,11 @@ type FailedFile struct { // error count in RunResult.Errors is exact regardless of this cap. const maxFailedFiles = 100 +// maxStderrCapture bounds RunResult.Stderr. rclone's fatal diagnostics are +// a line or two; 4 KiB keeps a useful tail of a chattier failure without +// letting a misbehaving child stream unbounded bytes into the run row. +const maxStderrCapture = 4 << 10 + // Run executes `rclone --config --use-json-log` with // the given extra arguments. It streams both stdout (rclone writes to // stderr by default but --use-json-log routes structured output to stderr @@ -278,23 +343,18 @@ func (r *Rclone) RunWithProgress(ctx context.Context, onProgress func(runevents. if r.Config == "" { return RunResult{}, errors.New("rclone wrapper: Config not set (call WriteRcloneConfig first)") } - full := append([]string{ - "--config", r.Config, - "--use-json-log", - "--log-level", "INFO", - "--stats", "1s", - }, args...) - cmd := exec.CommandContext(ctx, r.Binary, full...) - stderr, err := cmd.StderrPipe() - if err != nil { - return RunResult{}, fmt.Errorf("rclone stderr pipe: %w", err) + // A per-run cancel backs the no-progress guard: rclone's own timeouts + // bound a dead connection, the guard bounds a live-but-wedged one (F25). + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + var guard *stallGuard + if r.StallTimeout > 0 { + guard = newStallGuard(runCtx, r.StallTimeout, cancel) } - stdout, err := cmd.StdoutPipe() + + cmd, stderr, stdout, err := r.startRclone(runCtx, args) if err != nil { - return RunResult{}, fmt.Errorf("rclone stdout pipe: %w", err) - } - if err := cmd.Start(); err != nil { - return RunResult{}, fmt.Errorf("rclone start: %w", err) + return RunResult{}, err } var ( @@ -304,7 +364,7 @@ func (r *Rclone) RunWithProgress(ctx context.Context, onProgress func(runevents. wg.Add(2) go func() { defer wg.Done() - parseJSONLog(stderr, &result, onProgress) + parseJSONLog(stderr, &result, onProgress, guard.advance) }() go func() { defer wg.Done() @@ -315,13 +375,147 @@ func (r *Rclone) RunWithProgress(ctx context.Context, onProgress func(runevents. wg.Wait() waitErr := cmd.Wait() + cancel() + guard.wait() if waitErr != nil { result.FatalError = result.Errors == 0 - return result, fmt.Errorf("rclone exit: %w", waitErr) + return result, rcloneRunError(waitErr, result.Stderr, guard, r.StallTimeout) } return result, nil } +// startRclone assembles the full argument list — config, JSON logging, +// per-second stats, and the connect/IO timeouts every transfer carries +// (#160, F25) — and starts the child on runCtx, returning its stderr and +// stdout pipes. runCtx backs both the caller's cancellation and the stall +// guard's kill. +func (r *Rclone) startRclone(runCtx context.Context, args []string) (*exec.Cmd, io.Reader, io.Reader, error) { + full := append([]string{ + "--config", r.Config, + "--use-json-log", + "--log-level", "INFO", + "--stats", "1s", + "--contimeout", rcloneConnectTimeout, + "--timeout", rcloneIOTimeout, + }, args...) + cmd := exec.CommandContext(runCtx, r.Binary, full...) + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, nil, nil, fmt.Errorf("rclone stderr pipe: %w", err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, nil, fmt.Errorf("rclone stdout pipe: %w", err) + } + if err := cmd.Start(); err != nil { + return nil, nil, nil, fmt.Errorf("rclone start: %w", err) + } + return cmd, stderr, stdout, nil +} + +// stallGuard kills a running rclone child when a streaming transfer stops +// making progress for longer than timeout. rclone's --contimeout/--timeout +// bound a dead connection; the guard is the squirrel-side backstop for a +// connection that stays alive yet stops advancing — the storage-full hang +// of F25 that tripped neither. It is driven off rclone's periodic --stats +// events (via notifyAdvance): any growth in transferred+checked+bytes +// resets the timer, so a slow-but-moving transfer of a large volume is +// never killed while a wedged one is. +type stallGuard struct { + timeout time.Duration + cancel context.CancelFunc + poke chan struct{} + done chan struct{} + fired atomic.Bool +} + +func newStallGuard(ctx context.Context, timeout time.Duration, cancel context.CancelFunc) *stallGuard { + g := &stallGuard{ + timeout: timeout, + cancel: cancel, + poke: make(chan struct{}, 1), + done: make(chan struct{}), + } + go g.watch(ctx) + return g +} + +// advance signals progress from the stats-reader goroutine. Non-blocking: +// a pending poke already carries the reset, so a dropped signal is +// harmless. Safe on a nil guard so the disabled path needs no branch. +func (g *stallGuard) advance() { + if g == nil { + return + } + select { + case g.poke <- struct{}{}: + default: + } +} + +// watch resets its timer on every progress poke and, when the timer +// expires with no intervening progress, records the stall and cancels the +// run context — which makes exec.CommandContext kill the rclone child. It +// returns when ctx is done (the run finished or was cancelled elsewhere). +func (g *stallGuard) watch(ctx context.Context) { + defer close(g.done) + timer := time.NewTimer(g.timeout) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-g.poke: + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(g.timeout) + case <-timer.C: + g.fired.Store(true) + g.cancel() + return + } + } +} + +// wait blocks until the watch goroutine has returned so the caller can read +// fired without a race. Safe on a nil guard. +func (g *stallGuard) wait() { + if g == nil { + return + } + <-g.done +} + +// rcloneRunError builds the error for a non-zero rclone exit. When the +// stall guard fired, the exit is a kill squirrel caused, so we say so +// plainly (folding in the captured stderr tail, #157) rather than +// surfacing a bare "signal: killed"; otherwise it defers to +// rcloneExitError. +func rcloneRunError(waitErr error, stderrTail string, guard *stallGuard, timeout time.Duration) error { + if guard != nil && guard.fired.Load() { + if tail := strings.TrimSpace(stderrTail); tail != "" { + return fmt.Errorf("rclone stalled: no progress for %s, killed: %w: %s", timeout, waitErr, tail) + } + return fmt.Errorf("rclone stalled: no progress for %s, killed: %w", timeout, waitErr) + } + return rcloneExitError(waitErr, stderrTail) +} + +// rcloneExitError wraps a non-zero rclone exit, folding in the captured +// stderr tail when present so the scheduler log and the run row carry the +// actual reason (auth, host key, path) rather than a bare "exit status 1" +// (#157, F6/F15). +func rcloneExitError(waitErr error, stderrTail string) error { + if tail := strings.TrimSpace(stderrTail); tail != "" { + return fmt.Errorf("rclone exit: %w: %s", waitErr, tail) + } + return fmt.Errorf("rclone exit: %w", waitErr) +} + // runPlain executes rclone with the wrapper's --config but without the // JSON-log and stats flags Run uses, returning captured stdout. It backs // the auxiliary commands the snapshot ride-along needs (copyto, lsf, @@ -605,21 +799,42 @@ var hashFallbackRE = regexp.MustCompile(`no hashes in common`) func isHashFallback(msg string) bool { return hashFallbackRE.MatchString(msg) } +// isErrorLevel reports whether a JSON event's level should be captured as +// a failure diagnostic. rclone reports a fatal backend/config/auth failure +// at "fatal" (sometimes "critical"), not "error"; capturing only "error" +// dropped exactly those messages, leaving a failed run's ERROR column +// blank (#157, F15). All three are folded into FailedFiles. +func isErrorLevel(level string) bool { + switch level { + case "error", "fatal", "critical": + return true + } + return false +} + // parseJSONLog reads JSON-per-line events from r and updates result in -// place. Non-JSON lines (e.g. an early startup notice on an older rclone) -// are skipped — we cannot make decisions on them and surfacing them as -// errors would create false positives. onProgress, if non-nil, is -// invoked once per stats event so callers can drive a live UI. -func parseJSONLog(r io.Reader, result *RunResult, onProgress func(runevents.Progress)) { +// place. Non-JSON lines (rclone's pre-logger diagnostics — the backend, +// config, auth, and host-key failures it prints before the structured +// logger is live) are appended to result.Stderr, bounded, so a failure +// that never reaches the JSON log still carries its reason (#157, +// F6/F15). onProgress, if non-nil, is invoked once per stats event so +// callers can drive a live UI. +func parseJSONLog(r io.Reader, result *RunResult, onProgress func(runevents.Progress), onAdvance func()) { scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + var lastProgressTotal int64 for scanner.Scan() { line := scanner.Bytes() - if len(line) == 0 || line[0] != '{' { + if len(line) == 0 { + continue + } + if line[0] != '{' { + appendStderr(result, line) continue } var ev rcloneEvent if err := json.Unmarshal(line, &ev); err != nil { + appendStderr(result, line) continue } if isHashFallback(ev.Msg) { @@ -629,30 +844,11 @@ func parseJSONLog(r io.Reader, result *RunResult, onProgress func(runevents.Prog result.HashFallback = true } if ev.Stats != nil { - result.Transferred = ev.Stats.TotalTransfers - result.Checked = ev.Stats.TotalChecks - result.Bytes = ev.Stats.Bytes - // Don't overwrite Errors with a smaller value from a - // per-attempt stats line; take the maximum so retries that - // fail then succeed still surface the failure count. - if ev.Stats.Errors > result.Errors { - result.Errors = ev.Stats.Errors - } - if ev.Stats.FatalError { - result.FatalError = true - } - if onProgress != nil { - onProgress(runevents.Progress{ - Stage: runevents.StageUploading, - Done: result.Transferred, - Total: ev.Stats.TotalTransfers + ev.Stats.TotalChecks, - BytesDone: result.Bytes, - BytesTotal: ev.Stats.TotalBytes, - }) - } + applyStatsEvent(result, ev.Stats, onProgress) + notifyAdvance(result, &lastProgressTotal, onAdvance) continue } - if ev.Level == "error" && !isRetrySummary(ev.Msg) { + if isErrorLevel(ev.Level) && !isRetrySummary(ev.Msg) { // Capture object-less errors too: auth failures, listing // errors, "Failed to copy: …" diagnostics carry no Object // but are exactly the messages we want in runs.error. Filter @@ -666,3 +862,65 @@ func parseJSONLog(r io.Reader, result *RunResult, onProgress func(runevents.Prog } } } + +// notifyAdvance pokes the stall guard whenever the running total of +// transferred+checked+bytes has grown since the last stats event, so a +// transfer that is slow but genuinely moving keeps resetting the guard's +// timer while a wedged one — rclone still emitting per-second stats, but +// with flat counters — lets it fire. Checks count as progress so a long +// verification pass over an already-populated destination is not mistaken +// for a stall. +func notifyAdvance(result *RunResult, last *int64, onAdvance func()) { + if onAdvance == nil { + return + } + total := result.Transferred + result.Checked + result.Bytes + if total > *last { + *last = total + onAdvance() + } +} + +// applyStatsEvent folds one rclone stats event into result and drives the +// optional progress callback. Errors is taken as the running maximum so a +// per-attempt stats line that reports fewer errors than a prior one (a +// retry that partially recovered) can't erase the peak failure count. +func applyStatsEvent(result *RunResult, st *rcloneStats, onProgress func(runevents.Progress)) { + result.Transferred = st.TotalTransfers + result.Checked = st.TotalChecks + result.Bytes = st.Bytes + if st.Errors > result.Errors { + result.Errors = st.Errors + } + if st.FatalError { + result.FatalError = true + } + if onProgress != nil { + onProgress(runevents.Progress{ + Stage: runevents.StageUploading, + Done: result.Transferred, + Total: st.TotalTransfers + st.TotalChecks, + BytesDone: result.Bytes, + BytesTotal: st.TotalBytes, + }) + } +} + +// appendStderr accumulates a non-JSON stderr line into result.Stderr, +// keeping the last maxStderrCapture bytes. rclone prints its fatal error +// line last, so the tail is the most diagnostic slice to retain when a +// chatty run overflows the bound; older lines are trimmed from the front. +// Blank lines are skipped so the capture stays dense. +func appendStderr(result *RunResult, line []byte) { + trimmed := strings.TrimSpace(string(line)) + if trimmed == "" { + return + } + if result.Stderr != "" { + result.Stderr += "\n" + } + result.Stderr += trimmed + if len(result.Stderr) > maxStderrCapture { + result.Stderr = result.Stderr[len(result.Stderr)-maxStderrCapture:] + } +} diff --git a/sync/rclone_test.go b/sync/rclone_test.go index 560fdfa..3d2ec2a 100644 --- a/sync/rclone_test.go +++ b/sync/rclone_test.go @@ -2,10 +2,12 @@ package sync import ( "context" + "fmt" "os" "os/exec" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -13,6 +15,65 @@ import ( "github.com/mbertschler/squirrel/runevents" ) +// TestStallGuardFiresWithoutProgress proves the no-progress guard cancels +// the run — and records that it fired — when no advance arrives within the +// timeout. This is the F25 wedge: rclone alive but transferring nothing. +// The test waits for the guard rather than racing a deadline, so the exact +// timeout is not timing-sensitive; a generous value keeps it robust under +// load. +func TestStallGuardFiresWithoutProgress(t *testing.T) { + var cancelled atomic.Bool + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + g := newStallGuard(ctx, 100*time.Millisecond, func() { + cancelled.Store(true) + cancel() + }) + g.wait() // blocks until watch returns; with no poke it fires first + if !g.fired.Load() { + t.Fatal("guard did not fire after the stall timeout elapsed with no progress") + } + if !cancelled.Load() { + t.Fatal("guard fired but never cancelled the run context") + } +} + +// TestStallGuardResetsOnProgress proves a transfer that keeps advancing is +// never killed: pokes driven off a ticker at a fraction of the stall window +// keep resetting it across a span twice as long as one window, so only a +// working reset explains the guard staying quiet. The wide ratio between +// the stall timeout and the poke interval keeps scheduler jitter from +// faking a stall. +func TestStallGuardResetsOnProgress(t *testing.T) { + var cancelled atomic.Bool + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + const stall = 400 * time.Millisecond + g := newStallGuard(ctx, stall, func() { + cancelled.Store(true) + cancel() + }) + tick := time.NewTicker(stall / 20) // pokes at 20x the resolution of the bound + defer tick.Stop() + deadline := time.After(2 * stall) // outlast a single window, so a reset is required + for done := false; !done; { + select { + case <-tick.C: + g.advance() + case <-deadline: + done = true + } + } + if g.fired.Load() { + t.Fatal("guard fired while progress was still arriving") + } + cancel() // finishing the run stops the guard cleanly + g.wait() + if cancelled.Load() { + t.Fatal("guard cancelled the run despite steady progress") + } +} + // requireRclone skips the test if rclone is not on PATH. The wrapper tests // exercise the real binary against a local-filesystem destination; if a // developer wants to run them they need rclone installed. @@ -57,7 +118,7 @@ func TestParseJSONLogCapturesObjectlessErrors(t *testing.T) { `{"stats":{"errors":1,"fatalError":true,"totalTransfers":0,"totalChecks":0,"bytes":0}}`, }, "\n") var r RunResult - parseJSONLog(strings.NewReader(stream), &r, nil) + parseJSONLog(strings.NewReader(stream), &r, nil, nil) if len(r.FailedFiles) != 2 { t.Fatalf("FailedFiles = %+v, want 2 (auth + reading; retry summaries dropped)", r.FailedFiles) @@ -73,6 +134,74 @@ func TestParseJSONLogCapturesObjectlessErrors(t *testing.T) { } } +// TestParseJSONLogCapturesFatalLevelAndRawStderr: a fatal-level JSON event +// (previously dropped by the error-only filter) and a non-JSON pre-logger +// diagnostic are both captured, so a failing destination is diagnosable +// (#157, F6/F15) instead of leaving a blank ERROR column. +func TestParseJSONLogCapturesFatalLevelAndRawStderr(t *testing.T) { + stream := strings.Join([]string{ + `Failed to create file system for "cloudbox:/x": didn't find backend`, + `{"level":"fatal","msg":"NoCredentialProviders: no valid providers in chain"}`, + `{"stats":{"errors":0,"fatalError":true,"totalTransfers":0,"totalChecks":0,"bytes":0}}`, + }, "\n") + var r RunResult + parseJSONLog(strings.NewReader(stream), &r, nil, nil) + + if len(r.FailedFiles) != 1 || !strings.Contains(r.FailedFiles[0].Message, "NoCredentialProviders") { + t.Fatalf("FailedFiles = %+v, want the fatal-level message captured", r.FailedFiles) + } + if !strings.Contains(r.Stderr, "didn't find backend") { + t.Fatalf("Stderr = %q, want the non-JSON pre-logger diagnostic", r.Stderr) + } +} + +// TestParseJSONLogStderrBounded: a pathological non-JSON stream can't +// balloon RunResult.Stderr past the cap, and the capture keeps the tail +// (rclone's fatal line prints last) — the final line survives while an +// early one is trimmed from the front. +func TestParseJSONLogStderrBounded(t *testing.T) { + lines := make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + lines = append(lines, fmt.Sprintf("stderr line %04d", i)) + } + var r RunResult + parseJSONLog(strings.NewReader(strings.Join(lines, "\n")), &r, nil, nil) + if len(r.Stderr) == 0 { + t.Fatal("Stderr empty, want the tail of the stream") + } + if len(r.Stderr) > maxStderrCapture { + t.Fatalf("Stderr len = %d, want <= %d", len(r.Stderr), maxStderrCapture) + } + if !strings.Contains(r.Stderr, "stderr line 1999") { + t.Fatal("Stderr lost the final line — tail not kept") + } + if strings.Contains(r.Stderr, "stderr line 0000") { + t.Fatal("Stderr kept the first line — want the tail only") + } +} + +// TestDisplayErrors pins the summary error count: rclone's per-file count +// wins, but a fatal invocation failure with no per-file count still +// reports at least one error so a status=failed run never claims errors=0. +func TestDisplayErrors(t *testing.T) { + cases := []struct { + name string + r RunResult + want int64 + }{ + {"per-file errors", RunResult{Errors: 3}, 3}, + {"fatal, no count, no captured files", RunResult{FatalError: true}, 1}, + {"fatal, captured files", RunResult{FatalError: true, FailedFiles: []FailedFile{{Message: "a"}, {Message: "b"}}}, 2}, + {"fatal but per-file count wins", RunResult{FatalError: true, Errors: 5}, 5}, + {"clean", RunResult{}, 0}, + } + for _, c := range cases { + if got := c.r.DisplayErrors(); got != c.want { + t.Errorf("%s: DisplayErrors() = %d, want %d", c.name, got, c.want) + } + } +} + // TestParseJSONLogDetectsHashFallback: rclone's no-common-hash notice is // emitted at NOTICE level, which the error filter drops; parseJSONLog // still flags it so a flags-set, exit-0 run that silently degraded to a @@ -83,7 +212,7 @@ func TestParseJSONLogDetectsHashFallback(t *testing.T) { `{"stats":{"errors":0,"fatalError":false,"totalTransfers":2,"totalChecks":0,"bytes":10}}`, }, "\n") var r RunResult - parseJSONLog(strings.NewReader(stream), &r, nil) + parseJSONLog(strings.NewReader(stream), &r, nil, nil) if !r.HashFallback { t.Fatalf("HashFallback = false, want true (no-common-hash notice should be detected)") @@ -98,7 +227,7 @@ func TestParseJSONLogDetectsHashFallback(t *testing.T) { func TestParseJSONLogNoFalseHashFallback(t *testing.T) { stream := `{"stats":{"errors":0,"fatalError":false,"totalTransfers":2,"totalChecks":1,"bytes":10}}` var r RunResult - parseJSONLog(strings.NewReader(stream), &r, nil) + parseJSONLog(strings.NewReader(stream), &r, nil, nil) if r.HashFallback { t.Fatalf("HashFallback = true on a clean run, want false") } @@ -117,7 +246,7 @@ func TestParseJSONLogEmitsProgressWithByteTotal(t *testing.T) { var events []runevents.Progress parseJSONLog(strings.NewReader(stream), &r, func(p runevents.Progress) { events = append(events, p) - }) + }, nil) if len(events) != 2 { t.Fatalf("progress events = %d, want 2", len(events)) } diff --git a/sync/sync.go b/sync/sync.go index 89a6eef..df6319e 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -50,6 +50,28 @@ const RestoreHistoryDirName = ".squirrel-restore-history" // and from peer-sync so a snapshot is never mistaken for user content. const IndexDirName = ".squirrel-index" +// ErrRefused marks a preflight safety refusal: a gate that declined to +// proceed before any transfer began — a missing or mismatched +// .squirrel-volume marker, a kopia connect that found no repository +// without --init, or a layout guard protecting a destination whose +// recorded history belongs to a different layout. Gates wrap it into their +// error with %w so the run-finalising path records store.RunStatusRefused +// rather than RunStatusFailed: a refusal is a standing "won't do this" +// condition, not a mid-flight failure, and the audit trail keeps the two +// distinct (#157, F26). +var ErrRefused = errors.New("refused") + +// terminalStatus resolves a run's terminal status from the derived status +// and the operation error. A preflight refusal (errors.Is(err, +// ErrRefused)) overrides to 'refused'; every other outcome keeps the +// derived status. +func terminalStatus(status string, runErr error) string { + if runErr != nil && errors.Is(runErr, ErrRefused) { + return store.RunStatusRefused + } + return status +} + // Options shapes one Sync invocation. type Options struct { // Shallow drops --checksum and --hash blake3 so rclone uses its default @@ -98,6 +120,15 @@ type Report struct { RunID int64 RcloneResult RunResult Status string // success / partial / failed + // AlreadyCorrect counts the paths the destination already held + // correctly, so a no-op-because-in-sync is distinguishable from a + // no-op-because-empty in the summary (transferred=0 alone is + // ambiguous — friction F7). For rclone bucket pushes and restores it + // is rclone's already-matching count; for peer syncs the handler + // derives it as present-total minus the paths the sync acted on + // (the Merkle walk never sends identical folders to /plan, so it + // cannot be counted from the disposition list alone). + AlreadyCorrect int64 // Verification is the handler's typed durability report for this // push: which comparison backed it, what the tool counted, and — // via Verified() — whether the destination's copy was @@ -141,6 +172,16 @@ type Report struct { // and the new BLAKE3 plus the receiver-relative preserved path so // the CLI can render a meaningful "review at " pointer. NodeConflicts []syncproto.ConflictDetail + // NodeContested is non-empty when the receiver refused one or more + // paths with the `contested` disposition: paths frozen by a prior + // conflict whose divergent re-assertion is declined until an operator + // resolves it (#158, F27). The initiator's bytes for these paths did + // NOT land — both existing versions stay preserved on the receiver. + // Each record carries the frozen winner + preserved-loser digests and + // the receiver-relative preserved location so the initiator mirrors + // the freeze into its own contested marker and points the operator at + // both versions. + NodeContested []syncproto.ContestedDetail // SnapshotErr captures a failure to take the post-sync index // snapshot or to ride it along to the destination (#75). It is // strictly defense-in-depth: a snapshot failure must not flip a @@ -257,8 +298,12 @@ func Sync(ctx context.Context, s *store.Store, rcl *Rclone, vol *config.Volume, // an uninitialised destination would prevent the "preview what would // happen" workflow. if !opts.DryRun { - if err := ensureDestinationMarker(ctx, s, rcl, dest, vol.Name, opts.Init); err != nil { - return rep, err + if merr := ensureDestinationMarker(ctx, s, rcl, dest, vol.Name, opts.Init); merr != nil { + // A marker refusal fires before the sync run is allocated, so + // record it as its own terminal 'refused' run — otherwise a + // month-dead backup disk produces zero red anywhere but agent + // stderr (#157, F26). + return recordSyncRefusal(ctx, s, volID, dest.Name, &rep, merr) } } @@ -321,6 +366,28 @@ func captureDurabilityAdvance(ctx context.Context, s *store.Store, volumeID int6 return components, nil } +// recordSyncRefusal mints a terminal kind='sync' run in the 'refused' +// state for a preflight gate that declined before the transfer began, so +// the refusal is visible in `squirrel runs` and the TUI instead of living +// only in the returned error (#157, F26). The run is allocated ungated +// (BeginRun, not the concurrency gate) — a refusal transfers nothing and +// need not contend with an in-flight sync — and finished immediately with +// refErr as the row's error. refErr is returned unchanged so callers +// surface the same message; a failure to even record the refusal is folded +// into the returned error rather than masking the original refusal. +func recordSyncRefusal(ctx context.Context, s *store.Store, volID int64, destName string, rep *Report, refErr error) (Report, error) { + id, err := s.BeginRun(ctx, store.RunKindSync, volID, destName, false) + if err != nil { + return *rep, fmt.Errorf("%w (also failed to record the refusal as a run: %w)", refErr, err) + } + rep.RunID = id + rep.Status = store.RunStatusRefused + if ferr := s.FinishRun(ctx, id, store.RunStatusRefused, refErr.Error(), 0); ferr != nil { + rep.FinishErr = ferr + } + return *rep, refErr +} + // beginSyncRunGuarded is the sync-allocator the bucket and peer paths // share. It honours dry-run (returns 0 with no DB write) and delegates // to store.BeginSyncRunIfClear for the atomic gate. A blocked attempt @@ -373,11 +440,19 @@ func runRcloneOperation( return err } rep.RcloneResult, err = rcl.RunWithProgress(ctx, progress, args...) - if err != nil && rep.RcloneResult.Errors == 0 && !rep.RcloneResult.FatalError { - // Invocation failed without a parseable error count: treat as fatal. - rep.RcloneResult.FatalError = true - } if err != nil { + if rep.RcloneResult.Errors == 0 && !rep.RcloneResult.FatalError { + // Invocation failed without a parseable error count: treat as fatal. + rep.RcloneResult.FatalError = true + } + // Ensure a diagnostic reaches the run row's error column and the + // CLI output even when rclone logged no per-file error event — auth, + // host-key, and config failures print to stderr, not the JSON log + // (#157, F6/F15). The returned error already folds in the stderr + // tail, so it is the most complete single message available. + if len(rep.RcloneResult.FailedFiles) == 0 { + rep.RcloneResult.FailedFiles = []FailedFile{{Message: err.Error()}} + } return fmt.Errorf("rclone: %w", err) } return nil @@ -439,6 +514,10 @@ func beginRestoreRun(ctx context.Context, s *store.Store, dryRun bool, volID int // to the rclone outcome on this very run. func finishRun(ctx context.Context, s *store.Store, dryRun bool, runID int64, rep *Report) { rep.Status = deriveStatus(rep.RcloneResult) + // rclone's "checks" are the files it found already matching at the + // destination and did not transfer — exactly the already-correct + // count the summary reports (F7). Peer syncs set this themselves. + rep.AlreadyCorrect = rep.RcloneResult.Checked if dryRun || runID == 0 { return } @@ -533,13 +612,13 @@ func ensureLocalDestinationMarker(ctx context.Context, s *store.Store, dest *con return nil } if _, ok := errors.AsType[*volmark.ErrMismatch](err); ok { - return fmt.Errorf("destination %q: %w (refuse to init over a different volume's tree)", dest.Name, err) + return fmt.Errorf("destination %q: %w (refuse to init over a different volume's tree): %w", dest.Name, err, ErrRefused) } if !errors.Is(err, volmark.ErrMissing) { return fmt.Errorf("destination %q marker check: %w", dest.Name, err) } if !init { - return fmt.Errorf("destination %q at %s has no %s marker — re-run with --init to bootstrap (refusing in case the root is a typo)", dest.Name, root, volmark.MarkerName) + return fmt.Errorf("destination %q at %s has no %s marker — re-run with --init to bootstrap (refusing in case the root is a typo): %w", dest.Name, root, volmark.MarkerName, ErrRefused) } if err := os.MkdirAll(root, 0o755); err != nil { return fmt.Errorf("destination %q: mkdir %s: %w", dest.Name, root, err) diff --git a/sync/sync_test.go b/sync/sync_test.go index a035564..4086519 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -2,6 +2,7 @@ package sync import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -836,10 +837,26 @@ func TestSyncRefusesUninitialisedDestination(t *testing.T) { t.Fatalf("remove seeded marker: %v", err) } - _, err := Sync(context.Background(), f.store, f.rcl, f.vol, f.dest, Options{}) + rep, err := Sync(context.Background(), f.store, f.rcl, f.vol, f.dest, Options{}) if err == nil || !strings.Contains(err.Error(), "--init") { t.Fatalf("expected --init hint, got %v", err) } + if !errors.Is(err, ErrRefused) { + t.Fatalf("marker refusal should wrap ErrRefused, got %v", err) + } + // The refusal must mint a visible terminal run row, not vanish into + // the returned error with run_id=0 (#157, F26). + if rep.RunID == 0 || rep.Status != store.RunStatusRefused { + t.Fatalf("rep = %+v, want a refused run row", rep) + } + run, gerr := f.store.GetRun(context.Background(), rep.RunID) + if gerr != nil { + t.Fatalf("GetRun(%d): %v", rep.RunID, gerr) + } + if run.Kind != store.RunKindSync || run.Status != store.RunStatusRefused || + !run.Error.Valid || !strings.Contains(run.Error.String, "--init") { + t.Fatalf("run = %+v, want a refused sync run carrying the refusal message", run) + } } // TestSyncInitWritesMarker confirms that passing Init: true creates diff --git a/sync/verify_remote.go b/sync/verify_remote.go index 3eca4b5..2901111 100644 --- a/sync/verify_remote.go +++ b/sync/verify_remote.go @@ -56,6 +56,15 @@ type RemoteVerifyReport struct { // PackMismatched lists packs whose provider checksum no longer matches // the recorded one (Hash holds the pack key hex). PackMismatched []RemoteObjectMismatch + + // AlarmRaised is true when this pass latched a new standing alarm on + // the destination because it was not clean (#157, F30). A pass on an + // already-alarmed destination leaves it false (the latch was already + // there). AlarmCleared is true when a clean pass auto-cleared a + // previously standing alarm. Both drive the CLI's loud surfacing; the + // authoritative state lives in the destination_alarms latch. + AlarmRaised bool + AlarmCleared bool } // RemoteObjectMismatch is one object whose provider checksum no longer @@ -439,5 +448,38 @@ func recordVerifyOutcome(ctx context.Context, s *store.Store, rep *RemoteVerifyR if err := s.FinishRun(ctx, rep.RunID, status, errMsg, int64(rep.Objects+rep.Packs)); err != nil { return fmt.Errorf("finish verify run %d: %w", rep.RunID, err) } + return applyVerifyAlarm(ctx, s, rep, verifyErr) +} + +// applyVerifyAlarm latches or clears the destination's standing alarm from +// this pass's outcome (#157, F30). A pass that detected a mismatch or a +// missing object/pack raises the alarm (idempotent — a re-detection keeps +// the original "in alarm since"); a clean pass auto-clears any standing +// alarm, recording the clear against this verify run. A pass that aborted +// (verifyErr != nil) proves nothing about the destination's integrity, so +// it neither raises nor clears — its failed run row is the record. +func applyVerifyAlarm(ctx context.Context, s *store.Store, rep *RemoteVerifyReport, verifyErr error) error { + if verifyErr != nil { + return nil + } + if !rep.Clean() { + detail := fmt.Sprintf("objects mismatched=%d missing=%d, packs mismatched=%d missing=%d", + len(rep.Mismatched), len(rep.Missing), len(rep.PackMismatched), len(rep.PacksMissing)) + // AlarmRaised reflects only a *newly* created latch, taken from the + // atomic insert itself — so the CLI shouts on first detection but a + // concurrent second raise (which found the latch already there) + // stays quiet. No pre-read: that would be a TOCTOU race. + raised, err := s.RaiseDestinationAlarm(ctx, rep.Destination, store.AlarmKindVerifyMismatch, detail, rep.RunID) + if err != nil { + return fmt.Errorf("raise standing alarm for %q: %w", rep.Destination, err) + } + rep.AlarmRaised = raised + return nil + } + cleared, err := s.ClearDestinationAlarm(ctx, rep.Destination, rep.RunID, "") + if err != nil { + return fmt.Errorf("auto-clear standing alarm for %q: %w", rep.Destination, err) + } + rep.AlarmCleared = cleared return nil } diff --git a/sync/verify_remote_test.go b/sync/verify_remote_test.go index 7cc479b..786e9ea 100644 --- a/sync/verify_remote_test.go +++ b/sync/verify_remote_test.go @@ -59,6 +59,72 @@ func TestVerifyRemoteMatchStampsVerified(t *testing.T) { } } +// TestVerifyRemoteMismatchLatchesAlarmThenClears: a mismatch latches a +// standing per-destination alarm (#157, F30), and a subsequent clean pass +// auto-clears it. Both transitions land in the runs_audit trail. +func TestVerifyRemoteMismatchLatchesAlarmThenClears(t *testing.T) { + f := setupContentAddressedFixture(t) + f.write(t, "a.txt", "alpha") + f.index(t) + if _, err := f.sync(t); err != nil { + t.Fatalf("sync: %v", err) + } + ctx := context.Background() + + if err := os.Setenv("RCLONE_FAKE_HASH_PREFIX", "tampered-"); err != nil { + t.Fatalf("set tamper: %v", err) + } + dirty, err := VerifyRemote(ctx, f.store, f.rcl, f.pair.Destination) + if err != nil { + t.Fatalf("VerifyRemote (tampered): %v", err) + } + if !dirty.AlarmRaised { + t.Fatalf("tampered pass did not raise an alarm: %+v", dirty) + } + alarm, err := f.store.GetDestinationAlarm(ctx, f.pair.Destination.Name) + if err != nil { + t.Fatalf("GetDestinationAlarm: %v", err) + } + if alarm.Kind != store.AlarmKindVerifyMismatch || alarm.RaisedRunID != dirty.RunID { + t.Fatalf("alarm = %+v, want verify-mismatch raised by run %d", alarm, dirty.RunID) + } + if n := countAuditTransition(t, f.store, dirty.RunID, store.TransitionAlarmRaise); n != 1 { + t.Fatalf("alarm-raise audit count = %d, want 1", n) + } + + if err := os.Unsetenv("RCLONE_FAKE_HASH_PREFIX"); err != nil { + t.Fatalf("unset tamper: %v", err) + } + clean, err := VerifyRemote(ctx, f.store, f.rcl, f.pair.Destination) + if err != nil { + t.Fatalf("VerifyRemote (clean): %v", err) + } + if !clean.Clean() || !clean.AlarmCleared { + t.Fatalf("clean pass did not auto-clear the alarm: %+v", clean) + } + if _, err := f.store.GetDestinationAlarm(ctx, f.pair.Destination.Name); !store.IsNotFound(err) { + t.Fatalf("alarm still latched after clean pass: %v", err) + } + if n := countAuditTransition(t, f.store, clean.RunID, store.TransitionAlarmClear); n != 1 { + t.Fatalf("alarm-clear audit count = %d, want 1", n) + } +} + +func countAuditTransition(t *testing.T, s *store.Store, runID int64, transition string) int { + t.Helper() + audits, err := s.ListRunAudit(context.Background(), runID) + if err != nil { + t.Fatalf("ListRunAudit(%d): %v", runID, err) + } + n := 0 + for _, a := range audits { + if a.Transition == transition { + n++ + } + } + return n +} + // TestVerifyRemoteMismatchIsLoudAndPreservesEvidence: a changed provider // checksum is reported per object, marks the run partial, and leaves // both the recorded fingerprint and the verification stamp untouched. diff --git a/syncproto/syncproto.go b/syncproto/syncproto.go index 7a006cd..f6618cf 100644 --- a/syncproto/syncproto.go +++ b/syncproto/syncproto.go @@ -67,6 +67,20 @@ const DispositionSupersede = "supersede" // declared content origin. const DispositionConflict = "conflict" +// DispositionContested — the path is frozen: a prior conflict at this +// path raised a contested latch on the receiver (see +// store.ContestedPath), and this initiator's bytes differ from the +// frozen winner. Rather than mint yet another `.squirrel-conflicts/` +// copy every cadence tick (the F27 ping-pong), the receiver refuses to +// re-supersede: no bytes move, no new row is written, and both existing +// versions stay preserved (the winner live at the path, the loser under +// `.squirrel-conflicts/`). The initiator surfaces the refusal to its +// operator; only an explicit `squirrel conflicts resolve` clears the +// latch and lets syncs flow again. The disposition is version-gated — +// only sent to an initiator that negotiated ProtocolVersionContested; +// an older initiator falls back to the legacy `conflict` disposition. +const DispositionContested = "contested" + // DispositionCopyFromExisting — receiver has no live row at this path // but holds the requested blake3 at a different path in the same volume. // Instead of forcing the initiator to re-transfer the bytes over the @@ -105,6 +119,16 @@ const ProtocolVersionFlat = 1 // scope reduction on the input, not a new disposition. const ProtocolVersionMerkleWalk = 2 +// ProtocolVersionContested adds the contested-freeze disposition (#158, +// F27): once a path is frozen, the receiver answers a divergent +// re-assertion with DispositionContested instead of minting another +// `.squirrel-conflicts/` copy. The version gate keeps an older initiator +// safe — it never learns the `contested` disposition it can't interpret, +// so the receiver falls back to the legacy `conflict` for that peer. The +// folder walk is unchanged from v2; this only widens the /plan verdict +// set for initiators that opt in. +const ProtocolVersionContested = 3 + // BeginRequest opens a peer-sync session. type BeginRequest struct { // Volume is the volume name (matched against the receiver's @@ -263,6 +287,15 @@ type PlanResponse struct { // has already pre-staged the loser under .squirrel-conflicts/ and // the initiator's bytes are still in scope for the rclone transfer. Conflicts []ConflictDetail `json:"conflicts,omitempty"` + // Contested captures the paths whose disposition was "contested": + // frozen by a prior conflict, so this initiator's divergent bytes + // were refused rather than superseding (no new copy minted). Each + // entry carries the frozen winner + preserved-loser digests and the + // receiver-relative location the loser is preserved at, so the + // initiator can raise its own local contested marker and its CLI can + // point the operator at both versions. Only populated for an + // initiator that negotiated ProtocolVersionContested. + Contested []ContestedDetail `json:"contested,omitempty"` } // PlanDisposition is the receiver's verdict on one path. @@ -298,6 +331,21 @@ type ConflictDetail struct { PreservedAtPath string `json:"preserved_at_path,omitempty"` } +// ContestedDetail names a path whose disposition was "contested" and +// surfaces the two frozen versions plus where the loser is preserved, so +// the initiator's CLI and its local contested marker describe the same +// state the receiver already froze. LiveBlake3Hex is the frozen winner +// (live at the path on the receiver); PreservedBlake3Hex is the loser +// held under .squirrel-conflicts/; PreservedAtPath is that loser's +// receiver-relative path. The initiator's own refused digest is the +// entry it sent, so it isn't echoed here. +type ContestedDetail struct { + Path string `json:"path"` + LiveBlake3Hex string `json:"live_blake3,omitempty"` + PreservedBlake3Hex string `json:"preserved_blake3,omitempty"` + PreservedAtPath string `json:"preserved_at_path,omitempty"` +} + // VerifyRequest tells the receiver "rclone said it finished, please // re-hash the affected paths now". type VerifyRequest struct { diff --git a/tui/dashboard.go b/tui/dashboard.go index e9455e7..d6eb60b 100644 --- a/tui/dashboard.go +++ b/tui/dashboard.go @@ -10,20 +10,25 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/status" "github.com/mbertschler/squirrel/store" ) // dashboardModel surfaces squirrel's live state in one screen: // // - the local agent's health (one-line probe of /v1/health) +// - standing per-destination alarms // - runs currently in-flight (kind, volume, destination, elapsed) -// - per-volume health (name, path, last-index, last-sync) +// - the per-(volume × target) coverage + durability grid (the "am I +// safe?" panel, from the shared status query layer) // - the most recent terminal runs // // Data is pulled on each tickMsg via a single SQL pass plus one HTTP probe; // both run as Bubble Tea commands so the UI never blocks on I/O. type dashboardModel struct { store *store.Store + cfg *config.Config client *agentClient width, height int @@ -32,8 +37,22 @@ type dashboardModel struct { loaded bool loadErr error agentStatus agentStatus + + // readinessByVol caches the last computed offload-readiness tally per + // volume name. The coverage/durability grid rebuilds every one-second + // tick with readiness skipped (its dominant cost — a whole-index gate + // pass); the tally is refreshed on the slower readinessRefreshTicks + // cadence and overlaid onto the grid at render time, so "N offloadable + // now" stays visible without paying for it every tick. + readinessByVol map[string]status.OffloadReadiness + ticksSinceReadiness int } +// readinessRefreshTicks is how many one-second ticks pass between +// offload-readiness recomputations. Coverage and durability still refresh +// every tick; only the expensive gate-pass tally is throttled. +const readinessRefreshTicks = 15 + // dashboardData is the snapshot rendered by the dashboard. Built fresh on // each tick — there is no incremental update path, since the queries are // trivial and the resulting struct is small. @@ -42,15 +61,29 @@ type dashboardData struct { volumes []store.Volume activeRuns []store.Run recentRuns []store.Run - // latestByVol[volID][kind] is the most recent terminal-status run for - // that (volume, kind) pair. Used to fill the "last index" / "last sync" - // columns of the volumes table. - latestByVol map[int64]map[string]store.Run + // coverage is the per-(volume × target) sync-coverage and durability + // grid from the shared status query layer — the same facts and + // severities `squirrel status` prints. Empty when no config is loaded + // (the grid needs sync_to / offload_requires / cadences to render). + coverage status.Report + // alarms are the standing per-destination alarms (#157, F30). A verify + // mismatch latches one until cleared; the dashboard shows them so the + // trust surface answers "am I safe?" with a red panel, not silence. + alarms []store.DestinationAlarm + // contested are the standing contested-path freezes (#158, F27). A + // peer-sync conflict latches one until an operator resolves it; the + // dashboard shows them as a badge on *this* machine — the losing edge, + // not only the hub — so a silently diverging file surfaces. + contested []store.ContestedPath } type dashboardDataMsg struct { data dashboardData err error + // readinessFresh is true when this fetch recomputed the offload tally + // (a slow-cadence tick), so the receiver should refresh its cache from + // data.coverage; a fast tick leaves it false and the cache stands. + readinessFresh bool } type agentStatus struct { @@ -61,8 +94,8 @@ type agentStatus struct { type agentStatusMsg agentStatus -func newDashboardModel(s *store.Store) *dashboardModel { - return &dashboardModel{store: s} +func newDashboardModel(s *store.Store, cfg *config.Config) *dashboardModel { + return &dashboardModel{store: s, cfg: cfg} } // attachClient is called by the root model after construction so the @@ -71,7 +104,10 @@ func newDashboardModel(s *store.Store) *dashboardModel { func (m *dashboardModel) attachClient(c *agentClient) { m.client = c } func (m *dashboardModel) Init() tea.Cmd { - return tea.Batch(m.fetchData(), m.probeAgent()) + // Recompute readiness on (re)activation so the tally is current the + // moment the user opens the dashboard, then throttle on the tick path. + m.ticksSinceReadiness = 0 + return tea.Batch(m.fetchData(true), m.probeAgent()) } func (m *dashboardModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { @@ -80,12 +116,20 @@ func (m *dashboardModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.width, m.height = msg.Width, msg.Height return m, nil case tickMsg: - return m, tea.Batch(m.fetchData(), m.probeAgent()) + m.ticksSinceReadiness++ + refresh := m.ticksSinceReadiness >= readinessRefreshTicks + if refresh { + m.ticksSinceReadiness = 0 + } + return m, tea.Batch(m.fetchData(refresh), m.probeAgent()) case dashboardDataMsg: m.loaded = true m.loadErr = msg.err if msg.err == nil { m.data = msg.data + if msg.readinessFresh { + m.cacheReadiness(msg.data.coverage) + } } return m, nil case agentStatusMsg: @@ -104,13 +148,80 @@ func (m *dashboardModel) View() string { } sections := []string{ m.renderAgentBlock(), + } + if alarms := m.renderAlarms(); alarms != "" { + sections = append(sections, alarms) + } + if contested := m.renderContested(); contested != "" { + sections = append(sections, contested) + } + sections = append(sections, m.renderActiveRuns(), - m.renderVolumes(), + m.renderCoverage(), m.renderRecentRuns(), - } + ) return strings.Join(sections, "\n\n") } +// renderAlarms shows the standing per-destination alarms (#157, F30) high +// on the dashboard, right below agent health, because a latched verify +// mismatch is exactly the "am I safe?" answer the trust surface must not +// bury. Returns "" when nothing is in alarm so the section is absent on a +// healthy install rather than showing an empty green box. +func (m *dashboardModel) renderAlarms() string { + if len(m.data.alarms) == 0 { + return "" + } + header := styleErr.Render(fmt.Sprintf("Alarms (%d)", len(m.data.alarms))) + rows := [][]string{{"DESTINATION", "KIND", "SINCE", "RUN", "DETAIL"}} + for _, a := range m.data.alarms { + rows = append(rows, []string{ + a.Destination, + a.Kind, + whenAgo(sql.NullInt64{Int64: a.RaisedAtNs, Valid: true}, m.data.now), + fmt.Sprintf("#%d", a.RaisedRunID), + a.Detail, + }) + } + colours := []lipgloss.Color{colourFailure, "", "", "", ""} + return header + "\n" + renderTable(rows, colours) +} + +// renderContested shows standing contested-path freezes (#158, F27) as a +// badge just below the alarms, because a divergent edit frozen on this +// machine is part of the "am I safe?" answer the trust surface must not +// bury. Returns "" when nothing is frozen so the section is absent on a +// healthy install. The count in the header is the badge the losing edge +// needs — the hub is no longer the only place the conflict is visible. +func (m *dashboardModel) renderContested() string { + if len(m.data.contested) == 0 { + return "" + } + header := styleErr.Render(fmt.Sprintf("Contested paths (%d)", len(m.data.contested))) + rows := [][]string{{"VOLUME", "PATH", "PRESERVED AT", "SINCE"}} + for _, c := range m.data.contested { + rows = append(rows, []string{ + m.volumeName(sql.NullInt64{Int64: c.VolumeID, Valid: true}), + c.Path, + contestedPreservedCell(c.PreservedAtPath), + whenAgo(sql.NullInt64{Int64: c.RaisedAtNs, Valid: true}, m.data.now), + }) + } + colours := []lipgloss.Color{colourFailure, "", "", ""} + hint := styleMuted.Render("resolve with `squirrel conflicts resolve `") + return header + "\n" + renderTable(rows, colours) + "\n" + hint +} + +// contestedPreservedCell renders the preserved-loser location, or a dash +// when the freeze record carries none (an initiator that learned of the +// freeze without a preserved path). +func contestedPreservedCell(p string) string { + if p == "" { + return "—" + } + return p +} + func (m *dashboardModel) renderAgentBlock() string { header := styleHeader.Render("Agent") var body string @@ -150,21 +261,92 @@ func (m *dashboardModel) renderActiveRuns() string { return header + "\n" + renderTable(rows, []lipgloss.Color{"", "", "", "", colourRunning}) } -func (m *dashboardModel) renderVolumes() string { - header := styleHeader.Render(fmt.Sprintf("Volumes (%d)", len(m.data.volumes))) - if len(m.data.volumes) == 0 { - return header + "\n" + styleMuted.Render("no volumes configured") +// renderCoverage is the "am I safe?" panel: per volume, the per-target +// sync-coverage and durability grid from the shared status layer, replacing +// the old single LAST SYNC cell that hid a week-behind target behind a +// fresh ✓ (friction-log F16/F17/F23). Each volume gets a header line +// (name, path, index freshness, offloadable total) coloured by its worst +// level, then a target sub-table with the STATE and DURABLE cells coloured +// per target. +func (m *dashboardModel) renderCoverage() string { + vols := m.data.coverage.Volumes + header := styleHeader.Render(fmt.Sprintf("Coverage (%d)", len(vols))) + if len(vols) == 0 { + hint := "no volumes configured" + if m.cfg == nil { + hint = "no config loaded — coverage needs sync_to / offload_requires to render" + } + return header + "\n" + styleMuted.Render(hint) } - rows := [][]string{{"NAME", "PATH", "LAST INDEX", "LAST SYNC"}} - for _, v := range m.data.volumes { + blocks := []string{header} + for _, v := range vols { + blocks = append(blocks, m.renderVolumeCoverage(v)) + } + return strings.Join(blocks, "\n\n") +} + +// renderVolumeCoverage renders one volume's coverage block. The +// offload-readiness figure comes from the throttled cache (the grid itself +// rebuilds every tick with readiness skipped), falling back to the volume's +// own tally when the cache has no entry yet. +func (m *dashboardModel) renderVolumeCoverage(v status.VolumeStatus) string { + dot := lipgloss.NewStyle().Foreground(levelColour(v.Level())).Render("●") + title := fmt.Sprintf("%s %s %s", dot, v.Name, styleMuted.Render(v.Path)) + offload := v.Offload + if cached, ok := m.readinessByVol[v.Name]; ok { + offload = cached + } + meta := styleMuted.Render(fmt.Sprintf("index %s · %s", + status.IndexLabel(v), status.OffloadLabel(offload))) + if len(v.Targets) == 0 { + return title + "\n" + meta + "\n" + styleMuted.Render(" no targets configured") + } + rows := [][]string{{"TARGET", "ROLE", "LAST SYNC", "STATE", "DURABLE", "METHOD", "EVIDENCE"}} + for _, t := range v.Targets { rows = append(rows, []string{ - v.Name, - v.Path, - m.formatLast(v.ID, store.RunKindIndex), - m.formatLast(v.ID, store.RunKindSync), + t.Name, status.RoleLabel(t), status.LastSyncLabel(t), status.StateLabel(t), + status.DurableLabel(t), status.MethodLabel(t), status.EvidenceLabel(t), }) } - return header + "\n" + renderTable(rows, nil) + tbl := renderTableColoured(rows, nil, coverageCellColour(v.Targets)) + return title + "\n" + meta + "\n" + tbl +} + +// coverageCellColour paints the STATE column by each target's coverage +// level and the DURABLE column by its durability level, so the two "am I +// safe?" dimensions read at a glance without decoding the words. +func coverageCellColour(targets []status.TargetStatus) func(rowIdx, colIdx int) lipgloss.Color { + const stateCol, durableCol = 3, 4 + return func(rowIdx, colIdx int) lipgloss.Color { + if rowIdx == 0 || rowIdx > len(targets) { + return "" + } + t := targets[rowIdx-1] + switch colIdx { + case stateCol: + return levelColour(t.SyncLevel) + case durableCol: + if t.Durability != nil { + return levelColour(t.Durability.Level) + } + } + return "" + } +} + +// levelColour maps a status level onto the dashboard palette. Neutral gets +// no colour (the default foreground) so informational cells don't shout. +func levelColour(l status.Level) lipgloss.Color { + switch l { + case status.LevelOK: + return colourSuccess + case status.LevelWarn: + return colourWarning + case status.LevelCritical: + return colourFailure + default: + return "" + } } func (m *dashboardModel) renderRecentRuns() string { @@ -206,28 +388,29 @@ func (m *dashboardModel) volumeName(id sql.NullInt64) string { return fmt.Sprintf("vol#%d", id.Int64) } -func (m *dashboardModel) formatLast(volID int64, kind string) string { - byKind := m.data.latestByVol[volID] - if byKind == nil { - return styleMuted.Render("—") - } - r, ok := byKind[kind] - if !ok { - return styleMuted.Render("—") - } - ago := whenAgo(r.EndedAtNs, m.data.now) - statusGlyph := lipgloss.NewStyle().Foreground(statusColour(r.Status)).Render(glyphForStatus(r.Status)) - return fmt.Sprintf("%s %s", ago, statusGlyph) -} - -func (m *dashboardModel) fetchData() tea.Cmd { +// fetchData loads the dashboard snapshot. withReadiness selects whether the +// coverage build pays for the offload-readiness tally: false on the +// one-second tick (the grid still refreshes; the cached tally is overlaid +// at render), true on the slow cadence and on activation. +func (m *dashboardModel) fetchData(withReadiness bool) tea.Cmd { return func() tea.Msg { // Use a tight per-fetch deadline so a stuck DB doesn't freeze the UI. ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - data, err := loadDashboardData(ctx, m.store) - return dashboardDataMsg{data: data, err: err} + data, err := loadDashboardData(ctx, m.store, m.cfg, !withReadiness) + return dashboardDataMsg{data: data, err: err, readinessFresh: withReadiness && err == nil} + } +} + +// cacheReadiness refreshes the per-volume offload tally from a report that +// was built with readiness computed. Fast-tick reports carry only the +// policy flag (zero counts) and must not overwrite this cache. +func (m *dashboardModel) cacheReadiness(rep status.Report) { + cache := make(map[string]status.OffloadReadiness, len(rep.Volumes)) + for _, v := range rep.Volumes { + cache[v.Name] = v.Offload } + m.readinessByVol = cache } func (m *dashboardModel) probeAgent() tea.Cmd { @@ -246,13 +429,16 @@ func (m *dashboardModel) probeAgent() tea.Cmd { } } -// loadDashboardData runs the SQL queries that back the dashboard. The +// loadDashboardData runs the queries that back the dashboard. The // recent-runs bucket comes from a bounded ListRuns scan (200 is plenty -// for "what happened today"); the per-(volume,kind) "last successful" -// table comes from its own helper that scans every run, so volumes -// whose last index sits beyond the recent window still surface -// correctly. -func loadDashboardData(ctx context.Context, s *store.Store) (dashboardData, error) { +// for "what happened today"); the coverage grid comes from the shared +// status query layer, which scans per (volume × target) so a target beyond +// the recent-runs window still surfaces correctly. The coverage build is +// skipped when no config is loaded — it needs sync_to / offload_requires / +// cadences — leaving the grid to render its own "no config" hint. +// skipReadiness omits the expensive offload-readiness tally (see fetchData +// and readinessRefreshTicks). +func loadDashboardData(ctx context.Context, s *store.Store, cfg *config.Config, skipReadiness bool) (dashboardData, error) { now := time.Now() vols, err := s.ListVolumes(ctx) if err != nil { @@ -262,9 +448,19 @@ func loadDashboardData(ctx context.Context, s *store.Store) (dashboardData, erro if err != nil { return dashboardData{}, fmt.Errorf("list runs: %w", err) } - latestByVol, err := s.LatestSuccessfulRunsByVolumeAndKind(ctx) + alarms, err := s.ListDestinationAlarms(ctx) + if err != nil { + return dashboardData{}, fmt.Errorf("list alarms: %w", err) + } + contested, err := s.ListContestedPaths(ctx) if err != nil { - return dashboardData{}, fmt.Errorf("latest by volume: %w", err) + return dashboardData{}, fmt.Errorf("list contested paths: %w", err) + } + var coverage status.Report + if cfg != nil { + if coverage, err = status.BuildWithOptions(ctx, s, cfg, status.Options{SkipOffloadReadiness: skipReadiness}); err != nil { + return dashboardData{}, fmt.Errorf("build coverage: %w", err) + } } var active, recent []store.Run for _, r := range runs { @@ -277,10 +473,12 @@ func loadDashboardData(ctx context.Context, s *store.Store) (dashboardData, erro } } return dashboardData{ - now: now, - volumes: vols, - activeRuns: active, - recentRuns: recent, - latestByVol: latestByVol, + now: now, + volumes: vols, + activeRuns: active, + recentRuns: recent, + coverage: coverage, + alarms: alarms, + contested: contested, }, nil } diff --git a/tui/dashboard_test.go b/tui/dashboard_test.go index c6f78b9..29bdea2 100644 --- a/tui/dashboard_test.go +++ b/tui/dashboard_test.go @@ -36,8 +36,6 @@ func TestLoadDashboardDataPartitionsRuns(t *testing.T) { // a successful sync run. We expect: // - activeRuns: the one running sync // - recentRuns: every terminal run, newest first, capped at 10 - // - latestByVol[vol.ID]: index = most recent success, sync = the - // successful one (failed audit doesn't establish "last audit") finishedIdx1, _ := s.BeginIndexRun(ctx, store.RunKindIndex, vol.ID, false) if err := s.FinishRun(ctx, finishedIdx1, store.RunStatusSuccess, "", 100); err != nil { t.Fatalf("FinishRun: %v", err) @@ -57,7 +55,10 @@ func TestLoadDashboardDataPartitionsRuns(t *testing.T) { t.Fatalf("FinishRun: %v", err) } - data, err := loadDashboardData(ctx, s) + // cfg is nil here: this test pins the run-partitioning behaviour, which + // is independent of the config-driven coverage grid (that is exercised + // against the shared status layer in the status package's own tests). + data, err := loadDashboardData(ctx, s, nil, false) if err != nil { t.Fatalf("loadDashboardData: %v", err) } @@ -80,18 +81,8 @@ func TestLoadDashboardDataPartitionsRuns(t *testing.T) { t.Errorf("recentRuns[0] = %d, want %d (most recent)", data.recentRuns[0].ID, successSync) } - byKind := data.latestByVol[vol.ID] - if byKind == nil { - t.Fatalf("latestByVol missing entry for vol#%d", vol.ID) - } - if byKind[store.RunKindIndex].ID != finishedIdx2 { - t.Errorf("latest index = %d, want %d (the partial-but-newer one)", - byKind[store.RunKindIndex].ID, finishedIdx2) - } - if byKind[store.RunKindSync].ID != successSync { - t.Errorf("latest sync = %d, want %d", byKind[store.RunKindSync].ID, successSync) - } - if _, present := byKind[store.RunKindAudit]; present { - t.Errorf("latestByVol must not include the failed audit run") + // With no config, the coverage grid renders nothing. + if len(data.coverage.Volumes) != 0 { + t.Errorf("coverage.Volumes = %d, want 0 without config", len(data.coverage.Volumes)) } } diff --git a/tui/format.go b/tui/format.go index 1dd0367..b9a3bd4 100644 --- a/tui/format.go +++ b/tui/format.go @@ -99,6 +99,10 @@ func glyphForStatus(status string) string { return "~" case "running": return "●" + case "refused": + return "⊘" + case "aborted": + return "⊗" default: return "·" } diff --git a/tui/runs.go b/tui/runs.go index 43eabdd..73548b1 100644 --- a/tui/runs.go +++ b/tui/runs.go @@ -28,9 +28,14 @@ type runsModel struct { rows []store.Run volumesByID map[int64]store.Volume filterKind string // empty = no filter + fold bool // collapse consecutive no-op rows (F19) + foldedCount int // no-op rows hidden by the current fold loaded bool loadErr error selectedDetail *store.Run + // displayRuns aligns 1:1 with the table's rows: the run behind each + // row, or nil for a fold-marker row (which has no detail view). + displayRuns []*store.Run } type runsDataMsg struct { @@ -55,7 +60,7 @@ func newRunsModel(s *store.Store) *runsModel { Background(colourAccent). Bold(false) t.SetStyles(style) - return &runsModel{store: s, table: t} + return &runsModel{store: s, table: t, fold: true} } func (m *runsModel) Init() tea.Cmd { return m.fetch() } @@ -102,13 +107,11 @@ func (m *runsModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } switch msg.String() { case "enter": - if len(m.rows) == 0 { - return m, nil - } + // A fold-marker row maps to nil in displayRuns and has no + // detail view; ignore enter on it. idx := m.table.Cursor() - filtered := m.filteredRows() - if idx >= 0 && idx < len(filtered) { - r := filtered[idx] + if idx >= 0 && idx < len(m.displayRuns) && m.displayRuns[idx] != nil { + r := *m.displayRuns[idx] m.selectedDetail = &r } return m, nil @@ -127,6 +130,10 @@ func (m *runsModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case "A": m.setFilter("") return m, nil + case "f": + m.fold = !m.fold + m.applyFilter() + return m, nil } } var cmd tea.Cmd @@ -144,9 +151,18 @@ func (m *runsModel) View() string { if m.selectedDetail != nil { return m.renderDetail(*m.selectedDetail) } - header := styleHeader.Render("Runs") + " " + m.renderFilterChips() + header := styleHeader.Render("Runs") + " " + m.renderFilterChips() + m.renderFoldChip() return header + "\n" + m.table.View() + "\n" + - styleMuted.Render("i index · s sync · a audit · r restore · A all · enter detail") + styleMuted.Render("i index · s sync · a audit · r restore · A all · f fold · enter detail") +} + +// renderFoldChip notes how many no-op rows the fold is hiding, so a folded +// view never looks like runs simply went missing. +func (m *runsModel) renderFoldChip() string { + if !m.fold || m.foldedCount == 0 { + return "" + } + return " " + styleMuted.Render(fmt.Sprintf("(%d no-op rows folded — f to expand)", m.foldedCount)) } // renderFilterChips renders the active-filter pill, or muted text when no @@ -238,23 +254,84 @@ func (m *runsModel) applyFilter() { filtered := m.filteredRows() now := time.Now() rows := make([]table.Row, 0, len(filtered)) - for _, r := range filtered { - rows = append(rows, table.Row{ - fmt.Sprintf("#%d", r.ID), - r.Kind, - m.volumeName(r.VolumeID), - nullStr(r.Destination), - r.Status, - whenAgo(r.EndedAtNs, now), - runDuration(r), - fmt.Sprintf("%d", r.FileCount), - shallowGlyph(r.Shallow), - }) + display := make([]*store.Run, 0, len(filtered)) + m.foldedCount = 0 + for i := 0; i < len(filtered); { + // Collapse a maximal run of consecutive no-op rows into one marker, + // but only when there is more than one — a lone no-op isn't worth + // a fold line (F19). + if m.fold && runIsNoOp(filtered[i]) { + j := i + for j < len(filtered) && runIsNoOp(filtered[j]) { + j++ + } + if n := j - i; n > 1 { + rows = append(rows, foldMarkerRow(n)) + display = append(display, nil) + m.foldedCount += n + i = j + continue + } + } + r := filtered[i] + rows = append(rows, m.runTableRow(r, now)) + display = append(display, &r) + i++ } + m.displayRuns = display m.table.SetRows(rows) m.resizeColumns() } +// runTableRow renders one run as a table row. +func (m *runsModel) runTableRow(r store.Run, now time.Time) table.Row { + return table.Row{ + fmt.Sprintf("#%d", r.ID), + r.Kind, + m.volumeName(r.VolumeID), + destinationCell(r), + r.Status, + whenAgo(r.EndedAtNs, now), + runDuration(r), + fmt.Sprintf("%d", r.FileCount), + shallowGlyph(r.Shallow), + } +} + +// foldMarkerRow is the placeholder shown in place of a collapsed block of +// no-op rows. It carries the same column count as a real row so the table +// layout stays aligned. +func foldMarkerRow(n int) table.Row { + return table.Row{"⋯", "", fmt.Sprintf("%d no-op runs folded", n), "", "", "", "", "", ""} +} + +// runIsNoOp reports whether a run is routine no-op noise — a clean success +// that touched no files. Mirrors the CLI's `runs --changes` fold rule; a +// peer-sync no-op has file_count 0, so those collapse, while bucket/index +// no-ops (file_count counts files considered, not moved) stay visible. +func runIsNoOp(r store.Run) bool { + return r.Status == store.RunStatusSuccess && r.FileCount == 0 +} + +// destinationCell renders the DESTINATION column, prefixing a peer-sync row +// with its initiation-direction arrow so inbound and outbound are +// distinguishable (F18): "→" outbound (we pushed), "←" inbound (peer +// pushed). Only the initiator records runs.shallow, so a set flag marks the +// pushing side (see store.Run.Shallow). +func destinationCell(r store.Run) string { + if !r.Destination.Valid { + return nullStr(r.Destination) + } + if !r.PeerNodeID.Valid { + return r.Destination.String + } + arrow := "→" + if !r.Shallow.Valid { + arrow = "←" + } + return arrow + " " + r.Destination.String +} + func (m *runsModel) resizeColumns() { cols := []table.Column{ {Title: "ID", Width: 6}, diff --git a/tui/runs_polish_test.go b/tui/runs_polish_test.go new file mode 100644 index 0000000..09792d2 --- /dev/null +++ b/tui/runs_polish_test.go @@ -0,0 +1,47 @@ +package tui + +import ( + "database/sql" + "testing" + + "github.com/mbertschler/squirrel/store" +) + +// TestDestinationCell covers the F18 direction arrow in the runs table: +// bucket rows render the plain name, peer rows prefix the initiation arrow +// (→ outbound / ← inbound, keyed off runs.shallow). +func TestDestinationCell(t *testing.T) { + bucket := store.Run{Destination: sql.NullString{String: "s3archive", Valid: true}} + if got := destinationCell(bucket); got != "s3archive" { + t.Errorf("bucket cell = %q, want %q", got, "s3archive") + } + outbound := store.Run{ + Destination: sql.NullString{String: "htpc", Valid: true}, + PeerNodeID: sql.NullInt64{Int64: 3, Valid: true}, + Shallow: sql.NullBool{Valid: true}, + } + if got := destinationCell(outbound); got != "→ htpc" { + t.Errorf("outbound cell = %q, want %q", got, "→ htpc") + } + inbound := store.Run{ + Destination: sql.NullString{String: "laptop", Valid: true}, + PeerNodeID: sql.NullInt64{Int64: 4, Valid: true}, + } + if got := destinationCell(inbound); got != "← laptop" { + t.Errorf("inbound cell = %q, want %q", got, "← laptop") + } +} + +// TestRunIsNoOp covers the F19 fold rule: a clean success that touched no +// files is a no-op; anything else is not. +func TestRunIsNoOp(t *testing.T) { + if !runIsNoOp(store.Run{Status: store.RunStatusSuccess, FileCount: 0}) { + t.Error("clean 0-file success should be a no-op") + } + if runIsNoOp(store.Run{Status: store.RunStatusSuccess, FileCount: 3}) { + t.Error("success that touched files is not a no-op") + } + if runIsNoOp(store.Run{Status: store.RunStatusFailed}) { + t.Error("failed run is not a no-op") + } +} diff --git a/tui/styles.go b/tui/styles.go index 459f6a0..11e2d66 100644 --- a/tui/styles.go +++ b/tui/styles.go @@ -44,6 +44,14 @@ func statusColour(status string) lipgloss.Color { return colourWarning case "running": return colourRunning + case "refused": + // A refusal is a fail-closed safety gate; it must read as red so a + // month-dead backup disk produces visible red, not a muted dot. + return colourFailure + case "aborted": + // A reaped run never completed but did not fail; amber keeps it out + // of the green "you may close the laptop" set without crying failure. + return colourWarning default: return colourMuted } diff --git a/tui/tui.go b/tui/tui.go index b11d895..0ccfee6 100644 --- a/tui/tui.go +++ b/tui/tui.go @@ -90,7 +90,7 @@ type rootModel struct { func newRootModel(s *store.Store, cfg *config.Config) *rootModel { client := newAgentClient(cfg) - dash := newDashboardModel(s) + dash := newDashboardModel(s, cfg) dash.attachClient(client) return &rootModel{ active: screenDashboard,