From 8a5cabd2686f788cd9139c78151807e8bca89352 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:22:16 +0000 Subject: [PATCH 01/14] runs: every failure and refusal becomes a run row or a standing state (#157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit trail is the trust contract, but several failure classes never reached it. This makes each one visible. Schema v24→v25 (store/migrations.go): - runs.status widens to admit 'refused' (a preflight safety gate declined before the transfer began) and 'aborted' (a 'running' row reaped after the owning process died). Both terminal; neither a success. - new STRICT destination_alarms table: the per-destination standing-alarm latch. Snapshot regenerated (go test ./store -update-schema). F6/F15 — rclone failures were undiagnosable: capture rclone's non-JSON stderr (bounded, 4 KiB) plus 'fatal'/'critical' JSON levels, fold the tail into the returned error (so the scheduler log and run row carry the real reason), and synthesise a FailedFile so the run row's error is never blank. RunResult.DisplayErrors() fixes the errors=0-on-status=failed summary contradiction. F26 — preflight refusals (marker, kopia-connect, layout guard) now mint a terminal 'refused' run row via ErrRefused instead of vanishing into a run_id=0 error. A refused run consumes the cadence window (LatestFinishedRun counts it) so the scheduler backs off a dead destination instead of re-minting an identical refusal every tick. F14 — agent startup reaps its own orphaned 'running' runs to 'aborted' (store.AbortRunningRuns), loudly logged; aborted runs are excluded from LatestFinishedRun so a crashed run is re-attempted. F30 — a verify mismatch latches a per-destination alarm (destination_alarms) that survives in `squirrel runs`, the TUI dashboard, and verify output until cleared. Both clear paths implemented: an auto-clear on a subsequent clean `squirrel verify`, and an explicit `squirrel verify ack `. Every raise and clear is recorded in runs_audit, so deleting the derived latch loses no history. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/agent.go | 22 +++++ cmd/squirrel/runs.go | 28 +++++- cmd/squirrel/sync.go | 4 +- cmd/squirrel/verify.go | 21 ++++- cmd/squirrel/verify_ack.go | 72 ++++++++++++++ store/abort_runs_test.go | 113 ++++++++++++++++++++++ store/destination_alarms.go | 150 ++++++++++++++++++++++++++++++ store/destination_alarms_test.go | 115 +++++++++++++++++++++++ store/destination_run_ids_test.go | 27 ++++++ store/migrations.go | 116 ++++++++++++++++++++++- store/runs.go | 119 ++++++++++++++++++++---- store/runs_audit.go | 14 +++ store/schema.sql | 12 ++- sync/content_addressed.go | 2 +- sync/content_addressed_test.go | 8 +- sync/handler.go | 17 +++- sync/kopia.go | 2 +- sync/kopia_test.go | 7 +- sync/packed.go | 2 +- sync/packed_test.go | 14 ++- sync/rclone.go | 146 +++++++++++++++++++++++------ sync/rclone_test.go | 56 +++++++++++ sync/sync.go | 72 ++++++++++++-- sync/sync_test.go | 19 +++- sync/verify_remote.go | 41 ++++++++ sync/verify_remote_test.go | 66 +++++++++++++ tui/dashboard.go | 40 +++++++- tui/format.go | 4 + tui/styles.go | 8 ++ 29 files changed, 1235 insertions(+), 82 deletions(-) create mode 100644 cmd/squirrel/verify_ack.go create mode 100644 store/abort_runs_test.go create mode 100644 store/destination_alarms.go create mode 100644 store/destination_alarms_test.go diff --git a/cmd/squirrel/agent.go b/cmd/squirrel/agent.go index d554c77..ad18923 100644 --- a/cmd/squirrel/agent.go +++ b/cmd/squirrel/agent.go @@ -51,6 +51,9 @@ func runAgent(cmd *cobra.Command) error { defer s.Close() logger := slog.New(slog.NewTextHandler(cmd.ErrOrStderr(), &slog.HandlerOptions{Level: slog.LevelInfo})) + if err := reapOrphanedRunsAtStartup(cmd.Context(), s, logger); err != nil { + return err + } rcl, err := resolveSchedulerRclone(cmd, cfg) if err != nil { return err @@ -85,6 +88,25 @@ func runAgent(cmd *cobra.Command) error { return srv.Serve(cmd.Context(), ln) } +// reapOrphanedRunsAtStartup transitions any 'running' run left behind by a +// previously-killed agent to 'aborted' before the schedulers start (#157, +// F14). A freshly started agent has kicked nothing yet, so every 'running' +// row necessarily predates it; leaving them would block the run gates +// forever and render as a live, elapsed-ticking banner in the TUI hours +// later. The reap is loud — one info line naming the reaped ids — because +// automatic recovery must never be invisible (ux-principle 5). +func reapOrphanedRunsAtStartup(ctx context.Context, s *store.Store, logger *slog.Logger) error { + ids, err := s.AbortRunningRuns(ctx, "reaped at agent startup: the agent that owned this run was killed mid-run") + if err != nil { + return fmt.Errorf("reap orphaned runs: %w", err) + } + if len(ids) > 0 { + logger.Warn("reaped orphaned runs", + "count", len(ids), "run_ids", ids, "status", store.RunStatusAborted) + } + return nil +} + // openAgentStore extends the standard resolveDBPath precedence with the // agent-specific override: --db > cfg.Agent.DB > cfg.DB > default. We // can't reuse openStore directly because it doesn't know about the diff --git a/cmd/squirrel/runs.go b/cmd/squirrel/runs.go index 0bbf08b..67e96a0 100644 --- a/cmd/squirrel/runs.go +++ b/cmd/squirrel/runs.go @@ -75,7 +75,33 @@ func runRuns(cmd *cobra.Command, volumeName string, limit int) error { if err != nil { return err } - return printRuns(cmd.OutOrStdout(), runs, volumes, nodes, conflicts) + if err := printRuns(cmd.OutOrStdout(), runs, volumes, nodes, conflicts); err != nil { + return err + } + alarms, err := s.ListDestinationAlarms(cmd.Context()) + if err != nil { + return fmt.Errorf("list destination alarms: %w", err) + } + printActiveAlarms(cmd.OutOrStdout(), alarms) + return nil +} + +// printActiveAlarms surfaces standing per-destination alarms beneath the +// runs listing (#157, F30). A verify mismatch latches an alarm that must +// stay visible until an operator clears it — a single old run row scrolls +// away, so the audit surface repeats the standing condition on every +// `squirrel runs` until it is resolved. +func printActiveAlarms(w io.Writer, alarms []store.DestinationAlarm) { + if len(alarms) == 0 { + return + } + fmt.Fprintf(w, "\n%d destination(s) in alarm — clear with a clean `squirrel verify` or `squirrel verify ack `:\n", + len(alarms)) + for _, a := range alarms { + fmt.Fprintf(w, " ALARM %s (%s) since %s, run %d: %s\n", + a.Destination, a.Kind, + formatStarted(a.RaisedAtNs), a.RaisedRunID, a.Detail) + } } // loadNodeNames builds an id→name map for the peer_node_id values diff --git a/cmd/squirrel/sync.go b/cmd/squirrel/sync.go index a5b24bd..9abcc6f 100644 --- a/cmd/squirrel/sync.go +++ b/cmd/squirrel/sync.go @@ -186,13 +186,13 @@ func printSyncReport(w io.Writer, rep sync.Report, runErr error) { // fresh uploads. fmt.Fprintf(w, "%s → %s status=%s objects=%d skipped=%d errors=%d bytes=%d entries=%d fingerprints=%d run=%d\n", rep.Volume, rep.Destination, rep.Status, - r.Transferred, r.Checked, r.Errors, r.Bytes, + r.Transferred, r.Checked, r.DisplayErrors(), r.Bytes, rep.Verification.Files, rep.Fingerprints, rep.RunID, ) default: fmt.Fprintf(w, "%s → %s status=%s transferred=%d checked=%d errors=%d bytes=%d run=%d\n", rep.Volume, rep.Destination, rep.Status, - r.Transferred, r.Checked, r.Errors, r.Bytes, rep.RunID, + r.Transferred, r.Checked, r.DisplayErrors(), r.Bytes, rep.RunID, ) } if rep.NodeReceiverRunID != 0 { diff --git a/cmd/squirrel/verify.go b/cmd/squirrel/verify.go index d696903..bb27e3f 100644 --- a/cmd/squirrel/verify.go +++ b/cmd/squirrel/verify.go @@ -22,7 +22,7 @@ import ( // recorded fingerprint left exactly as found so the operator inspects the // evidence. func newVerifyCmd() *cobra.Command { - return &cobra.Command{ + cmd := &cobra.Command{ Use: "verify []", Short: "Re-check recorded offsite objects and packs against their upload fingerprints", Args: cobra.MaximumNArgs(1), @@ -34,6 +34,8 @@ func newVerifyCmd() *cobra.Command { return runVerify(cmd, destName) }, } + cmd.AddCommand(newVerifyAckCmd()) + return cmd } func runVerify(cmd *cobra.Command, destName string) error { @@ -108,7 +110,8 @@ func verifiableLayout(layout string) bool { } // printVerifyReport renders one destination's pass: a loud stderr line -// per missing or mismatched object or pack, then the summary counters. +// per missing or mismatched object or pack, then the summary counters, +// then the standing-alarm transition this pass caused (#157, F30). func printVerifyReport(out, errOut io.Writer, rep sync.RemoteVerifyReport, runErr error) { printVerifyFailures(errOut, "object", rep.Destination, rep.Missing, rep.Mismatched) printVerifyFailures(errOut, "pack", rep.Destination, rep.PacksMissing, rep.PackMismatched) @@ -125,6 +128,20 @@ func printVerifyReport(out, errOut io.Writer, rep sync.RemoteVerifyReport, runEr len(rep.Mismatched), len(rep.Missing), rep.Unrecorded, rep.Packs, rep.PacksVerified, rep.PacksPopulated, rep.PacksPending, len(rep.PackMismatched), len(rep.PacksMissing)) + printVerifyAlarmTransition(out, errOut, rep) +} + +// printVerifyAlarmTransition surfaces the standing-alarm change this pass +// made: a raised alarm shouts on stderr with how to clear it, an +// auto-cleared alarm is a quiet confidence line on stdout. +func printVerifyAlarmTransition(out, errOut io.Writer, rep sync.RemoteVerifyReport) { + if rep.AlarmRaised { + fmt.Fprintf(errOut, "ALARM %s: verification failed — this destination stays in alarm until a clean `squirrel verify %s` clears it or you run `squirrel verify ack %s`\n", + rep.Destination, rep.Destination, rep.Destination) + } + if rep.AlarmCleared { + fmt.Fprintf(out, "verify %s: standing alarm cleared by this clean pass\n", rep.Destination) + } } // printVerifyFailures writes one stderr line per missing or mismatched diff --git a/cmd/squirrel/verify_ack.go b/cmd/squirrel/verify_ack.go new file mode 100644 index 0000000..1c87ed6 --- /dev/null +++ b/cmd/squirrel/verify_ack.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "os/user" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/store" +) + +// newVerifyAckCmd returns `squirrel verify ack `: the explicit +// operator path to clear a standing verify alarm (#157, F30). It is the +// deliberate counterpart to the automatic clear a clean `squirrel verify` +// performs — for when the operator has inspected the destination and +// restored or re-uploaded the affected artifacts out of band, and wants +// the alarm gone without waiting for (or being able to run) a full +// re-verification pass. The clear is recorded against the run that raised +// the alarm and tagged with the operator, so the audit trail shows who +// cleared it and when. +func newVerifyAckCmd() *cobra.Command { + return &cobra.Command{ + Use: "ack ", + Short: "Acknowledge and clear a standing verify alarm on a destination", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runVerifyAck(cmd, args[0]) + }, + } +} + +func runVerifyAck(cmd *cobra.Command, destName string) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + s, err := openStore(cmd, cfg) + if err != nil { + return err + } + defer s.Close() + + alarm, err := s.GetDestinationAlarm(cmd.Context(), destName) + if err != nil { + if store.IsNotFound(err) { + return fmt.Errorf("destination %q has no standing alarm to acknowledge", destName) + } + return fmt.Errorf("look up standing alarm for %q: %w", destName, err) + } + cleared, err := s.ClearDestinationAlarm(cmd.Context(), destName, alarm.RaisedRunID, ackOperator()) + if err != nil { + return err + } + if !cleared { + // Raced with an auto-clear (a clean verify) between the read and + // the write — the alarm is gone either way, which is the outcome + // the operator asked for. + return fmt.Errorf("destination %q has no standing alarm to acknowledge", destName) + } + fmt.Fprintf(cmd.OutOrStdout(), "verify %s: standing alarm acknowledged and cleared\n", destName) + return nil +} + +// ackOperator names who acked for the runs_audit operator column: the OS +// user when resolvable, else a generic label. Best-effort attribution for +// the audit trail, not an authorization check. +func ackOperator() string { + if u, err := user.Current(); err == nil && u.Username != "" { + return u.Username + } + return "operator" +} diff --git a/store/abort_runs_test.go b/store/abort_runs_test.go new file mode 100644 index 0000000..5a37aa0 --- /dev/null +++ b/store/abort_runs_test.go @@ -0,0 +1,113 @@ +package store + +import ( + "context" + "testing" +) + +// TestAbortRunningRuns reaps every in-flight run to 'aborted', leaves +// terminal rows untouched, is idempotent, and records an 'abort' +// transition per reaped row. +func TestAbortRunningRuns(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + vol, err := s.CreateVolume(ctx, "v", "/v") + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + + idx, err := s.BeginIndexRun(ctx, RunKindIndex, vol.ID, true) + if err != nil { + t.Fatalf("BeginIndexRun: %v", err) + } + syncing, err := s.BeginRun(ctx, RunKindSync, vol.ID, "offsite", false) + if err != nil { + t.Fatalf("BeginRun sync: %v", err) + } + done, err := s.BeginRun(ctx, RunKindSync, vol.ID, "other", false) + if err != nil { + t.Fatalf("BeginRun done: %v", err) + } + if err := s.FinishRun(ctx, done, RunStatusSuccess, "", 5); err != nil { + t.Fatalf("FinishRun done: %v", err) + } + + ids, err := s.AbortRunningRuns(ctx, "killed") + if err != nil { + t.Fatalf("AbortRunningRuns: %v", err) + } + if len(ids) != 2 || ids[0] != idx || ids[1] != syncing { + t.Fatalf("reaped ids = %v, want [%d %d]", ids, idx, syncing) + } + + for _, id := range []int64{idx, syncing} { + run, err := s.GetRun(ctx, id) + if err != nil { + t.Fatalf("GetRun(%d): %v", id, err) + } + if run.Status != RunStatusAborted { + t.Fatalf("run %d status = %q, want aborted", id, run.Status) + } + if !run.EndedAtNs.Valid { + t.Fatalf("aborted run %d has no ended_at_ns", id) + } + if run.Error.String != "killed" { + t.Fatalf("aborted run %d error = %q, want the reap reason", id, run.Error.String) + } + if n := countTransition(t, s, id, TransitionAbort); n != 1 { + t.Fatalf("run %d abort-transition count = %d, want 1", id, n) + } + } + + // The already-terminal success is untouched. + if run, _ := s.GetRun(ctx, done); run.Status != RunStatusSuccess { + t.Fatalf("finished run status = %q, want success", run.Status) + } + + // Idempotent: a second reap finds nothing. + again, err := s.AbortRunningRuns(ctx, "killed") + if err != nil { + t.Fatalf("second AbortRunningRuns: %v", err) + } + if len(again) != 0 { + t.Fatalf("second reap returned %v, want none", again) + } +} + +// TestLatestFinishedRunStatusScope pins the cadence-watermark policy: a +// 'refused' run consumes the cadence window (so the scheduler backs off a +// dead destination) while an 'aborted' run does not (so a crashed run is +// re-attempted). +func TestLatestFinishedRunStatusScope(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + vol, err := s.CreateVolume(ctx, "v", "/v") + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + + refused, err := s.BeginRun(ctx, RunKindSync, vol.ID, "dead", false) + if err != nil { + t.Fatalf("BeginRun refused: %v", err) + } + if err := s.FinishRun(ctx, refused, RunStatusRefused, "no marker", 0); err != nil { + t.Fatalf("FinishRun refused: %v", err) + } + got, err := s.LatestFinishedRun(ctx, RunKindSync, vol.ID, "dead") + if err != nil { + t.Fatalf("refused run should be a finished run: %v", err) + } + if got.ID != refused || got.Status != RunStatusRefused { + t.Fatalf("LatestFinishedRun = %+v, want the refused run", got) + } + + if _, err := s.BeginRun(ctx, RunKindSync, vol.ID, "crashed", false); err != nil { + t.Fatalf("BeginRun crashed: %v", err) + } + if _, err := s.AbortRunningRuns(ctx, "killed"); err != nil { + t.Fatalf("AbortRunningRuns: %v", err) + } + if _, err := s.LatestFinishedRun(ctx, RunKindSync, vol.ID, "crashed"); !IsNotFound(err) { + t.Fatalf("aborted run must not count as a finished run, got %v", err) + } +} diff --git a/store/destination_alarms.go b/store/destination_alarms.go new file mode 100644 index 0000000..97bf36f --- /dev/null +++ b/store/destination_alarms.go @@ -0,0 +1,150 @@ +package store + +import ( + "context" + "fmt" +) + +// AlarmKindVerifyMismatch is the destination_alarms.kind for a standing +// alarm raised when `squirrel verify` finds a recorded object or pack whose +// provider checksum no longer matches its upload fingerprint, or that has +// gone missing from the remote (#157, F30). It is the only alarm kind +// today; the column is free text so a future standing condition can share +// the table without a migration. +const AlarmKindVerifyMismatch = "verify-mismatch" + +// DestinationAlarm is one row of the destination_alarms latch: an abnormal +// condition on a destination that persists ("latches") until a subsequent +// clean check auto-clears it or an operator acks it. It is derived standing +// state — the permanent forensic record of every raise and clear lives in +// the runs and runs_audit tables — so the live latch can be cleared in +// place without losing history. RaisedRunID references the run that first +// detected the condition; RaisedAtNs is when that first detection landed +// (stable across repeated detections, so the surface can show "in alarm +// since"). +type DestinationAlarm struct { + Destination string + Kind string + Detail string + RaisedRunID int64 + RaisedAtNs int64 +} + +// RaiseDestinationAlarm latches an alarm on destination when none is +// already active. The raise is idempotent: an existing alarm keeps its +// original raised-at timestamp and run so a repeated detection does not +// reset "in alarm since" or append a duplicate audit row. An 'alarm-raise' +// runs_audit entry is written against raisedRunID (in the same transaction +// as the latch insert) only when a fresh latch is created, keeping the +// append-only trail to one row per distinct alarm episode. +func (s *Store) RaiseDestinationAlarm(ctx context.Context, destination, kind, detail string, raisedRunID int64) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin raise alarm %q: %w", destination, err) + } + defer func() { _ = tx.Rollback() }() + + atNs := NowNs() + res, err := tx.ExecContext(ctx, ` + INSERT INTO destination_alarms (destination, kind, detail, raised_run_id, raised_at_ns) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(destination) DO NOTHING + `, destination, kind, detail, raisedRunID, atNs) + if err != nil { + return fmt.Errorf("raise alarm %q: %w", destination, err) + } + inserted, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("raise alarm %q rows affected: %w", destination, err) + } + if inserted == 0 { + // Already in alarm: keep the original latch untouched. + return tx.Commit() + } + note := fmt.Sprintf("destination=%s %s", destination, detail) + if err := appendRunAuditTx(ctx, tx, + RunAuditEntry{RunID: raisedRunID, Transition: TransitionAlarmRaise, Note: note}, atNs); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit raise alarm %q: %w", destination, err) + } + return nil +} + +// ClearDestinationAlarm clears the active alarm on destination, if any, +// and appends an 'alarm-clear' runs_audit entry against auditRunID. +// operator is "" for an automatic clear by a clean verify pass (the +// clearing run is that pass) and the operator name for an explicit ack +// (the clearing run is the one that raised the alarm). Returns whether an +// alarm was active. Deleting the latch loses no history: the raise and +// this clear both survive in runs_audit, and every verify run that +// touched the destination survives in runs. +func (s *Store) ClearDestinationAlarm(ctx context.Context, destination string, auditRunID int64, operator string) (bool, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, fmt.Errorf("begin clear alarm %q: %w", destination, err) + } + defer func() { _ = tx.Rollback() }() + + res, err := tx.ExecContext(ctx, `DELETE FROM destination_alarms WHERE destination = ?`, destination) + if err != nil { + return false, fmt.Errorf("clear alarm %q: %w", destination, err) + } + deleted, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("clear alarm %q rows affected: %w", destination, err) + } + if deleted == 0 { + return false, tx.Commit() + } + if err := appendRunAuditTx(ctx, tx, RunAuditEntry{ + RunID: auditRunID, Transition: TransitionAlarmClear, Operator: operator, + Note: "destination=" + destination, + }, NowNs()); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("commit clear alarm %q: %w", destination, err) + } + return true, nil +} + +// GetDestinationAlarm returns the active alarm on destination, or +// sql.ErrNoRows (IsNotFound) when the destination is not in alarm. +func (s *Store) GetDestinationAlarm(ctx context.Context, destination string) (DestinationAlarm, error) { + row := s.db.QueryRowContext(ctx, + `SELECT destination, kind, detail, raised_run_id, raised_at_ns + FROM destination_alarms WHERE destination = ?`, destination) + return scanDestinationAlarm(row.Scan) +} + +// ListDestinationAlarms returns every active alarm, destination-sorted so +// the listing is deterministic. An empty slice means nothing is in alarm. +func (s *Store) ListDestinationAlarms(ctx context.Context) ([]DestinationAlarm, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT destination, kind, detail, raised_run_id, raised_at_ns + FROM destination_alarms ORDER BY destination`) + if err != nil { + return nil, fmt.Errorf("list destination alarms: %w", err) + } + defer rows.Close() + var out []DestinationAlarm + for rows.Next() { + a, err := scanDestinationAlarm(rows.Scan) + if err != nil { + return nil, err + } + out = append(out, a) + } + return out, rows.Err() +} + +func scanDestinationAlarm(scan func(...any) error) (DestinationAlarm, error) { + var a DestinationAlarm + err := scan(&a.Destination, &a.Kind, &a.Detail, &a.RaisedRunID, &a.RaisedAtNs) + if err != nil { + return DestinationAlarm{}, err + } + return a, nil +} diff --git a/store/destination_alarms_test.go b/store/destination_alarms_test.go new file mode 100644 index 0000000..1c2c11e --- /dev/null +++ b/store/destination_alarms_test.go @@ -0,0 +1,115 @@ +package store + +import ( + "context" + "testing" +) + +// TestDestinationAlarmLifecycle exercises the raise → idempotent re-raise → +// clear cycle and the runs_audit trail each transition leaves. +func TestDestinationAlarmLifecycle(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + + first, err := s.BeginRemoteVerifyRun(ctx) + if err != nil { + t.Fatalf("BeginRemoteVerifyRun: %v", err) + } + if err := s.RaiseDestinationAlarm(ctx, "offsite", AlarmKindVerifyMismatch, "mismatched=1 missing=0", first); err != nil { + t.Fatalf("RaiseDestinationAlarm: %v", err) + } + got, err := s.GetDestinationAlarm(ctx, "offsite") + if err != nil { + t.Fatalf("GetDestinationAlarm: %v", err) + } + if got.Kind != AlarmKindVerifyMismatch || got.RaisedRunID != first || got.RaisedAtNs == 0 { + t.Fatalf("alarm = %+v, want kind/verify-run/timestamp set", got) + } + firstAt := got.RaisedAtNs + + // A second detection keeps the original latch: same run, same + // timestamp, same detail — so "in alarm since" never resets. + second, err := s.BeginRemoteVerifyRun(ctx) + if err != nil { + t.Fatalf("BeginRemoteVerifyRun: %v", err) + } + if err := s.RaiseDestinationAlarm(ctx, "offsite", AlarmKindVerifyMismatch, "mismatched=2 missing=1", second); err != nil { + t.Fatalf("re-raise: %v", err) + } + got, err = s.GetDestinationAlarm(ctx, "offsite") + if err != nil { + t.Fatalf("GetDestinationAlarm after re-raise: %v", err) + } + if got.RaisedRunID != first || got.RaisedAtNs != firstAt || got.Detail != "mismatched=1 missing=0" { + t.Fatalf("re-raise mutated the latch: %+v", got) + } + + all, err := s.ListDestinationAlarms(ctx) + if err != nil { + t.Fatalf("ListDestinationAlarms: %v", err) + } + if len(all) != 1 || all[0].Destination != "offsite" { + t.Fatalf("active alarms = %+v, want exactly offsite", all) + } + + // Exactly one alarm-raise audit row against the first run; none against + // the second (the re-raise was a no-op). + if n := countTransition(t, s, first, TransitionAlarmRaise); n != 1 { + t.Fatalf("first run alarm-raise count = %d, want 1", n) + } + if n := countTransition(t, s, second, TransitionAlarmRaise); n != 0 { + t.Fatalf("second run alarm-raise count = %d, want 0 (idempotent re-raise)", n) + } + + // Clear via ack: recorded against the raising run, tagged operator. + cleared, err := s.ClearDestinationAlarm(ctx, "offsite", first, "alice") + if err != nil { + t.Fatalf("ClearDestinationAlarm: %v", err) + } + if !cleared { + t.Fatal("ClearDestinationAlarm reported no active alarm") + } + if _, err := s.GetDestinationAlarm(ctx, "offsite"); !IsNotFound(err) { + t.Fatalf("alarm still present after clear: %v", err) + } + audits, err := s.ListRunAudit(ctx, first) + if err != nil { + t.Fatalf("ListRunAudit: %v", err) + } + var sawClear bool + for _, a := range audits { + if a.Transition == TransitionAlarmClear { + sawClear = true + if a.Operator.String != "alice" { + t.Fatalf("alarm-clear operator = %q, want alice", a.Operator.String) + } + } + } + if !sawClear { + t.Fatal("no alarm-clear audit row against the raising run") + } + + // Clearing an already-clear destination reports false and writes nothing. + cleared, err = s.ClearDestinationAlarm(ctx, "offsite", first, "alice") + if err != nil { + t.Fatalf("second clear: %v", err) + } + if cleared { + t.Fatal("second clear reported an active alarm") + } +} + +func countTransition(t *testing.T, s *Store, runID int64, transition string) int { + t.Helper() + audits, err := s.ListRunAudit(context.Background(), runID) + if err != nil { + t.Fatalf("ListRunAudit(%d): %v", runID, err) + } + n := 0 + for _, a := range audits { + if a.Transition == transition { + n++ + } + } + return n +} diff --git a/store/destination_run_ids_test.go b/store/destination_run_ids_test.go index 4046a64..47275b5 100644 --- a/store/destination_run_ids_test.go +++ b/store/destination_run_ids_test.go @@ -9,6 +9,30 @@ import ( "time" ) +// minimalRunsDDL is the runs table at its v15-through-v24 shape. The +// destination_run_ids migration fixtures below carry it because the +// v24→v25 migration rebuilds runs (widening its status CHECK), which reads +// from the existing table — a real database at any of these versions +// always has runs, so the minimal fixtures must too. +const minimalRunsDDL = `CREATE TABLE runs ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL CHECK (kind IN ('index','sync','restore','audit','offload')), + volume_id INTEGER REFERENCES volumes(id), + destination TEXT, + started_at_ns INTEGER NOT NULL, + ended_at_ns INTEGER, + status TEXT NOT NULL CHECK (status IN ('running','success','failed','partial')), + error TEXT, + file_count INTEGER NOT NULL DEFAULT 0, + peer_node_id INTEGER REFERENCES nodes(id), + correlated_run_id INTEGER, + shallow INTEGER CHECK (shallow IS NULL OR shallow IN (0, 1)), + CHECK ( + (kind IN ('index','audit','offload') AND destination IS NULL) OR + (kind IN ('sync','restore') AND destination IS NOT NULL AND destination != '') + ) +)` + // TestMigrateV18ToV19AddsVerifyMethod builds a minimal v18 database with // a pre-existing durability component and confirms the migration adds // verify_method (NULL on the carried-over row) without disturbing the @@ -23,6 +47,7 @@ func TestMigrateV18ToV19AddsVerifyMethod(t *testing.T) { `CREATE TABLE schema_version (version INTEGER NOT NULL PRIMARY KEY)`, `CREATE TABLE volumes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, path TEXT NOT NULL)`, `CREATE TABLE nodes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, endpoint TEXT, public_key_fingerprint TEXT)`, + minimalRunsDDL, // contents exists from v14 on; a real v18 DB carries it, and the // v20→v21 triggers attach to it, so the minimal fixture must too. `CREATE TABLE contents ( @@ -97,6 +122,7 @@ func TestMigrateV22ToV23AddsVerifiedAt(t *testing.T) { `CREATE TABLE schema_version (version INTEGER NOT NULL PRIMARY KEY)`, `CREATE TABLE volumes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, path TEXT NOT NULL)`, `CREATE TABLE nodes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, endpoint TEXT, public_key_fingerprint TEXT)`, + minimalRunsDDL, `CREATE TABLE contents ( id INTEGER PRIMARY KEY, blake3 BLOB NOT NULL UNIQUE CHECK (length(blake3) = 32), @@ -822,6 +848,7 @@ func TestMigrateV21ToV22AddsSourceNodeID(t *testing.T) { `CREATE TABLE schema_version (version INTEGER NOT NULL PRIMARY KEY)`, `CREATE TABLE volumes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, path TEXT NOT NULL)`, `CREATE TABLE nodes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, endpoint TEXT, public_key_fingerprint TEXT)`, + minimalRunsDDL, `CREATE TABLE contents ( id INTEGER PRIMARY KEY, blake3 BLOB NOT NULL UNIQUE CHECK (length(blake3) = 32), diff --git a/store/migrations.go b/store/migrations.go index e200c5e..927ebb6 100644 --- a/store/migrations.go +++ b/store/migrations.go @@ -10,7 +10,7 @@ import ( ) // SchemaVersion is the schema version this binary writes and reads. -const SchemaVersion = 24 +const SchemaVersion = 25 // freshSchemaBaseline is the version applied to a brand-new database. The // chain in `migrations` continues from here. v1 is no longer reachable from @@ -60,6 +60,7 @@ func buildMigrations(mctx migrationCtx) []migration { {version: 22, up: migrateV21ToV22}, {version: 23, up: migrateV22ToV23}, {version: 24, up: migrateV23ToV24}, + {version: 25, up: migrateV24ToV25}, } } @@ -2059,3 +2060,116 @@ func migrateV23ToV24(ctx context.Context, db *sql.DB) error { } return tx.Commit() } + +// --- v24 → v25 --- + +// migrateV24ToV25 lays the audit-trail foundation for issue #157 — every +// failure and refusal becomes a run row, every abnormal condition a +// standing state: +// +// - The runs.status CHECK widens to admit two terminal states. +// 'refused' records a preflight safety gate declining before the +// transfer began (a missing/mismatched volume marker, a kopia connect +// that found no repository without --init, a layout guard tripping) — +// distinct from 'failed', which is a mid-flight failure. 'aborted' +// records a 'running' row reaped at agent startup because the process +// that owned it was killed mid-run (F14). Both are terminal; neither +// is a success. Reuses the FK-off rebuild recipe (v6→v7) because runs +// is referenced by files, hook_runs, packs, remote_objects, +// remote_packs, and runs_audit. +// - destination_alarms is the per-destination standing-alarm latch +// (F30): a verify mismatch raises a row that stays until a clean +// verify auto-clears it or an operator acks it. It is derived standing +// state — like peer_sync_state, the live row is maintained in place +// while the permanent forensic record of every raise and clear lives +// in runs/runs_audit — so clearing the latch loses no history. STRICT +// per the new-table convention. +func migrateV24ToV25(ctx context.Context, db *sql.DB) error { + conn, restore, err := disableForeignKeys(ctx, db) + if err != nil { + return err + } + defer restore() + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + if err := rebuildRunsTableV25(ctx, tx); err != nil { + return err + } + if err := createDestinationAlarmsV25(ctx, tx); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO schema_version (version) VALUES (25)`); err != nil { + return fmt.Errorf("record schema v25: %w", err) + } + if err := verifyForeignKeysClean(ctx, tx, "v24→v25"); err != nil { + return err + } + return tx.Commit() +} + +func rebuildRunsTableV25(ctx context.Context, tx *sql.Tx) error { + stmts := []string{ + `CREATE TABLE runs_v25 ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL CHECK (kind IN ('index','sync','restore','audit','offload')), + volume_id INTEGER REFERENCES volumes(id), + destination TEXT, + started_at_ns INTEGER NOT NULL, + ended_at_ns INTEGER, + status TEXT NOT NULL CHECK (status IN ('running','success','failed','partial','refused','aborted')), + error TEXT, + file_count INTEGER NOT NULL DEFAULT 0, + peer_node_id INTEGER REFERENCES nodes(id), + correlated_run_id INTEGER, + shallow INTEGER CHECK (shallow IS NULL OR shallow IN (0, 1)), + CHECK ( + (kind IN ('index','audit','offload') AND destination IS NULL) OR + (kind IN ('sync','restore') AND destination IS NOT NULL AND destination != '') + ) + )`, + `INSERT INTO runs_v25 ( + id, kind, volume_id, destination, started_at_ns, ended_at_ns, + status, error, file_count, peer_node_id, correlated_run_id, shallow + ) + SELECT id, kind, volume_id, destination, started_at_ns, ended_at_ns, + status, error, file_count, peer_node_id, correlated_run_id, shallow + FROM runs`, + `DROP TABLE runs`, + `ALTER TABLE runs_v25 RENAME TO runs`, + `CREATE INDEX idx_runs_volume_started ON runs(volume_id, started_at_ns)`, + `CREATE INDEX idx_runs_destination ON runs(destination) WHERE destination IS NOT NULL`, + } + for _, q := range stmts { + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rebuild runs: %w", err) + } + } + return nil +} + +// createDestinationAlarmsV25 creates the per-destination standing-alarm +// latch. destination is the PRIMARY KEY: at most one active alarm per +// destination, which the raise path keeps idempotent so "in alarm since" +// stays stable across repeated mismatches. raised_run_id is a real FK to +// the verify run that raised it. No secondary index: the table holds one +// row per destination in alarm (usually zero), so every read is a PK +// lookup or a full scan of a handful of rows — an index would never be +// used and would violate the low-cardinality-index guidance. +func createDestinationAlarmsV25(ctx context.Context, tx *sql.Tx) error { + const ddl = `CREATE TABLE destination_alarms ( + destination TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + detail TEXT NOT NULL, + raised_run_id INTEGER NOT NULL REFERENCES runs(id), + raised_at_ns INTEGER NOT NULL + ) STRICT` + if _, err := tx.ExecContext(ctx, ddl); err != nil { + return fmt.Errorf("create destination_alarms: %w", err) + } + return nil +} diff --git a/store/runs.go b/store/runs.go index ed48fa0..027e4ac 100644 --- a/store/runs.go +++ b/store/runs.go @@ -29,29 +29,38 @@ const ( // Run statuses. A run begins in 'running' and is moved to a terminal state by // FinishRun. 'partial' means the walk completed but some files errored. +// 'refused' means a preflight safety gate declined before the operation +// began — a missing/mismatched volume marker, a kopia connect that found no +// repository without --init, a layout guard — so the refusal reaches the +// audit trail instead of living only in a caller's error (#157, F26). It is +// distinct from 'failed', which is a mid-flight failure. 'aborted' means a +// 'running' row was reaped because the process that owned it died mid-run +// (the agent reaps its own orphans at startup — #157, F14); it is terminal +// but records that the run never completed, not that it failed. const ( RunStatusRunning = "running" RunStatusSuccess = "success" RunStatusFailed = "failed" RunStatusPartial = "partial" + RunStatusRefused = "refused" + RunStatusAborted = "aborted" ) // ErrAlreadyFinished is returned by FinishRun when the target run is -// already in a terminal status (success/partial/failed). The first -// terminal write wins: its status, error, and ended_at_ns are the audit -// record, and a second FinishRun would silently rewrite them. Callers -// that may legitimately race a double-finish (e.g. agent/sync.go's +// already in a terminal status (success/partial/failed/refused/aborted). +// The first terminal write wins: its status, error, and ended_at_ns are +// the audit record, and a second FinishRun would silently rewrite them. +// Callers that may legitimately race a double-finish (e.g. agent/sync.go's // handleClose firing after closeSession already finalised) detect this // with errors.Is and fall back to a log-only path rather than treating // it as a hard error. var ErrAlreadyFinished = errors.New("run is already in a terminal status") -// isTerminalStatus reports whether status is one of the three terminal -// run states. A row in any of these must not be re-finalised by -// FinishRun. +// isTerminalStatus reports whether status is one of the terminal run +// states. A row in any of these must not be re-finalised by FinishRun. func isTerminalStatus(status string) bool { switch status { - case RunStatusSuccess, RunStatusPartial, RunStatusFailed: + case RunStatusSuccess, RunStatusPartial, RunStatusFailed, RunStatusRefused, RunStatusAborted: return true } return false @@ -715,18 +724,22 @@ func (s *Store) LatestSuccessfulIndexRun(ctx context.Context, volumeID int64) (R return scanRun(row.Scan) } -// LatestFinishedRun returns the most recent terminal-status (success, -// partial, or failed) run of the given kind for (volumeID, destination). -// destination must be "" for index/audit kinds (the schema CHECK ensures -// those carry no destination) and the rclone destination or peer node -// name for sync/restore kinds. Returns sql.ErrNoRows when no matching -// run exists. +// LatestFinishedRun returns the most recent terminal-status run of the +// given kind for (volumeID, destination). destination must be "" for +// index/audit kinds (the schema CHECK ensures those carry no destination) +// and the rclone destination or peer node name for sync/restore kinds. +// Returns sql.ErrNoRows when no matching run exists. // // The scheduler (#39) computes `now - last_finished` from this row to -// decide whether a cadence-driven run is due. Failed terminal states -// count: per the issue's failure-policy (no special retry, the next -// tick re-evaluates), a failed run consumes the cadence window like -// any other. +// decide whether a cadence-driven run is due. success, partial, failed, +// and refused all count: per the issue's failure-policy (no special +// retry, the next tick re-evaluates), each consumes the cadence window +// like any other — a 'refused' run in particular backs the scheduler off +// a dead destination (a missing marker, an un-bootstrapped repository) +// instead of re-minting an identical refusal every tick. 'aborted' is +// deliberately excluded: a run reaped mid-flight never completed, so the +// cadence should re-attempt rather than treat the crash as a finished +// window. func (s *Store) LatestFinishedRun(ctx context.Context, kind string, volumeID int64, destination string) (Run, error) { var row *sql.Row if destination == "" { @@ -734,7 +747,7 @@ func (s *Store) LatestFinishedRun(ctx context.Context, kind string, volumeID int `SELECT `+runColumns+` FROM runs WHERE kind = ? AND volume_id = ? AND destination IS NULL - AND status IN ('success','partial','failed') + AND status IN ('success','partial','failed','refused') ORDER BY id DESC LIMIT 1`, kind, volumeID) } else { @@ -742,7 +755,7 @@ func (s *Store) LatestFinishedRun(ctx context.Context, kind string, volumeID int `SELECT `+runColumns+` FROM runs WHERE kind = ? AND volume_id = ? AND destination = ? - AND status IN ('success','partial','failed') + AND status IN ('success','partial','failed','refused') ORDER BY id DESC LIMIT 1`, kind, volumeID, destination) } @@ -881,3 +894,69 @@ func (s *Store) HasRunningRun(ctx context.Context, kind string, volumeID int64, } return found, nil } + +// AbortRunningRuns transitions every run currently in 'running' to +// 'aborted', stamping ended_at_ns, recording reason as the row's error, +// and appending an 'abort' runs_audit transition per reaped row so the +// append-only trail records who moved the row and why. It returns the ids +// reaped, oldest first, so the caller can log them. +// +// Called once at agent startup to clear phantom 'running' rows left by a +// prior agent that was killed mid-run (#157, F14): those rows otherwise +// block BeginSyncRunIfClear/BeginIndexRunIfClear forever and render as a +// live, elapsed-ticking banner in the TUI hours later. A freshly started +// agent has kicked nothing yet, so at startup every 'running' row +// necessarily predates it — the reap assumes the agent is the single +// writer of runs on this node (the reference-setup model), and a +// concurrent CLI run started against the same DB is out of scope. +func (s *Store) AbortRunningRuns(ctx context.Context, reason string) ([]int64, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("begin abort running runs: %w", err) + } + defer func() { _ = tx.Rollback() }() + + ids, err := runningRunIDsTx(ctx, tx) + if err != nil { + return nil, err + } + if len(ids) == 0 { + return nil, nil + } + atNs := NowNs() + for _, id := range ids { + if _, err := tx.ExecContext(ctx, ` + UPDATE runs SET ended_at_ns = ?, status = ?, error = ? + WHERE id = ? AND status = 'running' + `, atNs, RunStatusAborted, reason, id); err != nil { + return nil, fmt.Errorf("abort run %d: %w", id, err) + } + if err := appendRunAuditTx(ctx, tx, + RunAuditEntry{RunID: id, Transition: TransitionAbort, Note: reason}, atNs); err != nil { + return nil, err + } + } + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("commit abort running runs: %w", err) + } + return ids, nil +} + +// runningRunIDsTx returns the ids of every 'running' run inside tx, +// ascending, so the reap order and the returned slice are deterministic. +func runningRunIDsTx(ctx context.Context, tx *sql.Tx) ([]int64, error) { + rows, err := tx.QueryContext(ctx, `SELECT id FROM runs WHERE status = 'running' ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("list running runs: %w", err) + } + defer rows.Close() + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("scan running run id: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() +} diff --git a/store/runs_audit.go b/store/runs_audit.go index e74fb48..9a9d04e 100644 --- a/store/runs_audit.go +++ b/store/runs_audit.go @@ -33,6 +33,20 @@ const ( // CHECK keeps destination NULL on audit rows, so this entry is where // the audit trail names the verified destination. TransitionVerifyDestination = "verify-destination" + // TransitionAbort records a 'running' row reaped to 'aborted' by + // AbortRunningRuns at agent startup (#157, F14). The note carries the + // reap reason so a forensic reader can tell a crash-reap apart from a + // real failure. + TransitionAbort = "abort" + // TransitionAlarmRaise records a destination_alarms latch being raised + // against the verify run that detected the mismatch (#157, F30). The + // note carries the destination and a bounded detail summary. + TransitionAlarmRaise = "alarm-raise" + // TransitionAlarmClear records a destination_alarms latch being + // cleared: against the clean verify run that auto-cleared it (operator + // NULL) or against the run that raised it when an operator acks + // (operator set). The note carries the destination. + TransitionAlarmClear = "alarm-clear" ) // RunAudit is one row of the insert-only runs_audit log: a single diff --git a/store/schema.sql b/store/schema.sql index 5bfab8b..692cc9d 100644 --- a/store/schema.sql +++ b/store/schema.sql @@ -1,6 +1,6 @@ -- Generated by `go test ./store -update-schema` — DO NOT EDIT. -- --- Flattened snapshot of the squirrel index schema at version 24, for humans +-- Flattened snapshot of the squirrel index schema at version 25, for humans -- and agents who want the current shape without replaying the migration -- chain in migrations.go. It is NOT used to create or migrate databases — -- a fresh DB is built by applyV5 plus the migration registry. The golden @@ -27,6 +27,14 @@ CREATE TRIGGER contents_no_update BEFORE UPDATE ON contents SELECT RAISE(ABORT, 'contents is append-only; supersede the files row and insert new content instead of updating'); END; +CREATE TABLE destination_alarms ( + destination TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + detail TEXT NOT NULL, + raised_run_id INTEGER NOT NULL REFERENCES runs(id), + raised_at_ns INTEGER NOT NULL + ) STRICT; + CREATE TABLE destination_push_freshness ( volume_id INTEGER NOT NULL REFERENCES volumes(id), destination TEXT NOT NULL, @@ -181,7 +189,7 @@ CREATE TABLE "runs" ( destination TEXT, started_at_ns INTEGER NOT NULL, ended_at_ns INTEGER, - status TEXT NOT NULL CHECK (status IN ('running','success','failed','partial')), + status TEXT NOT NULL CHECK (status IN ('running','success','failed','partial','refused','aborted')), error TEXT, file_count INTEGER NOT NULL DEFAULT 0, peer_node_id INTEGER REFERENCES nodes(id), diff --git a/sync/content_addressed.go b/sync/content_addressed.go index bbe0ecc..5cb5a4b 100644 --- a/sync/content_addressed.go +++ b/sync/content_addressed.go @@ -211,7 +211,7 @@ func (h *contentAddressedHandler) watermark(ctx context.Context, volID int64) (i } segURI := h.segmentURI(last.ID) if _, err := h.rcl.statRemote(ctx, segURI, checkersArgs(h.dest)...); err != nil { - return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no manifest segment at %s — its history does not look content-addressed; point the layout at a fresh destination or root instead of switching an existing one: %w", h.dest.Name, last.ID, segURI, err) + return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no manifest segment at %s — its history does not look content-addressed; point the layout at a fresh destination or root instead of switching an existing one: %w: %w", h.dest.Name, last.ID, segURI, err, ErrRefused) } return last.ID, nil } diff --git a/sync/content_addressed_test.go b/sync/content_addressed_test.go index fbb3f6d..2d40deb 100644 --- a/sync/content_addressed_test.go +++ b/sync/content_addressed_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -627,8 +628,11 @@ func TestContentAddressedWatermarkGuard(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "does not look content-addressed") { t.Fatalf("expected layout-flip refusal, got %v", err) } - if rep.Status != store.RunStatusFailed { - t.Fatalf("Status = %q, want failed", rep.Status) + if !errors.Is(err, ErrRefused) { + t.Fatalf("layout guard should be a refusal (ErrRefused), got %v", err) + } + if rep.Status != store.RunStatusRefused { + t.Fatalf("Status = %q, want refused", rep.Status) } } diff --git a/sync/handler.go b/sync/handler.go index ab7e21a..cac45ff 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -171,15 +171,22 @@ func (h *peerHandler) Push(ctx context.Context, opts Options) (Report, error) { func (h *peerHandler) sealed() {} -// finishHandlerRun writes a handler-driven run's terminal state from -// rep.Status, mirroring the rclone scaffold's finishRun contract: a -// FinishRun failure lands on rep.FinishErr so the caller surfaces it -// next to the push outcome. The kopia and content-addressed handlers -// share it; their file counts ride on rep.Verification.Files. +// finishHandlerRun writes a handler-driven run's terminal state, +// mirroring the rclone scaffold's finishRun contract: a FinishRun failure +// lands on rep.FinishErr so the caller surfaces it next to the push +// outcome. The kopia, content-addressed, and packed handlers share it; +// their file counts ride on rep.Verification.Files. +// +// A preflight refusal (runErr wrapping ErrRefused — a kopia connect +// without --init, a layout guard) overrides the derived status to +// 'refused' via terminalStatus, and rep.Status is updated in place so the +// CLI summary and TUI show the refusal as its own condition rather than a +// generic failure (#157, F26). func finishHandlerRun(ctx context.Context, s *store.Store, rep *Report, runErr error) { if rep.RunID == 0 { return } + rep.Status = terminalStatus(rep.Status, runErr) errMsg := "" if runErr != nil { errMsg = runErr.Error() diff --git a/sync/kopia.go b/sync/kopia.go index 26fe2e4..9915ce8 100644 --- a/sync/kopia.go +++ b/sync/kopia.go @@ -100,7 +100,7 @@ func (k *Kopia) ensureRepository(ctx context.Context, cfgFile, password, repoPat return nil } if !init { - return fmt.Errorf("kopia repository at %s: connect failed (%w) — re-run with --init to create a new repository (refusing to auto-create in case the path is wrong or the destination is temporarily unreachable)", repoPath, connectErr) + return fmt.Errorf("kopia repository at %s: connect failed (%w) — re-run with --init to create a new repository (refusing to auto-create in case the path is wrong or the destination is temporarily unreachable): %w", repoPath, connectErr, ErrRefused) } if _, createErr := k.run(ctx, cfgFile, password, "repository", "create", "filesystem", "--path", repoPath, "--no-persist-credentials"); createErr != nil { return fmt.Errorf("kopia repository at %s: connect failed (%w); create failed: %w", repoPath, connectErr, createErr) diff --git a/sync/kopia_test.go b/sync/kopia_test.go index e8e838d..bae2e13 100644 --- a/sync/kopia_test.go +++ b/sync/kopia_test.go @@ -248,8 +248,11 @@ func TestKopiaConnectFailWithoutInitRefuses(t *testing.T) { if !strings.Contains(err.Error(), "--init") { t.Fatalf("error should point at --init, got %v", err) } - if rep.Status != store.RunStatusFailed { - t.Fatalf("Status = %q, want failed", rep.Status) + if !errors.Is(err, ErrRefused) { + t.Fatalf("connect-without-init should be a refusal (ErrRefused), got %v", err) + } + if rep.Status != store.RunStatusRefused { + t.Fatalf("Status = %q, want refused", rep.Status) } argv, _ := readCallLog(t, logPath) for _, line := range argv { diff --git a/sync/packed.go b/sync/packed.go index 88bca90..041bca2 100644 --- a/sync/packed.go +++ b/sync/packed.go @@ -264,7 +264,7 @@ func (h *packedHandler) watermark(ctx context.Context, volID int64) (int64, erro } mapURI := h.mapURI(last.ID) if _, err := h.rcl.statRemote(ctx, mapURI, checkersArgs(h.dest)...); err != nil { - return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no pack placement map at %s — its history is not packed (a mirror or content-addressed root); point the layout at a fresh destination or root instead of switching an existing one: %w", h.dest.Name, last.ID, mapURI, err) + return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no pack placement map at %s — its history is not packed (a mirror or content-addressed root); point the layout at a fresh destination or root instead of switching an existing one: %w: %w", h.dest.Name, last.ID, mapURI, err, ErrRefused) } return last.ID, nil } diff --git a/sync/packed_test.go b/sync/packed_test.go index ba285b3..9941986 100644 --- a/sync/packed_test.go +++ b/sync/packed_test.go @@ -430,8 +430,11 @@ func TestPackedWatermarkGuardRefusesContentAddressed(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "not packed") { t.Fatalf("expected packed-history refusal, got %v", err) } - if rep.Status != store.RunStatusFailed { - t.Fatalf("Status = %q, want failed", rep.Status) + if !errors.Is(err, ErrRefused) { + t.Fatalf("layout guard should be a refusal (ErrRefused), got %v", err) + } + if rep.Status != store.RunStatusRefused { + t.Fatalf("Status = %q, want refused", rep.Status) } } @@ -454,8 +457,11 @@ func TestPackedWatermarkGuardRefusesMirror(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "not packed") { t.Fatalf("expected packed-history refusal, got %v", err) } - if rep.Status != store.RunStatusFailed { - t.Fatalf("Status = %q, want failed", rep.Status) + if !errors.Is(err, ErrRefused) { + t.Fatalf("layout guard should be a refusal (ErrRefused), got %v", err) + } + if rep.Status != store.RunStatusRefused { + t.Fatalf("Status = %q, want refused", rep.Status) } } diff --git a/sync/rclone.go b/sync/rclone.go index 19316e7..aef11ff 100644 --- a/sync/rclone.go +++ b/sync/rclone.go @@ -238,6 +238,33 @@ type RunResult struct { // for BLAKE3 verification but hit this path was not content-verified, // however rclone exited, so the caller must not record it as verified. HashFallback bool + // Stderr is a bounded tail of rclone's non-JSON stderr — the + // human-readable diagnostics it prints before (or instead of) the + // structured JSON log: backend construction, config, auth, and host-key + // failures. Bounded to maxStderrCapture bytes so a pathological run + // can't balloon memory. It is the evidence that made a failing + // destination undiagnosable when it was dropped on the floor (#157, + // F6/F15); callers fold it into the run row's error and the scheduler + // log. Empty when rclone emitted only structured events. + Stderr string +} + +// DisplayErrors is the error count for summary lines and status +// reconciliation. It returns rclone's per-file Errors, except that a fatal +// invocation failure that produced no per-file count reports the number of +// captured diagnostics (at least one) — so a run that is status=failed +// never simultaneously claims errors=0, the contradiction F6 flagged. +func (r RunResult) DisplayErrors() int64 { + if r.Errors > 0 { + return r.Errors + } + if r.FatalError { + if n := int64(len(r.FailedFiles)); n > 0 { + return n + } + return 1 + } + return 0 } // FailedFile is one per-object error from the JSON log. Object may be @@ -252,6 +279,11 @@ type FailedFile struct { // error count in RunResult.Errors is exact regardless of this cap. const maxFailedFiles = 100 +// maxStderrCapture bounds RunResult.Stderr. rclone's fatal diagnostics are +// a line or two; 4 KiB keeps a useful head of a chattier failure without +// letting a misbehaving child stream unbounded bytes into the run row. +const maxStderrCapture = 4 << 10 + // Run executes `rclone --config --use-json-log` with // the given extra arguments. It streams both stdout (rclone writes to // stderr by default but --use-json-log routes structured output to stderr @@ -315,11 +347,22 @@ func (r *Rclone) RunWithProgress(ctx context.Context, onProgress func(runevents. waitErr := cmd.Wait() if waitErr != nil { result.FatalError = result.Errors == 0 - return result, fmt.Errorf("rclone exit: %w", waitErr) + return result, rcloneExitError(waitErr, result.Stderr) } return result, nil } +// rcloneExitError wraps a non-zero rclone exit, folding in the captured +// stderr tail when present so the scheduler log and the run row carry the +// actual reason (auth, host key, path) rather than a bare "exit status 1" +// (#157, F6/F15). +func rcloneExitError(waitErr error, stderrTail string) error { + if tail := strings.TrimSpace(stderrTail); tail != "" { + return fmt.Errorf("rclone exit: %w: %s", waitErr, tail) + } + return fmt.Errorf("rclone exit: %w", waitErr) +} + // runPlain executes rclone with the wrapper's --config but without the // JSON-log and stats flags Run uses, returning captured stdout. It backs // the auxiliary commands the snapshot ride-along needs (copyto, lsf, @@ -491,21 +534,41 @@ var hashFallbackRE = regexp.MustCompile(`no hashes in common`) func isHashFallback(msg string) bool { return hashFallbackRE.MatchString(msg) } +// isErrorLevel reports whether a JSON event's level should be captured as +// a failure diagnostic. rclone reports a fatal backend/config/auth failure +// at "fatal" (sometimes "critical"), not "error"; capturing only "error" +// dropped exactly those messages, leaving a failed run's ERROR column +// blank (#157, F15). All three are folded into FailedFiles. +func isErrorLevel(level string) bool { + switch level { + case "error", "fatal", "critical": + return true + } + return false +} + // parseJSONLog reads JSON-per-line events from r and updates result in -// place. Non-JSON lines (e.g. an early startup notice on an older rclone) -// are skipped — we cannot make decisions on them and surfacing them as -// errors would create false positives. onProgress, if non-nil, is -// invoked once per stats event so callers can drive a live UI. +// place. Non-JSON lines (rclone's pre-logger diagnostics — the backend, +// config, auth, and host-key failures it prints before the structured +// logger is live) are appended to result.Stderr, bounded, so a failure +// that never reaches the JSON log still carries its reason (#157, +// F6/F15). onProgress, if non-nil, is invoked once per stats event so +// callers can drive a live UI. func parseJSONLog(r io.Reader, result *RunResult, onProgress func(runevents.Progress)) { scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) for scanner.Scan() { line := scanner.Bytes() - if len(line) == 0 || line[0] != '{' { + if len(line) == 0 { + continue + } + if line[0] != '{' { + appendStderr(result, line) continue } var ev rcloneEvent if err := json.Unmarshal(line, &ev); err != nil { + appendStderr(result, line) continue } if isHashFallback(ev.Msg) { @@ -515,30 +578,10 @@ func parseJSONLog(r io.Reader, result *RunResult, onProgress func(runevents.Prog result.HashFallback = true } if ev.Stats != nil { - result.Transferred = ev.Stats.TotalTransfers - result.Checked = ev.Stats.TotalChecks - result.Bytes = ev.Stats.Bytes - // Don't overwrite Errors with a smaller value from a - // per-attempt stats line; take the maximum so retries that - // fail then succeed still surface the failure count. - if ev.Stats.Errors > result.Errors { - result.Errors = ev.Stats.Errors - } - if ev.Stats.FatalError { - result.FatalError = true - } - if onProgress != nil { - onProgress(runevents.Progress{ - Stage: runevents.StageUploading, - Done: result.Transferred, - Total: ev.Stats.TotalTransfers + ev.Stats.TotalChecks, - BytesDone: result.Bytes, - BytesTotal: ev.Stats.TotalBytes, - }) - } + applyStatsEvent(result, ev.Stats, onProgress) continue } - if ev.Level == "error" && !isRetrySummary(ev.Msg) { + if isErrorLevel(ev.Level) && !isRetrySummary(ev.Msg) { // Capture object-less errors too: auth failures, listing // errors, "Failed to copy: …" diagnostics carry no Object // but are exactly the messages we want in runs.error. Filter @@ -552,3 +595,50 @@ func parseJSONLog(r io.Reader, result *RunResult, onProgress func(runevents.Prog } } } + +// applyStatsEvent folds one rclone stats event into result and drives the +// optional progress callback. Errors is taken as the running maximum so a +// per-attempt stats line that reports fewer errors than a prior one (a +// retry that partially recovered) can't erase the peak failure count. +func applyStatsEvent(result *RunResult, st *rcloneStats, onProgress func(runevents.Progress)) { + result.Transferred = st.TotalTransfers + result.Checked = st.TotalChecks + result.Bytes = st.Bytes + if st.Errors > result.Errors { + result.Errors = st.Errors + } + if st.FatalError { + result.FatalError = true + } + if onProgress != nil { + onProgress(runevents.Progress{ + Stage: runevents.StageUploading, + Done: result.Transferred, + Total: st.TotalTransfers + st.TotalChecks, + BytesDone: result.Bytes, + BytesTotal: st.TotalBytes, + }) + } +} + +// appendStderr accumulates a non-JSON stderr line into result.Stderr up to +// maxStderrCapture bytes. Once the cap is reached further lines are +// dropped — the head of a fatal diagnostic is what identifies the failure, +// so keeping the first bytes is more useful than a tail. Blank lines are +// skipped so the capture stays dense. +func appendStderr(result *RunResult, line []byte) { + if len(result.Stderr) >= maxStderrCapture { + return + } + trimmed := strings.TrimSpace(string(line)) + if trimmed == "" { + return + } + if result.Stderr != "" { + result.Stderr += "\n" + } + result.Stderr += trimmed + if len(result.Stderr) > maxStderrCapture { + result.Stderr = result.Stderr[:maxStderrCapture] + } +} diff --git a/sync/rclone_test.go b/sync/rclone_test.go index 8615ff0..c8eb9e7 100644 --- a/sync/rclone_test.go +++ b/sync/rclone_test.go @@ -73,6 +73,62 @@ func TestParseJSONLogCapturesObjectlessErrors(t *testing.T) { } } +// TestParseJSONLogCapturesFatalLevelAndRawStderr: a fatal-level JSON event +// (previously dropped by the error-only filter) and a non-JSON pre-logger +// diagnostic are both captured, so a failing destination is diagnosable +// (#157, F6/F15) instead of leaving a blank ERROR column. +func TestParseJSONLogCapturesFatalLevelAndRawStderr(t *testing.T) { + stream := strings.Join([]string{ + `Failed to create file system for "cloudbox:/x": didn't find backend`, + `{"level":"fatal","msg":"NoCredentialProviders: no valid providers in chain"}`, + `{"stats":{"errors":0,"fatalError":true,"totalTransfers":0,"totalChecks":0,"bytes":0}}`, + }, "\n") + var r RunResult + parseJSONLog(strings.NewReader(stream), &r, nil) + + if len(r.FailedFiles) != 1 || !strings.Contains(r.FailedFiles[0].Message, "NoCredentialProviders") { + t.Fatalf("FailedFiles = %+v, want the fatal-level message captured", r.FailedFiles) + } + if !strings.Contains(r.Stderr, "didn't find backend") { + t.Fatalf("Stderr = %q, want the non-JSON pre-logger diagnostic", r.Stderr) + } +} + +// TestParseJSONLogStderrBounded: a pathological non-JSON stream can't +// balloon RunResult.Stderr past the cap. +func TestParseJSONLogStderrBounded(t *testing.T) { + var r RunResult + parseJSONLog(strings.NewReader(strings.Repeat("x", 10*maxStderrCapture)), &r, nil) + if len(r.Stderr) == 0 { + t.Fatal("Stderr empty, want the head of the long line") + } + if len(r.Stderr) > maxStderrCapture { + t.Fatalf("Stderr len = %d, want <= %d", len(r.Stderr), maxStderrCapture) + } +} + +// TestDisplayErrors pins the summary error count: rclone's per-file count +// wins, but a fatal invocation failure with no per-file count still +// reports at least one error so a status=failed run never claims errors=0. +func TestDisplayErrors(t *testing.T) { + cases := []struct { + name string + r RunResult + want int64 + }{ + {"per-file errors", RunResult{Errors: 3}, 3}, + {"fatal, no count, no captured files", RunResult{FatalError: true}, 1}, + {"fatal, captured files", RunResult{FatalError: true, FailedFiles: []FailedFile{{Message: "a"}, {Message: "b"}}}, 2}, + {"fatal but per-file count wins", RunResult{FatalError: true, Errors: 5}, 5}, + {"clean", RunResult{}, 0}, + } + for _, c := range cases { + if got := c.r.DisplayErrors(); got != c.want { + t.Errorf("%s: DisplayErrors() = %d, want %d", c.name, got, c.want) + } + } +} + // TestParseJSONLogDetectsHashFallback: rclone's no-common-hash notice is // emitted at NOTICE level, which the error filter drops; parseJSONLog // still flags it so a flags-set, exit-0 run that silently degraded to a diff --git a/sync/sync.go b/sync/sync.go index 24a1ca1..4fcf57f 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -50,6 +50,28 @@ const RestoreHistoryDirName = ".squirrel-restore-history" // and from peer-sync so a snapshot is never mistaken for user content. const IndexDirName = ".squirrel-index" +// ErrRefused marks a preflight safety refusal: a gate that declined to +// proceed before any transfer began — a missing or mismatched +// .squirrel-volume marker, a kopia connect that found no repository +// without --init, or a layout guard protecting a destination whose +// recorded history belongs to a different layout. Gates wrap it into their +// error with %w so the run-finalising path records store.RunStatusRefused +// rather than RunStatusFailed: a refusal is a standing "won't do this" +// condition, not a mid-flight failure, and the audit trail keeps the two +// distinct (#157, F26). +var ErrRefused = errors.New("refused") + +// terminalStatus resolves a run's terminal status from the derived status +// and the operation error. A preflight refusal (errors.Is(err, +// ErrRefused)) overrides to 'refused'; every other outcome keeps the +// derived status. +func terminalStatus(status string, runErr error) string { + if runErr != nil && errors.Is(runErr, ErrRefused) { + return store.RunStatusRefused + } + return status +} + // Options shapes one Sync invocation. type Options struct { // Shallow drops --checksum and --hash blake3 so rclone uses its default @@ -252,8 +274,12 @@ func Sync(ctx context.Context, s *store.Store, rcl *Rclone, vol *config.Volume, // dry-run on an uninitialised destination would prevent the // "preview what would happen" workflow. if !opts.DryRun && dest.Type == "local" { - if err := ensureDestinationMarker(ctx, s, dest, vol.Name, opts.Init); err != nil { - return rep, err + if merr := ensureDestinationMarker(ctx, s, dest, vol.Name, opts.Init); merr != nil { + // A marker refusal fires before the sync run is allocated, so + // record it as its own terminal 'refused' run — otherwise a + // month-dead backup disk produces zero red anywhere but agent + // stderr (#157, F26). + return recordSyncRefusal(ctx, s, volID, dest.Name, &rep, merr) } } @@ -316,6 +342,28 @@ func captureDurabilityAdvance(ctx context.Context, s *store.Store, volumeID int6 return components, nil } +// recordSyncRefusal mints a terminal kind='sync' run in the 'refused' +// state for a preflight gate that declined before the transfer began, so +// the refusal is visible in `squirrel runs` and the TUI instead of living +// only in the returned error (#157, F26). The run is allocated ungated +// (BeginRun, not the concurrency gate) — a refusal transfers nothing and +// need not contend with an in-flight sync — and finished immediately with +// refErr as the row's error. refErr is returned unchanged so callers +// surface the same message; a failure to even record the refusal is folded +// into the returned error rather than masking the original refusal. +func recordSyncRefusal(ctx context.Context, s *store.Store, volID int64, destName string, rep *Report, refErr error) (Report, error) { + id, err := s.BeginRun(ctx, store.RunKindSync, volID, destName, false) + if err != nil { + return *rep, fmt.Errorf("%w (also failed to record the refusal as a run: %w)", refErr, err) + } + rep.RunID = id + rep.Status = store.RunStatusRefused + if ferr := s.FinishRun(ctx, id, store.RunStatusRefused, refErr.Error(), 0); ferr != nil { + rep.FinishErr = ferr + } + return *rep, refErr +} + // beginSyncRunGuarded is the sync-allocator the bucket and peer paths // share. It honours dry-run (returns 0 with no DB write) and delegates // to store.BeginSyncRunIfClear for the atomic gate. A blocked attempt @@ -368,11 +416,19 @@ func runRcloneOperation( return err } rep.RcloneResult, err = rcl.RunWithProgress(ctx, progress, args...) - if err != nil && rep.RcloneResult.Errors == 0 && !rep.RcloneResult.FatalError { - // Invocation failed without a parseable error count: treat as fatal. - rep.RcloneResult.FatalError = true - } if err != nil { + if rep.RcloneResult.Errors == 0 && !rep.RcloneResult.FatalError { + // Invocation failed without a parseable error count: treat as fatal. + rep.RcloneResult.FatalError = true + } + // Ensure a diagnostic reaches the run row's error column and the + // CLI output even when rclone logged no per-file error event — auth, + // host-key, and config failures print to stderr, not the JSON log + // (#157, F6/F15). The returned error already folds in the stderr + // tail, so it is the most complete single message available. + if len(rep.RcloneResult.FailedFiles) == 0 { + rep.RcloneResult.FailedFiles = []FailedFile{{Message: err.Error()}} + } return fmt.Errorf("rclone: %w", err) } return nil @@ -519,13 +575,13 @@ func ensureDestinationMarker(ctx context.Context, s *store.Store, dest *config.D return nil } if _, ok := errors.AsType[*volmark.ErrMismatch](err); ok { - return fmt.Errorf("destination %q: %w (refuse to init over a different volume's tree)", dest.Name, err) + return fmt.Errorf("destination %q: %w (refuse to init over a different volume's tree): %w", dest.Name, err, ErrRefused) } if !errors.Is(err, volmark.ErrMissing) { return fmt.Errorf("destination %q marker check: %w", dest.Name, err) } if !init { - return fmt.Errorf("destination %q at %s has no %s marker — re-run with --init to bootstrap (refusing in case the root is a typo)", dest.Name, root, volmark.MarkerName) + return fmt.Errorf("destination %q at %s has no %s marker — re-run with --init to bootstrap (refusing in case the root is a typo): %w", dest.Name, root, volmark.MarkerName, ErrRefused) } if err := os.MkdirAll(root, 0o755); err != nil { return fmt.Errorf("destination %q: mkdir %s: %w", dest.Name, root, err) diff --git a/sync/sync_test.go b/sync/sync_test.go index a035564..4086519 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -2,6 +2,7 @@ package sync import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -836,10 +837,26 @@ func TestSyncRefusesUninitialisedDestination(t *testing.T) { t.Fatalf("remove seeded marker: %v", err) } - _, err := Sync(context.Background(), f.store, f.rcl, f.vol, f.dest, Options{}) + rep, err := Sync(context.Background(), f.store, f.rcl, f.vol, f.dest, Options{}) if err == nil || !strings.Contains(err.Error(), "--init") { t.Fatalf("expected --init hint, got %v", err) } + if !errors.Is(err, ErrRefused) { + t.Fatalf("marker refusal should wrap ErrRefused, got %v", err) + } + // The refusal must mint a visible terminal run row, not vanish into + // the returned error with run_id=0 (#157, F26). + if rep.RunID == 0 || rep.Status != store.RunStatusRefused { + t.Fatalf("rep = %+v, want a refused run row", rep) + } + run, gerr := f.store.GetRun(context.Background(), rep.RunID) + if gerr != nil { + t.Fatalf("GetRun(%d): %v", rep.RunID, gerr) + } + if run.Kind != store.RunKindSync || run.Status != store.RunStatusRefused || + !run.Error.Valid || !strings.Contains(run.Error.String, "--init") { + t.Fatalf("run = %+v, want a refused sync run carrying the refusal message", run) + } } // TestSyncInitWritesMarker confirms that passing Init: true creates diff --git a/sync/verify_remote.go b/sync/verify_remote.go index 117d4b5..4810f15 100644 --- a/sync/verify_remote.go +++ b/sync/verify_remote.go @@ -56,6 +56,15 @@ type RemoteVerifyReport struct { // PackMismatched lists packs whose provider checksum no longer matches // the recorded one (Hash holds the pack key hex). PackMismatched []RemoteObjectMismatch + + // AlarmRaised is true when this pass latched a new standing alarm on + // the destination because it was not clean (#157, F30). A pass on an + // already-alarmed destination leaves it false (the latch was already + // there). AlarmCleared is true when a clean pass auto-cleared a + // previously standing alarm. Both drive the CLI's loud surfacing; the + // authoritative state lives in the destination_alarms latch. + AlarmRaised bool + AlarmCleared bool } // RemoteObjectMismatch is one object whose provider checksum no longer @@ -404,5 +413,37 @@ func recordVerifyOutcome(ctx context.Context, s *store.Store, rep *RemoteVerifyR if err := s.FinishRun(ctx, rep.RunID, status, errMsg, int64(rep.Objects+rep.Packs)); err != nil { return fmt.Errorf("finish verify run %d: %w", rep.RunID, err) } + return applyVerifyAlarm(ctx, s, rep, verifyErr) +} + +// applyVerifyAlarm latches or clears the destination's standing alarm from +// this pass's outcome (#157, F30). A pass that detected a mismatch or a +// missing object/pack raises the alarm (idempotent — a re-detection keeps +// the original "in alarm since"); a clean pass auto-clears any standing +// alarm, recording the clear against this verify run. A pass that aborted +// (verifyErr != nil) proves nothing about the destination's integrity, so +// it neither raises nor clears — its failed run row is the record. +func applyVerifyAlarm(ctx context.Context, s *store.Store, rep *RemoteVerifyReport, verifyErr error) error { + if verifyErr != nil { + return nil + } + if !rep.Clean() { + detail := fmt.Sprintf("objects mismatched=%d missing=%d, packs mismatched=%d missing=%d", + len(rep.Mismatched), len(rep.Missing), len(rep.PackMismatched), len(rep.PacksMissing)) + // AlarmRaised reports only a *new* latch, so the CLI can shout on + // first detection without re-shouting every subsequent pass. + switch _, err := s.GetDestinationAlarm(ctx, rep.Destination); { + case store.IsNotFound(err): + rep.AlarmRaised = true + case err != nil: + return fmt.Errorf("check standing alarm for %q: %w", rep.Destination, err) + } + return s.RaiseDestinationAlarm(ctx, rep.Destination, store.AlarmKindVerifyMismatch, detail, rep.RunID) + } + cleared, err := s.ClearDestinationAlarm(ctx, rep.Destination, rep.RunID, "") + if err != nil { + return fmt.Errorf("auto-clear standing alarm for %q: %w", rep.Destination, err) + } + rep.AlarmCleared = cleared return nil } diff --git a/sync/verify_remote_test.go b/sync/verify_remote_test.go index 7cc479b..786e9ea 100644 --- a/sync/verify_remote_test.go +++ b/sync/verify_remote_test.go @@ -59,6 +59,72 @@ func TestVerifyRemoteMatchStampsVerified(t *testing.T) { } } +// TestVerifyRemoteMismatchLatchesAlarmThenClears: a mismatch latches a +// standing per-destination alarm (#157, F30), and a subsequent clean pass +// auto-clears it. Both transitions land in the runs_audit trail. +func TestVerifyRemoteMismatchLatchesAlarmThenClears(t *testing.T) { + f := setupContentAddressedFixture(t) + f.write(t, "a.txt", "alpha") + f.index(t) + if _, err := f.sync(t); err != nil { + t.Fatalf("sync: %v", err) + } + ctx := context.Background() + + if err := os.Setenv("RCLONE_FAKE_HASH_PREFIX", "tampered-"); err != nil { + t.Fatalf("set tamper: %v", err) + } + dirty, err := VerifyRemote(ctx, f.store, f.rcl, f.pair.Destination) + if err != nil { + t.Fatalf("VerifyRemote (tampered): %v", err) + } + if !dirty.AlarmRaised { + t.Fatalf("tampered pass did not raise an alarm: %+v", dirty) + } + alarm, err := f.store.GetDestinationAlarm(ctx, f.pair.Destination.Name) + if err != nil { + t.Fatalf("GetDestinationAlarm: %v", err) + } + if alarm.Kind != store.AlarmKindVerifyMismatch || alarm.RaisedRunID != dirty.RunID { + t.Fatalf("alarm = %+v, want verify-mismatch raised by run %d", alarm, dirty.RunID) + } + if n := countAuditTransition(t, f.store, dirty.RunID, store.TransitionAlarmRaise); n != 1 { + t.Fatalf("alarm-raise audit count = %d, want 1", n) + } + + if err := os.Unsetenv("RCLONE_FAKE_HASH_PREFIX"); err != nil { + t.Fatalf("unset tamper: %v", err) + } + clean, err := VerifyRemote(ctx, f.store, f.rcl, f.pair.Destination) + if err != nil { + t.Fatalf("VerifyRemote (clean): %v", err) + } + if !clean.Clean() || !clean.AlarmCleared { + t.Fatalf("clean pass did not auto-clear the alarm: %+v", clean) + } + if _, err := f.store.GetDestinationAlarm(ctx, f.pair.Destination.Name); !store.IsNotFound(err) { + t.Fatalf("alarm still latched after clean pass: %v", err) + } + if n := countAuditTransition(t, f.store, clean.RunID, store.TransitionAlarmClear); n != 1 { + t.Fatalf("alarm-clear audit count = %d, want 1", n) + } +} + +func countAuditTransition(t *testing.T, s *store.Store, runID int64, transition string) int { + t.Helper() + audits, err := s.ListRunAudit(context.Background(), runID) + if err != nil { + t.Fatalf("ListRunAudit(%d): %v", runID, err) + } + n := 0 + for _, a := range audits { + if a.Transition == transition { + n++ + } + } + return n +} + // TestVerifyRemoteMismatchIsLoudAndPreservesEvidence: a changed provider // checksum is reported per object, marks the run partial, and leaves // both the recorded fingerprint and the verification stamp untouched. diff --git a/tui/dashboard.go b/tui/dashboard.go index e9455e7..30bac28 100644 --- a/tui/dashboard.go +++ b/tui/dashboard.go @@ -46,6 +46,10 @@ type dashboardData struct { // that (volume, kind) pair. Used to fill the "last index" / "last sync" // columns of the volumes table. latestByVol map[int64]map[string]store.Run + // alarms are the standing per-destination alarms (#157, F30). A verify + // mismatch latches one until cleared; the dashboard shows them so the + // trust surface answers "am I safe?" with a red panel, not silence. + alarms []store.DestinationAlarm } type dashboardDataMsg struct { @@ -104,13 +108,42 @@ func (m *dashboardModel) View() string { } sections := []string{ m.renderAgentBlock(), + } + if alarms := m.renderAlarms(); alarms != "" { + sections = append(sections, alarms) + } + sections = append(sections, m.renderActiveRuns(), m.renderVolumes(), m.renderRecentRuns(), - } + ) return strings.Join(sections, "\n\n") } +// renderAlarms shows the standing per-destination alarms (#157, F30) high +// on the dashboard, right below agent health, because a latched verify +// mismatch is exactly the "am I safe?" answer the trust surface must not +// bury. Returns "" when nothing is in alarm so the section is absent on a +// healthy install rather than showing an empty green box. +func (m *dashboardModel) renderAlarms() string { + if len(m.data.alarms) == 0 { + return "" + } + header := styleErr.Render(fmt.Sprintf("Alarms (%d)", len(m.data.alarms))) + rows := [][]string{{"DESTINATION", "KIND", "SINCE", "RUN", "DETAIL"}} + for _, a := range m.data.alarms { + rows = append(rows, []string{ + a.Destination, + a.Kind, + whenAgo(sql.NullInt64{Int64: a.RaisedAtNs, Valid: true}, m.data.now), + fmt.Sprintf("#%d", a.RaisedRunID), + a.Detail, + }) + } + colours := []lipgloss.Color{colourFailure, "", "", "", ""} + return header + "\n" + renderTable(rows, colours) +} + func (m *dashboardModel) renderAgentBlock() string { header := styleHeader.Render("Agent") var body string @@ -266,6 +299,10 @@ func loadDashboardData(ctx context.Context, s *store.Store) (dashboardData, erro if err != nil { return dashboardData{}, fmt.Errorf("latest by volume: %w", err) } + alarms, err := s.ListDestinationAlarms(ctx) + if err != nil { + return dashboardData{}, fmt.Errorf("list alarms: %w", err) + } var active, recent []store.Run for _, r := range runs { if r.Status == store.RunStatusRunning { @@ -282,5 +319,6 @@ func loadDashboardData(ctx context.Context, s *store.Store) (dashboardData, erro activeRuns: active, recentRuns: recent, latestByVol: latestByVol, + alarms: alarms, }, nil } diff --git a/tui/format.go b/tui/format.go index 1dd0367..b9a3bd4 100644 --- a/tui/format.go +++ b/tui/format.go @@ -99,6 +99,10 @@ func glyphForStatus(status string) string { return "~" case "running": return "●" + case "refused": + return "⊘" + case "aborted": + return "⊗" default: return "·" } diff --git a/tui/styles.go b/tui/styles.go index 459f6a0..11e2d66 100644 --- a/tui/styles.go +++ b/tui/styles.go @@ -44,6 +44,14 @@ func statusColour(status string) lipgloss.Color { return colourWarning case "running": return colourRunning + case "refused": + // A refusal is a fail-closed safety gate; it must read as red so a + // month-dead backup disk produces visible red, not a muted dot. + return colourFailure + case "aborted": + // A reaped run never completed but did not fail; amber keeps it out + // of the green "you may close the laptop" set without crying failure. + return colourWarning default: return colourMuted } From 6b6e12888c382719843d7f84593e5429eae58e05 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:32:42 +0000 Subject: [PATCH 02/14] runs: address Copilot review on #174 (alarm-raise race, stderr tail) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FIX 1 (TOCTOU): RaiseDestinationAlarm now returns whether it actually created the latch, derived from the atomic INSERT ... ON CONFLICT DO NOTHING RowsAffected. applyVerifyAlarm keys rep.AlarmRaised off that return and no longer does a separate GetDestinationAlarm pre-read, so concurrent verify runs can't both report "newly raised". The lifecycle test now asserts the first raise reports raised=true and the idempotent second raise reports false. FIX 2 (stderr tail): appendStderr now retains the LAST maxStderrCapture bytes instead of the first, matching the "tail" wording already used on RunResult.Stderr, rcloneExitError, and the runRcloneOperation comment — rclone prints its fatal line last, so the tail is the diagnostic slice. The bound test now feeds many distinct lines and asserts the final line survives while an early one is trimmed. go vet, go test ./..., gofmt -l, and golangci-lint all clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- store/destination_alarms.go | 33 +++++++++++++++++++------------- store/destination_alarms_test.go | 12 ++++++++++-- sync/rclone.go | 17 +++++++--------- sync/rclone_test.go | 20 ++++++++++++++++--- sync/verify_remote.go | 17 ++++++++-------- 5 files changed, 63 insertions(+), 36 deletions(-) diff --git a/store/destination_alarms.go b/store/destination_alarms.go index 97bf36f..3b031a8 100644 --- a/store/destination_alarms.go +++ b/store/destination_alarms.go @@ -31,16 +31,23 @@ type DestinationAlarm struct { } // RaiseDestinationAlarm latches an alarm on destination when none is -// already active. The raise is idempotent: an existing alarm keeps its +// already active and reports whether this call actually created the latch +// (raised == true). The raise is idempotent: an existing alarm keeps its // original raised-at timestamp and run so a repeated detection does not -// reset "in alarm since" or append a duplicate audit row. An 'alarm-raise' -// runs_audit entry is written against raisedRunID (in the same transaction -// as the latch insert) only when a fresh latch is created, keeping the -// append-only trail to one row per distinct alarm episode. -func (s *Store) RaiseDestinationAlarm(ctx context.Context, destination, kind, detail string, raisedRunID int64) error { +// reset "in alarm since" or append a duplicate audit row, and raised comes +// back false. An 'alarm-raise' runs_audit entry is written against +// raisedRunID (in the same transaction as the latch insert) only on the +// creating call, keeping the append-only trail to one row per alarm episode. +// +// raised is derived from the INSERT's RowsAffected inside the same atomic +// ON CONFLICT DO NOTHING statement, so it is race-free: concurrent verify +// runs contend on the destination PRIMARY KEY and exactly one sees +// RowsAffected == 1. Callers must key their "newly raised" UX off this +// return, never off a separate pre-read. +func (s *Store) RaiseDestinationAlarm(ctx context.Context, destination, kind, detail string, raisedRunID int64) (raised bool, err error) { tx, err := s.db.BeginTx(ctx, nil) if err != nil { - return fmt.Errorf("begin raise alarm %q: %w", destination, err) + return false, fmt.Errorf("begin raise alarm %q: %w", destination, err) } defer func() { _ = tx.Rollback() }() @@ -51,25 +58,25 @@ func (s *Store) RaiseDestinationAlarm(ctx context.Context, destination, kind, de ON CONFLICT(destination) DO NOTHING `, destination, kind, detail, raisedRunID, atNs) if err != nil { - return fmt.Errorf("raise alarm %q: %w", destination, err) + return false, fmt.Errorf("raise alarm %q: %w", destination, err) } inserted, err := res.RowsAffected() if err != nil { - return fmt.Errorf("raise alarm %q rows affected: %w", destination, err) + return false, fmt.Errorf("raise alarm %q rows affected: %w", destination, err) } if inserted == 0 { // Already in alarm: keep the original latch untouched. - return tx.Commit() + return false, tx.Commit() } note := fmt.Sprintf("destination=%s %s", destination, detail) if err := appendRunAuditTx(ctx, tx, RunAuditEntry{RunID: raisedRunID, Transition: TransitionAlarmRaise, Note: note}, atNs); err != nil { - return err + return false, err } if err := tx.Commit(); err != nil { - return fmt.Errorf("commit raise alarm %q: %w", destination, err) + return false, fmt.Errorf("commit raise alarm %q: %w", destination, err) } - return nil + return true, nil } // ClearDestinationAlarm clears the active alarm on destination, if any, diff --git a/store/destination_alarms_test.go b/store/destination_alarms_test.go index 1c2c11e..2f8d90d 100644 --- a/store/destination_alarms_test.go +++ b/store/destination_alarms_test.go @@ -15,9 +15,13 @@ func TestDestinationAlarmLifecycle(t *testing.T) { if err != nil { t.Fatalf("BeginRemoteVerifyRun: %v", err) } - if err := s.RaiseDestinationAlarm(ctx, "offsite", AlarmKindVerifyMismatch, "mismatched=1 missing=0", first); err != nil { + raised, err := s.RaiseDestinationAlarm(ctx, "offsite", AlarmKindVerifyMismatch, "mismatched=1 missing=0", first) + if err != nil { t.Fatalf("RaiseDestinationAlarm: %v", err) } + if !raised { + t.Fatal("first raise reported not-raised, want a fresh latch") + } got, err := s.GetDestinationAlarm(ctx, "offsite") if err != nil { t.Fatalf("GetDestinationAlarm: %v", err) @@ -33,9 +37,13 @@ func TestDestinationAlarmLifecycle(t *testing.T) { if err != nil { t.Fatalf("BeginRemoteVerifyRun: %v", err) } - if err := s.RaiseDestinationAlarm(ctx, "offsite", AlarmKindVerifyMismatch, "mismatched=2 missing=1", second); err != nil { + reraised, err := s.RaiseDestinationAlarm(ctx, "offsite", AlarmKindVerifyMismatch, "mismatched=2 missing=1", second) + if err != nil { t.Fatalf("re-raise: %v", err) } + if reraised { + t.Fatal("second raise reported a fresh latch, want not-raised (idempotent)") + } got, err = s.GetDestinationAlarm(ctx, "offsite") if err != nil { t.Fatalf("GetDestinationAlarm after re-raise: %v", err) diff --git a/sync/rclone.go b/sync/rclone.go index aef11ff..9a4ab86 100644 --- a/sync/rclone.go +++ b/sync/rclone.go @@ -280,7 +280,7 @@ type FailedFile struct { const maxFailedFiles = 100 // maxStderrCapture bounds RunResult.Stderr. rclone's fatal diagnostics are -// a line or two; 4 KiB keeps a useful head of a chattier failure without +// a line or two; 4 KiB keeps a useful tail of a chattier failure without // letting a misbehaving child stream unbounded bytes into the run row. const maxStderrCapture = 4 << 10 @@ -621,15 +621,12 @@ func applyStatsEvent(result *RunResult, st *rcloneStats, onProgress func(runeven } } -// appendStderr accumulates a non-JSON stderr line into result.Stderr up to -// maxStderrCapture bytes. Once the cap is reached further lines are -// dropped — the head of a fatal diagnostic is what identifies the failure, -// so keeping the first bytes is more useful than a tail. Blank lines are -// skipped so the capture stays dense. +// appendStderr accumulates a non-JSON stderr line into result.Stderr, +// keeping the last maxStderrCapture bytes. rclone prints its fatal error +// line last, so the tail is the most diagnostic slice to retain when a +// chatty run overflows the bound; older lines are trimmed from the front. +// Blank lines are skipped so the capture stays dense. func appendStderr(result *RunResult, line []byte) { - if len(result.Stderr) >= maxStderrCapture { - return - } trimmed := strings.TrimSpace(string(line)) if trimmed == "" { return @@ -639,6 +636,6 @@ func appendStderr(result *RunResult, line []byte) { } result.Stderr += trimmed if len(result.Stderr) > maxStderrCapture { - result.Stderr = result.Stderr[:maxStderrCapture] + result.Stderr = result.Stderr[len(result.Stderr)-maxStderrCapture:] } } diff --git a/sync/rclone_test.go b/sync/rclone_test.go index c8eb9e7..efb52e1 100644 --- a/sync/rclone_test.go +++ b/sync/rclone_test.go @@ -2,6 +2,7 @@ package sync import ( "context" + "fmt" "os" "os/exec" "path/filepath" @@ -95,16 +96,29 @@ func TestParseJSONLogCapturesFatalLevelAndRawStderr(t *testing.T) { } // TestParseJSONLogStderrBounded: a pathological non-JSON stream can't -// balloon RunResult.Stderr past the cap. +// balloon RunResult.Stderr past the cap, and the capture keeps the tail +// (rclone's fatal line prints last) — the final line survives while an +// early one is trimmed from the front. func TestParseJSONLogStderrBounded(t *testing.T) { + lines := make([]string, 0, 2000) + for i := 0; i < 2000; i++ { + lines = append(lines, fmt.Sprintf("stderr line %04d", i)) + } var r RunResult - parseJSONLog(strings.NewReader(strings.Repeat("x", 10*maxStderrCapture)), &r, nil) + parseJSONLog(strings.NewReader(strings.Join(lines, "\n")), &r, nil) + if len(r.Stderr) == 0 { - t.Fatal("Stderr empty, want the head of the long line") + t.Fatal("Stderr empty, want the tail of the stream") } if len(r.Stderr) > maxStderrCapture { t.Fatalf("Stderr len = %d, want <= %d", len(r.Stderr), maxStderrCapture) } + if !strings.Contains(r.Stderr, "stderr line 1999") { + t.Fatal("Stderr lost the final line — tail not kept") + } + if strings.Contains(r.Stderr, "stderr line 0000") { + t.Fatal("Stderr kept the first line — want the tail only") + } } // TestDisplayErrors pins the summary error count: rclone's per-file count diff --git a/sync/verify_remote.go b/sync/verify_remote.go index 4810f15..5809b9b 100644 --- a/sync/verify_remote.go +++ b/sync/verify_remote.go @@ -430,15 +430,16 @@ func applyVerifyAlarm(ctx context.Context, s *store.Store, rep *RemoteVerifyRepo if !rep.Clean() { detail := fmt.Sprintf("objects mismatched=%d missing=%d, packs mismatched=%d missing=%d", len(rep.Mismatched), len(rep.Missing), len(rep.PackMismatched), len(rep.PacksMissing)) - // AlarmRaised reports only a *new* latch, so the CLI can shout on - // first detection without re-shouting every subsequent pass. - switch _, err := s.GetDestinationAlarm(ctx, rep.Destination); { - case store.IsNotFound(err): - rep.AlarmRaised = true - case err != nil: - return fmt.Errorf("check standing alarm for %q: %w", rep.Destination, err) + // AlarmRaised reflects only a *newly* created latch, taken from the + // atomic insert itself — so the CLI shouts on first detection but a + // concurrent second raise (which found the latch already there) + // stays quiet. No pre-read: that would be a TOCTOU race. + raised, err := s.RaiseDestinationAlarm(ctx, rep.Destination, store.AlarmKindVerifyMismatch, detail, rep.RunID) + if err != nil { + return fmt.Errorf("raise standing alarm for %q: %w", rep.Destination, err) } - return s.RaiseDestinationAlarm(ctx, rep.Destination, store.AlarmKindVerifyMismatch, detail, rep.RunID) + rep.AlarmRaised = raised + return nil } cleared, err := s.ClearDestinationAlarm(ctx, rep.Destination, rep.RunID, "") if err != nil { From 7b4994cade0a5195e8f6d66ae82d29c71ee2cbeb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:53:01 +0000 Subject: [PATCH 03/14] status: shared coverage + durability query layer, CLI command, TUI panel (#159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two standing questions no surface could answer — "is every configured target caught up?" (F16) and "what is durable where, and what could I offload?" (F17/F23) — now have one answer, built once and shown two ways. New `status` package: a read-only query layer that produces, per configured volume and per (volume × target), the facts both surfaces render: - last successful sync + age, coloured against THAT pair's own sync_every (late is relative to the pair's cadence, not a global constant); - the #157 standing states — alarm (latched verify mismatch, red), refused (a gate that regressed from a prior success, red), and needs-bootstrap (a first-use `--init` refusal on a pair that never succeeded, amber); - durability: whether the target's vector covers this node's local content origin, the verify method behind it, and evidence age vs offload_max_evidence_age (stale flagged); - offload readiness: bytes/files currently passing the gate per volume ("N GB offloadable now" — the F17 decision-support without the side effects of `offload --dry-run`). Each cell carries a Level (neutral/ok/amber/red); the report's worst level is the single "am I safe?" answer. Step 1 — `squirrel status [volume]`: renders the grid and exits with the worst level (0 green, 1 amber, 2 red) so the same command scripts a health check. A read-only *question* per ux-principle 2; it mutates nothing. Step 2 — the TUI dashboard's top panel is now this grid (replacing the single LAST SYNC cell that hid a week-behind target behind a fresh check), built from the SAME query layer, with the STATE and DURABLE cells coloured. Offload readiness reuses the real gate (offload.Readiness shares loadGate + selectCandidates + gate.check), so the totals match what an offload would move; it reads only the index — no run, no disk, no marker check. No schema change: this is a query/display layer over existing tables. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/main.go | 19 ++- cmd/squirrel/root.go | 1 + cmd/squirrel/status.go | 102 +++++++++++++ cmd/squirrel/status_test.go | 108 ++++++++++++++ offload/readiness.go | 91 ++++++++++++ status/build.go | 241 ++++++++++++++++++++++++++++++ status/build_test.go | 239 ++++++++++++++++++++++++++++++ status/durability.go | 106 ++++++++++++++ status/labels.go | 166 +++++++++++++++++++++ status/logic_test.go | 162 ++++++++++++++++++++ status/runs.go | 101 +++++++++++++ status/status.go | 285 ++++++++++++++++++++++++++++++++++++ tui/dashboard.go | 159 +++++++++++++------- tui/dashboard_test.go | 23 +-- tui/tui.go | 2 +- 15 files changed, 1737 insertions(+), 68 deletions(-) create mode 100644 cmd/squirrel/status.go create mode 100644 cmd/squirrel/status_test.go create mode 100644 offload/readiness.go create mode 100644 status/build.go create mode 100644 status/build_test.go create mode 100644 status/durability.go create mode 100644 status/labels.go create mode 100644 status/logic_test.go create mode 100644 status/runs.go create mode 100644 status/status.go diff --git a/cmd/squirrel/main.go b/cmd/squirrel/main.go index 223fab0..bea55e0 100644 --- a/cmd/squirrel/main.go +++ b/cmd/squirrel/main.go @@ -2,16 +2,33 @@ package main import ( "context" + "errors" + "fmt" "os" "os/signal" "syscall" ) +// exitCodeError carries a specific process exit status out of a command's +// RunE. `squirrel status` uses it to report its scriptable green/amber/red +// contract (exit 0/1/2), which the generic "any error ⇒ exit 1" path below +// cannot express. errors.As recovers the code here; the command that +// returns it silences cobra's error printing so nothing but the code +// leaves the process. +type exitCodeError struct{ code int } + +func (e exitCodeError) Error() string { return fmt.Sprintf("exit status %d", e.code) } + func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - if err := newRootCmd().ExecuteContext(ctx); err != nil { + err := newRootCmd().ExecuteContext(ctx) + var ec exitCodeError + if errors.As(err, &ec) { + os.Exit(ec.code) + } + if err != nil { os.Exit(1) } } diff --git a/cmd/squirrel/root.go b/cmd/squirrel/root.go index 4bb9def..9083efb 100644 --- a/cmd/squirrel/root.go +++ b/cmd/squirrel/root.go @@ -46,6 +46,7 @@ func newRootCmd() *cobra.Command { root.AddCommand(newRunsCmd()) root.AddCommand(newHooksCmd()) root.AddCommand(newVolumesCmd()) + root.AddCommand(newStatusCmd()) root.AddCommand(newSyncCmd()) root.AddCommand(newRestoreCmd()) root.AddCommand(newOffloadCmd()) diff --git a/cmd/squirrel/status.go b/cmd/squirrel/status.go new file mode 100644 index 0000000..8bae4fc --- /dev/null +++ b/cmd/squirrel/status.go @@ -0,0 +1,102 @@ +package main + +import ( + "fmt" + "io" + "text/tabwriter" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/status" +) + +// newStatusCmd returns the `squirrel status [volume]` cobra command: a +// read-only "am I safe?" grid answering, per (volume × target), whether +// every configured target is caught up and whether the volume's local +// content is durable where its offload policy needs it (friction-log F16, +// F17, F23). It mutates nothing. The process exit code reflects the worst +// state — 0 green, 1 amber, 2 red — so the same command scripts a health +// check without parsing the grid. +func newStatusCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "status [volume]", + Short: "Show per-target sync coverage and durability, and the offload-ready total", + Args: cobra.MaximumNArgs(1), + // The amber/red signal is carried out as an exitCodeError; silence + // cobra's own error printing so the grid is the only output and the + // signal shows up purely in the exit status. + SilenceErrors: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + var volume string + if len(args) == 1 { + volume = args[0] + } + return runStatus(cmd, volume) + }, + } + return cmd +} + +func runStatus(cmd *cobra.Command, volume string) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + if volume != "" { + if _, ok := cfg.Volumes[volume]; !ok { + return fmt.Errorf("unknown volume %q (declare it in %s)", volume, cfg.Path) + } + } + s, err := openStore(cmd, cfg) + if err != nil { + return err + } + defer s.Close() + + rep, err := status.Build(cmd.Context(), s, cfg) + if err != nil { + return err + } + worst := renderStatus(cmd.OutOrStdout(), rep, volume) + if code := worst.ExitCode(); code != 0 { + return exitCodeError{code: code} + } + return nil +} + +// renderStatus prints the grid and returns the worst level across the +// volumes it printed. When volume is non-empty only that volume is shown, +// and only its level drives the return (so a scripted per-volume check +// isn't reddened by an unrelated volume). +func renderStatus(w io.Writer, rep status.Report, volume string) status.Level { + worst := status.LevelNeutral + for _, v := range rep.Volumes { + if volume != "" && v.Name != volume { + continue + } + renderStatusVolume(w, v) + if v.Level() > worst { + worst = v.Level() + } + } + fmt.Fprintf(w, "overall: %s\n", status.TrafficLight(worst)) + return worst +} + +// renderStatusVolume prints one volume's header, its target grid, and its +// offload-readiness line, reusing the shared status labels so the wording +// matches the TUI exactly. +func renderStatusVolume(w io.Writer, v status.VolumeStatus) { + fmt.Fprintf(w, "\n%s %s [%s]\n", v.Name, v.Path, status.TrafficLight(v.Level())) + fmt.Fprintf(w, " index: %s\n", status.IndexLabel(v)) + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, " TARGET\tROLE\tLAST SYNC\tSTATE\tDURABLE\tMETHOD\tEVIDENCE") + for _, t := range v.Targets { + fmt.Fprintf(tw, " %s\t%s\t%s\t%s\t%s\t%s\t%s\n", + t.Name, status.RoleLabel(t), status.LastSyncLabel(t), status.StateLabel(t), + status.DurableLabel(t), status.MethodLabel(t), status.EvidenceLabel(t)) + } + _ = tw.Flush() + fmt.Fprintf(w, " %s\n", status.OffloadLabel(v.Offload)) +} diff --git a/cmd/squirrel/status_test.go b/cmd/squirrel/status_test.go new file mode 100644 index 0000000..16cea9c --- /dev/null +++ b/cmd/squirrel/status_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// writeStatusConfig lays down a volume with a sync destination, plus its +// DB, and returns the config path. Callers index the volume themselves. +func writeStatusConfig(t *testing.T) (configPath string) { + t.Helper() + root := t.TempDir() + volumeDir := filepath.Join(root, "pics") + if err := os.MkdirAll(volumeDir, 0o755); err != nil { + t.Fatalf("mkdir volume: %v", err) + } + writeTestFile(t, filepath.Join(volumeDir, "a.txt"), "hello") + writeTestFile(t, filepath.Join(volumeDir, "b.txt"), "world") + + destDir := filepath.Join(root, "dst") + if err := os.MkdirAll(destDir, 0o755); err != nil { + t.Fatalf("mkdir dest: %v", err) + } + dbPath := filepath.Join(root, "index.db") + configPath = filepath.Join(root, "config.toml") + body := "" + + "db = \"" + dbPath + "\"\n\n" + + "[destinations.scratch]\n" + + "type = \"local\"\n" + + "root = \"" + destDir + "\"\n\n" + + "[volumes.pics]\n" + + "path = \"" + volumeDir + "\"\n" + + "sync_to = [\"scratch\"]\n" + + "sync_every = \"1h\"\n" + if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return configPath +} + +// TestCLIStatusNeverSyncedIsAmber: a configured but never-synced target +// leaves the volume amber, and the command exits with code 1 carried by an +// exitCodeError. +func TestCLIStatusNeverSyncedIsAmber(t *testing.T) { + cfg := writeStatusConfig(t) + runCLI(t, "--config", cfg, "index", "pics") + + out, err := runCLIExpectErr(t, "--config", cfg, "status") + var ec exitCodeError + if !errors.As(err, &ec) { + t.Fatalf("expected exitCodeError, got %v", err) + } + if ec.code != 1 { + t.Fatalf("exit code = %d, want 1 (amber)", ec.code) + } + if !strings.Contains(out, "pics") || !strings.Contains(out, "scratch") { + t.Fatalf("grid missing volume/target:\n%s", out) + } + if !strings.Contains(out, "never") { + t.Fatalf("never-synced target should read 'never':\n%s", out) + } + if !strings.Contains(out, "overall: amber") { + t.Fatalf("summary should be amber:\n%s", out) + } + // Cobra's own "Error:" line must be silenced — only the grid prints. + if strings.Contains(out, "Error:") { + t.Fatalf("cobra error line leaked into output:\n%s", out) + } +} + +// TestCLIStatusAfterSyncIsGreen: once a fresh sync lands, the pair is +// caught up within cadence and the command exits 0. +func TestCLIStatusAfterSyncIsGreen(t *testing.T) { + requireRcloneCLI(t) + cfg := writeStatusConfig(t) + runCLI(t, "--config", cfg, "index", "pics") + runCLI(t, "--config", cfg, "sync", "pics", "--init") + + out := runCLI(t, "--config", cfg, "status") + if !strings.Contains(out, "overall: green") { + t.Fatalf("expected green after a fresh sync:\n%s", out) + } + if !strings.Contains(out, "ok") { + t.Fatalf("expected an 'ok' target state:\n%s", out) + } +} + +// TestCLIStatusUnknownVolume errors clearly on a volume not in config. +func TestCLIStatusUnknownVolume(t *testing.T) { + cfg := writeStatusConfig(t) + _, err := runCLIExpectErr(t, "--config", cfg, "status", "nope") + if err == nil || !strings.Contains(err.Error(), `unknown volume "nope"`) { + t.Fatalf("expected unknown-volume error, got %v", err) + } +} + +// TestCLIStatusRequiresConfig: status is a question about configured +// coverage, so a missing config is a clear error, not an empty grid. +func TestCLIStatusRequiresConfig(t *testing.T) { + missing := filepath.Join(t.TempDir(), "no-config.toml") + _, err := runCLIExpectErr(t, "--config", missing, "status") + if err == nil || !strings.Contains(err.Error(), "no config at") { + t.Fatalf("expected missing-config error, got %v", err) + } +} diff --git a/offload/readiness.go b/offload/readiness.go new file mode 100644 index 0000000..a3e251d --- /dev/null +++ b/offload/readiness.go @@ -0,0 +1,91 @@ +package offload + +import ( + "context" + "time" + + "github.com/mbertschler/squirrel/store" +) + +// ReadinessOptions selects the volume and durability policy for a +// Readiness evaluation. It mirrors the durability-relevant subset of +// Options: no path/age selectors (readiness is always whole-volume) and no +// DryRun flag (readiness never touches disk or opens a run). +type ReadinessOptions struct { + // VolumeID is the local volumes row the gate is evaluated against. + VolumeID int64 + // Require is the volume's offload_requires policy. Empty means the + // volume has no offload policy, and Readiness reports Applicable=false + // without evaluating anything. + Require []string + // MaxEvidenceAge is the volume's offload_max_evidence_age; it feeds the + // same time-based staleness gate an offload would apply. Zero disables + // the staleness policy. + MaxEvidenceAge time.Duration +} + +// ReadinessReport totals what an offload of the whole volume would move +// right now. OffloadableFiles/Bytes are the subset of the present set that +// passes the durability gate; PresentFiles/Bytes are the whole gated +// candidate set, so a UI can render "N of M". +type ReadinessReport struct { + // Applicable is false when the volume declares no offload policy — the + // caller can then distinguish "nothing offloadable" from "offload not + // configured here" without inspecting the totals. + Applicable bool + OffloadableFiles int + OffloadableBytes int64 + PresentFiles int + PresentBytes int64 +} + +// Readiness reports how many present files, and how many bytes, currently +// pass the offload gate for the volume — the "N GB offloadable now" +// decision-support number (friction log F17) without the side effects of +// `offload --dry-run`. It reads only the local index (durability vectors, +// scan-back fingerprints), opens no run, reads no disk bytes, validates no +// volume marker, and deletes nothing, so it is safe to call from a +// read-only introspection surface at any time. +// +// The gate is evaluated exactly as Offload's dry-run path evaluates it — +// same loadGate, same candidate selection, same per-file check — so the +// totals match what an offload would actually move. +func Readiness(ctx context.Context, s *store.Store, opts ReadinessOptions) (ReadinessReport, error) { + if len(opts.Require) == 0 { + return ReadinessReport{Applicable: false}, nil + } + now := time.Now() + g, err := loadGate(ctx, s, opts.VolumeID, opts.Require, now.UnixNano(), opts.MaxEvidenceAge) + if err != nil { + return ReadinessReport{}, err + } + rows, err := s.LoadVolumeIndex(ctx, opts.VolumeID) + if err != nil { + return ReadinessReport{}, err + } + candidates, _ := selectCandidates(rows, nil, 0, now) + return tallyReadiness(ctx, g, candidates) +} + +// tallyReadiness runs the gate over every candidate and accumulates the +// present and offloadable totals. Split from Readiness so the loop body +// stays small and the context-cancellation check has one home. +func tallyReadiness(ctx context.Context, g *gate, candidates []store.FileRow) (ReadinessReport, error) { + rep := ReadinessReport{Applicable: true} + for _, row := range candidates { + if err := ctx.Err(); err != nil { + return ReadinessReport{}, err + } + rep.PresentFiles++ + rep.PresentBytes += row.SizeBytes + failures, err := g.check(ctx, row) + if err != nil { + return ReadinessReport{}, err + } + if len(failures) == 0 { + rep.OffloadableFiles++ + rep.OffloadableBytes += row.SizeBytes + } + } + return rep, nil +} diff --git a/status/build.go b/status/build.go new file mode 100644 index 0000000..ca37431 --- /dev/null +++ b/status/build.go @@ -0,0 +1,241 @@ +package status + +import ( + "context" + "database/sql" + "fmt" + "sort" + "strings" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/offload" + "github.com/mbertschler/squirrel/store" +) + +// lateFactor is how many cadence periods a pair may fall behind before its +// staleness reads red rather than amber. Within one cadence is on-time +// (green); up to lateFactor cadences is late (amber); beyond is a +// destination that has effectively stopped (red). +const lateFactor = 3 + +// Build produces the status Report for this node: every config-declared +// volume with its per-target coverage and durability. It is read-only — +// no run rows, no disk reads, no mutation — so it is safe to call from any +// introspection surface at any time. cfg is the source of truth for which +// volumes and targets should exist; the store supplies the observations. +func Build(ctx context.Context, s *store.Store, cfg *config.Config) (Report, error) { + rep := Report{Now: time.Now()} + self, err := s.GetSelfNode(ctx) + if err != nil { + return Report{}, fmt.Errorf("look up self node: %w", err) + } + alarms, err := s.ListDestinationAlarms(ctx) + if err != nil { + return Report{}, fmt.Errorf("list destination alarms: %w", err) + } + alarmByDest := indexAlarms(alarms) + for _, name := range sortedVolumeNames(cfg) { + vs, err := buildVolume(ctx, s, cfg, self, alarmByDest, cfg.Volumes[name], rep.Now) + if err != nil { + return Report{}, err + } + rep.Volumes = append(rep.Volumes, vs) + } + return rep, nil +} + +// indexAlarms keys the active alarms by destination name for O(1) lookup +// per target. Alarms latch per destination, not per volume, so one entry +// applies to every volume that names the destination. +func indexAlarms(alarms []store.DestinationAlarm) map[string]store.DestinationAlarm { + out := make(map[string]store.DestinationAlarm, len(alarms)) + for _, a := range alarms { + out[a.Destination] = a + } + return out +} + +// sortedVolumeNames returns the config's volume names in deterministic +// order so the report (and the CLI grid) is stable across runs. +func sortedVolumeNames(cfg *config.Config) []string { + names := make([]string, 0, len(cfg.Volumes)) + for name := range cfg.Volumes { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// buildVolume assembles one volume's status. A volume with no index row +// (declared but never indexed) still lists its planned targets so the +// grid shows the configured coverage rather than a blank line. +func buildVolume(ctx context.Context, s *store.Store, cfg *config.Config, self store.Node, alarmByDest map[string]store.DestinationAlarm, vol *config.Volume, now time.Time) (VolumeStatus, error) { + vs := VolumeStatus{Name: vol.Name, Path: vol.Path} + dbVol, err := s.GetVolumeByName(ctx, vol.Name) + if store.IsNotFound(err) { + vs.Targets, err = buildTargets(ctx, s, cfg, self, alarmByDest, vol, 0, false, 0, now) + return vs, err + } + if err != nil { + return VolumeStatus{}, fmt.Errorf("look up volume %q: %w", vol.Name, err) + } + vs.Indexed = true + vs.LastIndexAgo, err = lastIndexAge(ctx, s, dbVol.ID, now) + if err != nil { + return VolumeStatus{}, err + } + selfMax, err := selfOriginMax(ctx, s, dbVol.ID, self.ID) + if err != nil { + return VolumeStatus{}, err + } + vs.Targets, err = buildTargets(ctx, s, cfg, self, alarmByDest, vol, dbVol.ID, true, selfMax, now) + if err != nil { + return VolumeStatus{}, err + } + readiness, err := offload.Readiness(ctx, s, offload.ReadinessOptions{ + VolumeID: dbVol.ID, Require: vol.OffloadRequires, MaxEvidenceAge: vol.OffloadMaxEvidenceAge, + }) + if err != nil { + return VolumeStatus{}, fmt.Errorf("offload readiness for %q: %w", vol.Name, err) + } + vs.Offload = OffloadReadiness(readiness) + return vs, nil +} + +// buildTargets builds a TargetStatus for every target in the union of the +// volume's sync_to and offload_requires, sync targets first in config +// order then relayed offload targets, deduplicated. +func buildTargets(ctx context.Context, s *store.Store, cfg *config.Config, self store.Node, alarmByDest map[string]store.DestinationAlarm, vol *config.Volume, volID int64, indexed bool, selfMax int64, now time.Time) ([]TargetStatus, error) { + var out []TargetStatus + for _, name := range orderedTargets(vol) { + ts, err := buildTarget(ctx, s, cfg, self, alarmByDest, vol, name, volID, indexed, selfMax, now) + if err != nil { + return nil, err + } + out = append(out, ts) + } + return out, nil +} + +// orderedTargets returns the union of sync_to and offload_requires, +// sync_to first in declared order, then required targets not already +// listed, deduplicated. +func orderedTargets(vol *config.Volume) []string { + seen := make(map[string]struct{}) + var out []string + for _, group := range [][]string{vol.SyncTo, vol.OffloadRequires} { + for _, name := range group { + if _, dup := seen[name]; dup { + continue + } + seen[name] = struct{}{} + out = append(out, name) + } + } + return out +} + +// buildTarget assembles one (volume × target) cell: coverage freshness, +// standing state, and durability. +func buildTarget(ctx context.Context, s *store.Store, cfg *config.Config, self store.Node, alarmByDest map[string]store.DestinationAlarm, vol *config.Volume, name string, volID int64, indexed bool, selfMax int64, now time.Time) (TargetStatus, error) { + ts := TargetStatus{ + Name: name, + Kind: targetKind(cfg, name), + SyncTarget: contains(vol.SyncTo, name), + Required: contains(vol.OffloadRequires, name), + Cadence: vol.SyncEvery, + } + var lastTerminal *store.Run + if indexed { + lastSync, err := latestSuccessfulSync(ctx, s, volID, name) + if err != nil { + return TargetStatus{}, err + } + ts.LastSyncAgo = ageOf(lastSync, now) + if lastTerminal, err = latestTerminalSync(ctx, s, volID, name); err != nil { + return TargetStatus{}, err + } + if lastTerminal != nil { + ts.LastOutcome = lastTerminal.Status + } + } + if alarm, ok := alarmByDest[name]; ok { + ts.Standing = StandingAlarm + ts.AlarmDetail = alarm.Detail + } else { + ts.Standing = classifyStanding(lastTerminal, ts.LastSyncAgo != nil) + } + ts.SyncLevel = syncLevel(ts, lastTerminal) + if indexed { + dur, err := buildDurability(ctx, s, volID, self, selfMax, vol, name, now) + if err != nil { + return TargetStatus{}, err + } + ts.Durability = dur + } + return ts, nil +} + +// classifyStanding derives the standing state from the pair's latest +// terminal sync run. A refusal that regressed from a prior success is a +// broken destination (red); a bootstrap-style refusal on a pair that has +// never succeeded is a fresh destination awaiting one-time `--init` (amber). +func classifyStanding(lastTerminal *store.Run, hadSuccess bool) Standing { + if lastTerminal == nil || lastTerminal.Status != store.RunStatusRefused { + return StandingNone + } + if !hadSuccess && isBootstrapRefusal(lastTerminal.Error) { + return StandingNeedsBootstrap + } + return StandingRefused +} + +// isBootstrapRefusal reports whether a refused run's error is a first-use +// bootstrap refusal (a missing destination marker or an unconnected kopia +// repository), both of which tell the operator to "re-run with --init". +// The layout guard and the init-over-a-different-volume refusal carry no +// "--init" hint, so they stay classified as plain refusals. +func isBootstrapRefusal(errVal sql.NullString) bool { + return errVal.Valid && strings.Contains(errVal.String, "--init") +} + +// syncLevel scores the coverage dimension: standing states first, then a +// failed transfer, then freshness relative to the pair's own cadence. A +// relayed target (not pushed to by this node) has no coverage dimension — +// its safety is entirely its durability — so it reads neutral. +func syncLevel(ts TargetStatus, lastTerminal *store.Run) Level { + switch ts.Standing { + case StandingAlarm, StandingRefused: + return LevelCritical + case StandingNeedsBootstrap: + return LevelWarn + } + if lastTerminal != nil && lastTerminal.Status == store.RunStatusFailed { + return LevelCritical + } + if !ts.SyncTarget { + return LevelNeutral + } + if ts.LastSyncAgo == nil { + return LevelWarn + } + return cadenceLevel(*ts.LastSyncAgo, ts.Cadence) +} + +// cadenceLevel colours a successful sync's age against the pair's cadence: +// within cadence is green, up to lateFactor cadences amber, beyond red. A +// zero cadence (no scheduled sync) cannot be late, so any success is green. +func cadenceLevel(age, cadence time.Duration) Level { + if cadence <= 0 { + return LevelOK + } + switch { + case age <= cadence: + return LevelOK + case age <= lateFactor*cadence: + return LevelWarn + default: + return LevelCritical + } +} diff --git a/status/build_test.go b/status/build_test.go new file mode 100644 index 0000000..2e1c352 --- /dev/null +++ b/status/build_test.go @@ -0,0 +1,239 @@ +package status + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/index" + "github.com/mbertschler/squirrel/store" +) + +func setupStore(t *testing.T) *store.Store { + t.Helper() + s, err := store.Open(filepath.Join(t.TempDir(), "status.db")) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +// indexTree writes three files into a fresh directory and indexes it as +// volume name, returning the root and the index run. +func indexTree(t *testing.T, s *store.Store, name string) (string, index.Report) { + t.Helper() + root := t.TempDir() + for _, n := range []string{"a.txt", "b.txt", "c.txt"} { + if err := os.WriteFile(filepath.Join(root, n), []byte("content-"+n), 0o644); err != nil { + t.Fatalf("write %s: %v", n, err) + } + } + rep, err := index.Index(context.Background(), s, root, index.Options{Name: name, Workers: 2}) + if err != nil || rep.Errors > 0 { + t.Fatalf("index %s: err=%v errors=%v", name, err, rep.ErrorList) + } + return root, rep +} + +// cfgFor builds a minimal config with one volume and its targets. Every +// volume gets a one-hour sync cadence so a fresh sync reads as on-time. +func cfgFor(volName, root string, syncTo, requires []string, dests, nodes []string) *config.Config { + cfg := &config.Config{ + Volumes: map[string]*config.Volume{}, + Destinations: map[string]*config.Destination{}, + Nodes: map[string]*config.Node{}, + } + cfg.Volumes[volName] = &config.Volume{ + Name: volName, + Path: root, + SyncTo: syncTo, + OffloadRequires: requires, + SyncEvery: time.Hour, + } + for _, d := range dests { + cfg.Destinations[d] = &config.Destination{Name: d, Type: "s3", Layout: config.LayoutContentAddressed} + } + for _, n := range nodes { + cfg.Nodes[n] = &config.Node{Name: n} + } + return cfg +} + +func recordSuccessfulSync(t *testing.T, s *store.Store, volID int64, dest string) { + t.Helper() + id, blocker, err := s.BeginSyncRunIfClear(context.Background(), store.SyncRunSpec{VolumeID: volID, Destination: dest}) + if err != nil || blocker != nil { + t.Fatalf("BeginSyncRunIfClear(%s): err=%v blocker=%+v", dest, err, blocker) + } + if err := s.FinishRun(context.Background(), id, store.RunStatusSuccess, "", 3); err != nil { + t.Fatalf("FinishRun(%s): %v", dest, err) + } +} + +// TestBuildFullyDurable exercises the happy path end to end: an indexed +// volume, a fresh successful sync, a content-verified durability vector +// covering the local origin, and an offload policy — everything green and +// every file offloadable. +func TestBuildFullyDurable(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + root, idx := indexTree(t, s, "photos") + v, err := s.GetVolumeByName(ctx, "photos") + if err != nil { + t.Fatalf("GetVolumeByName: %v", err) + } + self, err := s.GetSelfNode(ctx) + if err != nil { + t.Fatalf("GetSelfNode: %v", err) + } + if err := s.UpsertDestinationRunIDVerified(ctx, v.ID, "dest", self.ID, idx.RunID, store.VerifyMethodBlake3, false); err != nil { + t.Fatalf("seed vector: %v", err) + } + recordSuccessfulSync(t, s, v.ID, "dest") + + cfg := cfgFor("photos", root, []string{"dest"}, []string{"dest"}, []string{"dest"}, nil) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + if len(rep.Volumes) != 1 { + t.Fatalf("volumes = %d, want 1", len(rep.Volumes)) + } + vs := rep.Volumes[0] + if !vs.Indexed || vs.LastIndexAgo == nil { + t.Fatalf("volume not indexed/aged: %+v", vs) + } + if len(vs.Targets) != 1 { + t.Fatalf("targets = %d, want 1", len(vs.Targets)) + } + tg := vs.Targets[0] + if tg.Kind != KindDestination || !tg.SyncTarget || !tg.Required { + t.Errorf("target role wrong: %+v", tg) + } + if tg.Standing != StandingNone || tg.SyncLevel != LevelOK { + t.Errorf("target coverage = standing %v level %v, want none/ok", tg.Standing, tg.SyncLevel) + } + if tg.Durability == nil || !tg.Durability.LocalContent || !tg.Durability.Covered || tg.Durability.Method != store.VerifyMethodBlake3 { + t.Errorf("durability wrong: %+v", tg.Durability) + } + if !vs.Offload.Applicable || vs.Offload.OffloadableFiles != 3 || vs.Offload.PresentFiles != 3 { + t.Errorf("offload readiness wrong: %+v", vs.Offload) + } + if vs.Offload.OffloadableBytes != vs.Offload.PresentBytes || vs.Offload.OffloadableBytes == 0 { + t.Errorf("offload bytes wrong: %+v", vs.Offload) + } + if rep.Level() != LevelOK { + t.Errorf("report level = %v, want ok", rep.Level()) + } +} + +// TestBuildNeedsBootstrap: a refused sync whose reason names --init, with +// no prior success, is the amber one-time-bootstrap state — not red. +func TestBuildNeedsBootstrap(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + root, _ := indexTree(t, s, "docs") + v, _ := s.GetVolumeByName(ctx, "docs") + id, err := s.BeginRun(ctx, store.RunKindSync, v.ID, "dest", false) + if err != nil { + t.Fatalf("BeginRun: %v", err) + } + if err := s.FinishRun(ctx, id, store.RunStatusRefused, "destination \"dest\" has no marker — re-run with --init to bootstrap", 0); err != nil { + t.Fatalf("FinishRun: %v", err) + } + cfg := cfgFor("docs", root, []string{"dest"}, nil, []string{"dest"}, nil) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + tg := rep.Volumes[0].Targets[0] + if tg.Standing != StandingNeedsBootstrap { + t.Errorf("standing = %v, want needs-bootstrap", tg.Standing) + } + if rep.Level() != LevelWarn { + t.Errorf("report level = %v, want amber", rep.Level()) + } +} + +// TestBuildAlarm: a latched destination alarm reddens its target. +func TestBuildAlarm(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + root, idx := indexTree(t, s, "media") + if err := s.RaiseDestinationAlarm(ctx, "dest", store.AlarmKindVerifyMismatch, "checksum mismatch on object", idx.RunID); err != nil { + t.Fatalf("RaiseDestinationAlarm: %v", err) + } + cfg := cfgFor("media", root, []string{"dest"}, nil, []string{"dest"}, nil) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + tg := rep.Volumes[0].Targets[0] + if tg.Standing != StandingAlarm || tg.AlarmDetail == "" { + t.Errorf("standing = %v detail=%q, want alarm", tg.Standing, tg.AlarmDetail) + } + if rep.Level() != LevelCritical { + t.Errorf("report level = %v, want red", rep.Level()) + } +} + +// TestBuildNeverIndexed: a configured-but-unindexed volume is amber and +// lists its planned targets rather than rendering blank (F4). +func TestBuildNeverIndexed(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + cfg := cfgFor("photos", "/does/not/matter", []string{"nas"}, nil, nil, []string{"nas"}) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + vs := rep.Volumes[0] + if vs.Indexed { + t.Errorf("volume should not be indexed") + } + if len(vs.Targets) != 1 || vs.Targets[0].Kind != KindNode { + t.Errorf("planned targets wrong: %+v", vs.Targets) + } + if vs.Level() != LevelWarn { + t.Errorf("volume level = %v, want amber", vs.Level()) + } +} + +// TestBuildRelayedRequiredTarget: a target named only in offload_requires +// (never in sync_to) reads as relayed for coverage but is scored on its +// durability; with no evidence yet it is amber, not red. +func TestBuildRelayedRequiredTarget(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + root, _ := indexTree(t, s, "photos") + cfg := cfgFor("photos", root, []string{"nas"}, []string{"nas", "s3archive"}, nil, []string{"nas"}) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + var relayed *TargetStatus + for i := range rep.Volumes[0].Targets { + if rep.Volumes[0].Targets[i].Name == "s3archive" { + relayed = &rep.Volumes[0].Targets[i] + } + } + if relayed == nil { + t.Fatalf("relayed target s3archive missing from grid") + } + if relayed.SyncTarget || !relayed.Required || relayed.Kind != KindRelayed { + t.Errorf("relayed target classification wrong: %+v", relayed) + } + if relayed.SyncLevel != LevelNeutral { + t.Errorf("relayed sync level = %v, want neutral", relayed.SyncLevel) + } + if relayed.Durability == nil || relayed.Durability.Covered { + t.Errorf("relayed durability should be uncovered: %+v", relayed.Durability) + } + if relayed.Level() != LevelWarn { + t.Errorf("relayed target level = %v, want amber (evidence not yet arrived)", relayed.Level()) + } +} diff --git a/status/durability.go b/status/durability.go new file mode 100644 index 0000000..e814a1a --- /dev/null +++ b/status/durability.go @@ -0,0 +1,106 @@ +package status + +import ( + "context" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" +) + +// buildDurability computes the local-content durability of a volume on one +// target. Durability is tracked when the target is offload-required or when +// a vector component for it already exists; a non-required target with no +// recorded evidence returns nil (nothing to report). +func buildDurability(ctx context.Context, s *store.Store, volID int64, self store.Node, selfMax int64, vol *config.Volume, name string, now time.Time) (*Durability, error) { + comp, hasComp, err := selfComponent(ctx, s, volID, name, self.ID) + if err != nil { + return nil, err + } + required := contains(vol.OffloadRequires, name) + if !required && !hasComp { + return nil, nil + } + d := &Durability{ + LocalContent: selfMax > 0, + MaxEvidenceAge: vol.OffloadMaxEvidenceAge, + } + if hasComp { + d.Method = comp.VerifyMethod + d.Covered = comp.OriginRunID >= selfMax + d.EvidenceAge = evidenceAge(comp.VerifiedAtNs, now) + } + d.Stale = d.LocalContent && evidenceStale(d) + d.Level = durabilityLevel(d, required) + return d, nil +} + +// evidenceStale reports whether a max-evidence-age policy is set and the +// covering component's verification is older than it. It is fail-closed: +// an unknown evidence age (no component, or one that never carried a +// verification instant) counts as stale. +func evidenceStale(d *Durability) bool { + if d.MaxEvidenceAge <= 0 { + return false + } + if d.EvidenceAge == nil { + return true + } + return *d.EvidenceAge > d.MaxEvidenceAge +} + +// durabilityLevel scores the durability dimension. Content that does not +// originate locally is vacuously safe (neutral). A required target +// escalates to amber when local content is not yet covered or the evidence +// has aged out; a non-required target never escalates "am I safe?" — it +// shows green when healthy and neutral otherwise, since offload does not +// gate on it. +func durabilityLevel(d *Durability, required bool) Level { + if !d.LocalContent { + return LevelNeutral + } + if !required { + if d.Covered && !d.Stale { + return LevelOK + } + return LevelNeutral + } + if !d.Covered || d.Stale { + return LevelWarn + } + return LevelOK +} + +// selfComponent returns the target's durability-vector component for this +// node's own origin — the coverage of locally-introduced content, which is +// what the offload gate checks on an edge machine. The bool is false when +// the target has no component for the self node. +func selfComponent(ctx context.Context, s *store.Store, volID int64, name string, selfID int64) (store.DestinationRunID, bool, error) { + comps, err := s.ListDestinationRunIDs(ctx, volID, name) + if err != nil { + return store.DestinationRunID{}, false, err + } + for _, c := range comps { + if c.OriginNodeID == selfID { + return c, true, nil + } + } + return store.DestinationRunID{}, false, nil +} + +// selfOriginMax returns the highest origin run this node contributes to the +// volume's present set — the coverage floor the durability vector must +// reach for local content to count as durable. Zero means this node +// originates no present content in the volume. +func selfOriginMax(ctx context.Context, s *store.Store, volID, selfID int64) (int64, error) { + comps, err := s.PresentOriginMaxima(ctx, volID, selfID) + if err != nil { + return 0, err + } + for _, c := range comps { + if c.OriginNodeID == selfID { + return c.OriginRunID, nil + } + } + return 0, nil +} diff --git a/status/labels.go b/status/labels.go new file mode 100644 index 0000000..50b3c70 --- /dev/null +++ b/status/labels.go @@ -0,0 +1,166 @@ +package status + +import ( + "fmt" + "time" +) + +// This file holds the neutral text labels every surface renders from a +// Report. They live in the query layer so the CLI grid and the TUI +// dashboard show identical words for identical facts (the "build the +// shared layer once, consume it from both" contract); each surface adds +// only its own colour and framing. The functions return strings and never +// write to any stream, keeping the library free of I/O. + +// HumanAge renders a duration in the largest useful whole unit (5s, 3m, +// 2h, 4d), the compact form both the CLI grid and the TUI use for ages. +func HumanAge(d time.Duration) string { + switch { + case d < time.Minute: + return fmt.Sprintf("%ds", int(d.Seconds())) + case d < time.Hour: + return fmt.Sprintf("%dm", int(d.Minutes())) + case d < 24*time.Hour: + return fmt.Sprintf("%dh", int(d.Hours())) + default: + return fmt.Sprintf("%dd", int(d.Hours()/24)) + } +} + +// IndexLabel renders a volume's index freshness or the never-indexed gap +// (friction-log F4 — a configured-but-unindexed volume must say so, not +// render blank). +func IndexLabel(v VolumeStatus) string { + if !v.Indexed || v.LastIndexAgo == nil { + return "never indexed" + } + return HumanAge(*v.LastIndexAgo) + " ago" +} + +// RoleLabel names why a target is in the grid: a sync destination, an +// offload-gating target, or both. +func RoleLabel(t TargetStatus) string { + switch { + case t.SyncTarget && t.Required: + return "sync+offload" + case t.SyncTarget: + return "sync" + case t.Required: + return "offload" + default: + return "—" + } +} + +// LastSyncLabel renders the last successful sync's age, "never" for a +// configured-but-unsynced target, or "relayed" for a target this node +// never pushes to (its evidence arrives via a peer durability pull). +func LastSyncLabel(t TargetStatus) string { + if !t.SyncTarget { + return "relayed" + } + if t.LastSyncAgo == nil { + return "never" + } + return HumanAge(*t.LastSyncAgo) + " ago" +} + +// StateLabel names the coverage state, most-specific first: standing states +// outrank a failed transfer, which outranks freshness. +func StateLabel(t TargetStatus) string { + switch t.Standing { + case StandingAlarm: + return "alarm" + case StandingRefused: + return "refused" + case StandingNeedsBootstrap: + return "needs-init" + } + if t.LastOutcome == "failed" { + return "failed" + } + if !t.SyncTarget { + return "—" + } + if t.LastSyncAgo == nil { + return "never-synced" + } + switch t.SyncLevel { + case LevelWarn: + return "late" + case LevelCritical: + return "stalled" + default: + return "ok" + } +} + +// DurableLabel answers "is my local content durable here": yes/no for a +// tracked target, n/a when this node originates no content in the volume, +// and — when nothing is tracked at all. +func DurableLabel(t TargetStatus) string { + if t.Durability == nil { + return "—" + } + if !t.Durability.LocalContent { + return "n/a" + } + if t.Durability.Covered { + return "yes" + } + return "no" +} + +// MethodLabel renders the verify method behind the covering component. +func MethodLabel(t TargetStatus) string { + if t.Durability == nil || t.Durability.Method == "" { + return "—" + } + return t.Durability.Method +} + +// EvidenceLabel renders the evidence age, and — when the volume sets +// offload_max_evidence_age — the age against that budget, flagging stale +// evidence loudly. +func EvidenceLabel(t TargetStatus) string { + d := t.Durability + if d == nil || !d.LocalContent { + return "—" + } + age := "unknown" + if d.EvidenceAge != nil { + age = HumanAge(*d.EvidenceAge) + } + if d.MaxEvidenceAge > 0 { + age = fmt.Sprintf("%s / %s", age, HumanAge(d.MaxEvidenceAge)) + } + if d.Stale { + age += " STALE" + } + return age +} + +// OffloadLabel renders the "N offloadable now" decision-support line +// (friction-log F17) or a note that the volume has no offload policy. +func OffloadLabel(o OffloadReadiness) string { + if !o.Applicable { + return "offload: no policy" + } + return fmt.Sprintf("offloadable now: %s (%d files) of %s present (%d files)", + HumanBytes(o.OffloadableBytes), o.OffloadableFiles, + HumanBytes(o.PresentBytes), o.PresentFiles) +} + +// TrafficLight maps a level to the green/amber/red word the CLI summary +// line and the surfaces' headers use. Neutral reads as green — no problem +// to report. +func TrafficLight(l Level) string { + switch l { + case LevelWarn: + return "amber" + case LevelCritical: + return "red" + default: + return "green" + } +} diff --git a/status/logic_test.go b/status/logic_test.go new file mode 100644 index 0000000..88c6131 --- /dev/null +++ b/status/logic_test.go @@ -0,0 +1,162 @@ +package status + +import ( + "database/sql" + "testing" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" +) + +func TestCadenceLevel(t *testing.T) { + h := time.Hour + cases := []struct { + age time.Duration + cadence time.Duration + want Level + }{ + {30 * time.Minute, h, LevelOK}, // within cadence + {h, h, LevelOK}, // exactly at cadence + {2 * h, h, LevelWarn}, // late, within lateFactor + {lateFactor * h, h, LevelWarn}, // at the boundary + {lateFactor*h + time.Minute, h, LevelCritical}, // beyond lateFactor + {1000 * h, 0, LevelOK}, // no cadence: any age is fine + } + for _, c := range cases { + if got := cadenceLevel(c.age, c.cadence); got != c.want { + t.Errorf("cadenceLevel(%s, %s) = %v, want %v", c.age, c.cadence, got, c.want) + } + } +} + +func refusedRun(errMsg string) *store.Run { + return &store.Run{Status: store.RunStatusRefused, Error: sql.NullString{String: errMsg, Valid: errMsg != ""}} +} + +func TestClassifyStanding(t *testing.T) { + bootstrapErr := "destination has no marker — re-run with --init to bootstrap" + cases := []struct { + name string + lastTerm *store.Run + hadSuccess bool + want Standing + }{ + {"no runs", nil, false, StandingNone}, + {"fresh bootstrap refusal", refusedRun(bootstrapErr), false, StandingNeedsBootstrap}, + {"refusal after prior success", refusedRun(bootstrapErr), true, StandingRefused}, + {"non-bootstrap refusal", refusedRun("layout guard: history is not packed"), false, StandingRefused}, + {"success", &store.Run{Status: store.RunStatusSuccess}, true, StandingNone}, + } + for _, c := range cases { + if got := classifyStanding(c.lastTerm, c.hadSuccess); got != c.want { + t.Errorf("%s: classifyStanding = %v, want %v", c.name, got, c.want) + } + } +} + +func TestDurabilityLevel(t *testing.T) { + dur := func(local, covered, stale bool) *Durability { + return &Durability{LocalContent: local, Covered: covered, Stale: stale} + } + cases := []struct { + name string + d *Durability + required bool + want Level + }{ + {"no local content", dur(false, false, false), true, LevelNeutral}, + {"required covered fresh", dur(true, true, false), true, LevelOK}, + {"required uncovered", dur(true, false, false), true, LevelWarn}, + {"required covered stale", dur(true, true, true), true, LevelWarn}, + {"non-required covered", dur(true, true, false), false, LevelOK}, + {"non-required uncovered", dur(true, false, false), false, LevelNeutral}, + } + for _, c := range cases { + if got := durabilityLevel(c.d, c.required); got != c.want { + t.Errorf("%s: durabilityLevel = %v, want %v", c.name, got, c.want) + } + } +} + +func TestEvidenceStale(t *testing.T) { + age := func(d time.Duration) *time.Duration { return &d } + cases := []struct { + name string + d *Durability + want bool + }{ + {"no policy", &Durability{MaxEvidenceAge: 0, EvidenceAge: age(time.Hour)}, false}, + {"unknown age fail-closed", &Durability{MaxEvidenceAge: time.Hour, EvidenceAge: nil}, true}, + {"fresh", &Durability{MaxEvidenceAge: time.Hour, EvidenceAge: age(30 * time.Minute)}, false}, + {"aged out", &Durability{MaxEvidenceAge: time.Hour, EvidenceAge: age(2 * time.Hour)}, true}, + } + for _, c := range cases { + if got := evidenceStale(c.d); got != c.want { + t.Errorf("%s: evidenceStale = %v, want %v", c.name, got, c.want) + } + } +} + +func TestLevelExitCode(t *testing.T) { + cases := map[Level]int{LevelNeutral: 0, LevelOK: 0, LevelWarn: 1, LevelCritical: 2} + for l, want := range cases { + if got := l.ExitCode(); got != want { + t.Errorf("%v.ExitCode() = %d, want %d", l, got, want) + } + } +} + +func TestHumanBytes(t *testing.T) { + cases := map[int64]string{ + 0: "0 B", + 512: "512 B", + 1024: "1.0 KiB", + 1536: "1.5 KiB", + 1024 * 1024: "1.0 MiB", + 3 * 1024 * 1024 * 1024: "3.0 GiB", + } + for n, want := range cases { + if got := HumanBytes(n); got != want { + t.Errorf("HumanBytes(%d) = %q, want %q", n, got, want) + } + } +} + +func TestOrderedTargets(t *testing.T) { + // sync_to first in declared order, then offload_requires entries not + // already listed, deduplicated ("nas" appears in both). + vol := &config.Volume{SyncTo: []string{"nas"}, OffloadRequires: []string{"nas", "s3archive"}} + got := orderedTargets(vol) + want := []string{"nas", "s3archive"} + if len(got) != len(want) { + t.Fatalf("orderedTargets = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("orderedTargets = %v, want %v", got, want) + } + } +} + +func TestStateLabel(t *testing.T) { + age := func(d time.Duration) *time.Duration { return &d } + cases := []struct { + name string + t TargetStatus + want string + }{ + {"alarm", TargetStatus{Standing: StandingAlarm}, "alarm"}, + {"needs-init", TargetStatus{Standing: StandingNeedsBootstrap}, "needs-init"}, + {"failed", TargetStatus{LastOutcome: "failed", SyncTarget: true}, "failed"}, + {"relayed", TargetStatus{SyncTarget: false, Required: true}, "—"}, + {"never", TargetStatus{SyncTarget: true}, "never-synced"}, + {"ok", TargetStatus{SyncTarget: true, LastSyncAgo: age(time.Minute), SyncLevel: LevelOK}, "ok"}, + {"late", TargetStatus{SyncTarget: true, LastSyncAgo: age(time.Hour), SyncLevel: LevelWarn}, "late"}, + } + for _, c := range cases { + if got := StateLabel(c.t); got != c.want { + t.Errorf("%s: StateLabel = %q, want %q", c.name, got, c.want) + } + } +} diff --git a/status/runs.go b/status/runs.go new file mode 100644 index 0000000..2883637 --- /dev/null +++ b/status/runs.go @@ -0,0 +1,101 @@ +package status + +import ( + "context" + "database/sql" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" +) + +// latestSuccessfulSync returns the most recent fully-landed sync of the +// pair, or nil when there is none. sql.ErrNoRows is the expected "never +// synced" signal and maps to nil; any other error propagates. +func latestSuccessfulSync(ctx context.Context, s *store.Store, volID int64, name string) (*store.Run, error) { + run, err := s.LatestSuccessfulSyncRun(ctx, volID, name) + if store.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, err + } + return &run, nil +} + +// latestTerminalSync returns the pair's most recent terminal sync run of +// any outcome (success, partial, failed, refused), or nil when there is +// none. It is the basis for the standing-state classification: a refused +// or failed latest run outranks an older success. +func latestTerminalSync(ctx context.Context, s *store.Store, volID int64, name string) (*store.Run, error) { + run, err := s.LatestFinishedRun(ctx, store.RunKindSync, volID, name) + if store.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, err + } + return &run, nil +} + +// lastIndexAge returns the age of the volume's most recent successful (or +// partial) index run, or nil when it has never been indexed. +func lastIndexAge(ctx context.Context, s *store.Store, volID int64, now time.Time) (*time.Duration, error) { + run, err := s.LatestSuccessfulIndexRun(ctx, volID) + if store.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, err + } + return ageOf(&run, now), nil +} + +// ageOf returns how long ago a run ended, or nil for a nil run or one that +// never recorded an end. Negative ages (clock skew) clamp to zero so a +// surface never renders "-3s ago". +func ageOf(run *store.Run, now time.Time) *time.Duration { + if run == nil || !run.EndedAtNs.Valid { + return nil + } + d := now.Sub(time.Unix(0, run.EndedAtNs.Int64)) + if d < 0 { + d = 0 + } + return &d +} + +// evidenceAge returns how long ago a durability component was last +// verified, or nil when the instant is unknown (NULL verified_at_ns). +// Clamped at zero for clock skew, matching ageOf. +func evidenceAge(verifiedAtNs sql.NullInt64, now time.Time) *time.Duration { + if !verifiedAtNs.Valid { + return nil + } + d := now.Sub(time.Unix(0, verifiedAtNs.Int64)) + if d < 0 { + d = 0 + } + return &d +} + +// targetKind classifies a target name against this node's config. +func targetKind(cfg *config.Config, name string) TargetKind { + if _, ok := cfg.Nodes[name]; ok { + return KindNode + } + if _, ok := cfg.Destinations[name]; ok { + return KindDestination + } + return KindRelayed +} + +// contains reports whether name appears in names. +func contains(names []string, name string) bool { + for _, n := range names { + if n == name { + return true + } + } + return false +} diff --git a/status/status.go b/status/status.go new file mode 100644 index 0000000..d6b7f83 --- /dev/null +++ b/status/status.go @@ -0,0 +1,285 @@ +// Package status is squirrel's shared "am I safe?" query layer. It reads +// the local index and config and produces, per configured volume and per +// (volume × target), the coverage and durability facts both the CLI +// `squirrel status` command and the TUI dashboard render — friction-log +// F16 (per-target sync coverage), F17 and F23 (durability and offload +// readiness). It is read-only introspection (ux-principle 2): it opens no +// run, reads no disk bytes, and mutates nothing. +// +// The package returns data and a severity per cell; it does not format for +// any particular surface beyond the neutral HumanBytes helper. Each +// consumer renders the Report its own way while sharing one source of +// truth for the facts and their severities. +package status + +import ( + "fmt" + "time" +) + +// Level is the "am I safe?" severity of a cell, a volume, or the whole +// report. It is ordered so the worst of a set is its maximum, and it maps +// directly onto the CLI's scriptable exit code via ExitCode. +type Level int + +const ( + // LevelNeutral is an informational state that says nothing about + // safety: a relayed target this node never pushes to, a durability + // figure for a volume with no offload policy, content that does not + // originate locally. It never escalates the worst-of aggregation and + // contributes exit code 0. + LevelNeutral Level = iota + // LevelOK is an affirmatively-healthy state — caught up within cadence, + // durable and freshly verified. "You may close the laptop." + LevelOK + // LevelWarn is amber: not caught up yet, needs a one-time bootstrap, + // local content not yet durable on a required target, evidence aged + // past the policy. Recoverable and expected, not a failure. + LevelWarn + // LevelCritical is red: a latched alarm, a refused sync that regressed + // from a prior success, a failed transfer, or a pair far past its + // cadence. Something is wrong and needs attention. + LevelCritical +) + +// String renders a Level as a short lowercase word for CLI output and +// tests. It is deliberately stable — scripts and golden tests key on it. +func (l Level) String() string { + switch l { + case LevelNeutral: + return "neutral" + case LevelOK: + return "ok" + case LevelWarn: + return "amber" + case LevelCritical: + return "red" + default: + return "unknown" + } +} + +// ExitCode maps a Level onto the process exit status `squirrel status` +// returns: 0 for green/neutral, 1 for amber, 2 for red. Neutral and OK +// share 0 because neither reports a problem to a script. +func (l Level) ExitCode() int { + switch l { + case LevelWarn: + return 1 + case LevelCritical: + return 2 + default: + return 0 + } +} + +// worst returns the higher-severity of two levels. +func worst(a, b Level) Level { + if b > a { + return b + } + return a +} + +// Standing is a per-target standing state carried over from #157: an +// abnormal condition that persists across cadence ticks until resolved, +// rather than a single run's outcome. +type Standing int + +const ( + // StandingNone means no standing condition — the target's health is + // whatever its latest sync freshness says. + StandingNone Standing = iota + // StandingNeedsBootstrap is a fresh destination awaiting its one-time + // `sync --init`: the latest run refused with a bootstrap reason and the + // pair has never had a successful sync. Amber — an expected human step, + // not a fault (F10). + StandingNeedsBootstrap + // StandingRefused is a preflight safety gate declining after the pair + // had previously succeeded (a marker gone missing, a layout guard) — + // the destination broke. Red (F26). + StandingRefused + // StandingAlarm is a latched verify mismatch on the destination (F30). + // Red until an operator clears it. + StandingAlarm +) + +// String renders a Standing as a short word for display and tests. +func (s Standing) String() string { + switch s { + case StandingNeedsBootstrap: + return "needs-init" + case StandingRefused: + return "refused" + case StandingAlarm: + return "alarm" + default: + return "ok" + } +} + +// TargetKind classifies a target name against this node's config: a local +// destination, a peer node, or a relayed target named only in +// offload_requires (evidence for it arrives via a peer durability pull; +// this node has no local config for it). +type TargetKind string + +const ( + KindDestination TargetKind = "destination" + KindNode TargetKind = "node" + KindRelayed TargetKind = "relayed" +) + +// HumanBytes renders a byte count in binary units with one decimal place +// above KiB, for the "N GB offloadable now" figure both surfaces show. It +// is the one formatting helper the library exposes, because the two +// consumers must render identical numbers. +func HumanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for x := n / unit; x >= unit; x /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} + +// Report is the whole snapshot for one node: every configured volume with +// its per-target coverage and durability. Now is the instant every age and +// staleness judgement in the report was taken against, so a consumer +// renders one coherent moment. +type Report struct { + Now time.Time + Volumes []VolumeStatus +} + +// Level is the worst level across every volume — the report's single +// "am I safe?" answer and the basis for the CLI exit code. +func (r Report) Level() Level { + l := LevelNeutral + for _, v := range r.Volumes { + l = worst(l, v.Level()) + } + return l +} + +// VolumeStatus is one configured volume's coverage and durability. +type VolumeStatus struct { + Name string + Path string + // Indexed is true when the volume has a row in the local index (it has + // been indexed at least once). A configured-but-never-indexed volume is + // Indexed=false with no targets carrying run history — the F4 "a freshly + // configured machine reports nothing" gap, surfaced as amber rather than + // a blank screen. + Indexed bool + // LastIndexAgo is the age of the most recent successful index run, or + // nil when the volume has never been indexed. + LastIndexAgo *time.Duration + Targets []TargetStatus + // Offload is the volume's offload-readiness tally ("N GB offloadable + // now"). Offload.Applicable is false for a volume with no offload + // policy. + Offload OffloadReadiness +} + +// Level is the worst level across the volume's targets, escalated to amber +// when the volume has never been indexed. +func (v VolumeStatus) Level() Level { + l := LevelNeutral + if !v.Indexed { + l = LevelWarn + } + for _, t := range v.Targets { + l = worst(l, t.Level()) + } + return l +} + +// TargetStatus is one (volume × target) cell: is this target caught up, +// and is the volume's local content durable on it? +type TargetStatus struct { + Name string + Kind TargetKind + // SyncTarget is true when the target is in the volume's sync_to (this + // node pushes to it). Required is true when it is in offload_requires + // (its durability gates offload). A relayed target is Required but not + // a SyncTarget. + SyncTarget bool + Required bool + // Cadence is the volume's sync_every; lateness of the last sync is + // judged relative to it, not a global constant. Zero means no scheduled + // cadence, so a successful sync of any age reads as OK. + Cadence time.Duration + // LastSyncAgo is the age of the most recent successful sync to this + // target, or nil when this node has never successfully synced it (or + // never pushes to it — a relayed target). + LastSyncAgo *time.Duration + // Standing is the target's standing state (alarm / refused / + // needs-bootstrap / none). + Standing Standing + // AlarmDetail is the latched alarm's detail text when Standing is + // StandingAlarm, for the surface to show what tripped. + AlarmDetail string + // LastOutcome is the status of the pair's most recent terminal sync run + // (success / partial / failed / refused / aborted), or "" when this + // node has never run a sync against the target. It lets a surface name + // a failed transfer precisely, distinct from an aged-out success. + LastOutcome string + // SyncLevel is the severity of the coverage dimension alone (freshness + // and standing state); durability is scored separately in Durability. + SyncLevel Level + // Durability is the local-content durability on this target, or nil when + // durability is not tracked for the pair (a non-required target with no + // recorded vector). + Durability *Durability +} + +// Level is the worst of the target's coverage and durability dimensions. +func (t TargetStatus) Level() Level { + l := t.SyncLevel + if t.Durability != nil { + l = worst(l, t.Durability.Level) + } + return l +} + +// Durability is the local-content durability of a volume on one target. +type Durability struct { + // LocalContent is true when this node originates present content in the + // volume. When false, "is my local content durable here" is vacuously + // satisfied and Level is neutral — the hub, whose photos originate on + // the edges, sees this for its inbound volumes. + LocalContent bool + // Covered is true when the target's durability vector covers this + // node's highest local origin run — every locally-originated present + // content is recorded durable there. + Covered bool + // Method is the verify method behind the covering component (blake3, + // peer-blake3, kopia-verify, presence+size, size+mtime), or "" when + // there is no component. + Method string + // EvidenceAge is how long ago the covering component was last verified, + // or nil when unknown (no component, or a component that never carried + // a verification instant). + EvidenceAge *time.Duration + // MaxEvidenceAge is the volume's offload_max_evidence_age; zero means no + // staleness policy. Stale is true when the policy is set and the + // evidence is older than it (or its age is unknown). + MaxEvidenceAge time.Duration + Stale bool + Level Level +} + +// OffloadReadiness is a volume's "what could I offload right now" tally. +type OffloadReadiness struct { + // Applicable is false when the volume declares no offload policy. + Applicable bool + OffloadableFiles int + OffloadableBytes int64 + PresentFiles int + PresentBytes int64 +} diff --git a/tui/dashboard.go b/tui/dashboard.go index 30bac28..f6a94a0 100644 --- a/tui/dashboard.go +++ b/tui/dashboard.go @@ -10,20 +10,25 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/status" "github.com/mbertschler/squirrel/store" ) // dashboardModel surfaces squirrel's live state in one screen: // // - the local agent's health (one-line probe of /v1/health) +// - standing per-destination alarms // - runs currently in-flight (kind, volume, destination, elapsed) -// - per-volume health (name, path, last-index, last-sync) +// - the per-(volume × target) coverage + durability grid (the "am I +// safe?" panel, from the shared status query layer) // - the most recent terminal runs // // Data is pulled on each tickMsg via a single SQL pass plus one HTTP probe; // both run as Bubble Tea commands so the UI never blocks on I/O. type dashboardModel struct { store *store.Store + cfg *config.Config client *agentClient width, height int @@ -42,10 +47,11 @@ type dashboardData struct { volumes []store.Volume activeRuns []store.Run recentRuns []store.Run - // latestByVol[volID][kind] is the most recent terminal-status run for - // that (volume, kind) pair. Used to fill the "last index" / "last sync" - // columns of the volumes table. - latestByVol map[int64]map[string]store.Run + // coverage is the per-(volume × target) sync-coverage and durability + // grid from the shared status query layer — the same facts and + // severities `squirrel status` prints. Empty when no config is loaded + // (the grid needs sync_to / offload_requires / cadences to render). + coverage status.Report // alarms are the standing per-destination alarms (#157, F30). A verify // mismatch latches one until cleared; the dashboard shows them so the // trust surface answers "am I safe?" with a red panel, not silence. @@ -65,8 +71,8 @@ type agentStatus struct { type agentStatusMsg agentStatus -func newDashboardModel(s *store.Store) *dashboardModel { - return &dashboardModel{store: s} +func newDashboardModel(s *store.Store, cfg *config.Config) *dashboardModel { + return &dashboardModel{store: s, cfg: cfg} } // attachClient is called by the root model after construction so the @@ -114,7 +120,7 @@ func (m *dashboardModel) View() string { } sections = append(sections, m.renderActiveRuns(), - m.renderVolumes(), + m.renderCoverage(), m.renderRecentRuns(), ) return strings.Join(sections, "\n\n") @@ -183,21 +189,85 @@ func (m *dashboardModel) renderActiveRuns() string { return header + "\n" + renderTable(rows, []lipgloss.Color{"", "", "", "", colourRunning}) } -func (m *dashboardModel) renderVolumes() string { - header := styleHeader.Render(fmt.Sprintf("Volumes (%d)", len(m.data.volumes))) - if len(m.data.volumes) == 0 { - return header + "\n" + styleMuted.Render("no volumes configured") +// renderCoverage is the "am I safe?" panel: per volume, the per-target +// sync-coverage and durability grid from the shared status layer, replacing +// the old single LAST SYNC cell that hid a week-behind target behind a +// fresh ✓ (friction-log F16/F17/F23). Each volume gets a header line +// (name, path, index freshness, offloadable total) coloured by its worst +// level, then a target sub-table with the STATE and DURABLE cells coloured +// per target. +func (m *dashboardModel) renderCoverage() string { + vols := m.data.coverage.Volumes + header := styleHeader.Render(fmt.Sprintf("Coverage (%d)", len(vols))) + if len(vols) == 0 { + hint := "no volumes configured" + if m.cfg == nil { + hint = "no config loaded — coverage needs sync_to / offload_requires to render" + } + return header + "\n" + styleMuted.Render(hint) } - rows := [][]string{{"NAME", "PATH", "LAST INDEX", "LAST SYNC"}} - for _, v := range m.data.volumes { + blocks := []string{header} + for _, v := range vols { + blocks = append(blocks, m.renderVolumeCoverage(v)) + } + return strings.Join(blocks, "\n\n") +} + +// renderVolumeCoverage renders one volume's coverage block. +func (m *dashboardModel) renderVolumeCoverage(v status.VolumeStatus) string { + dot := lipgloss.NewStyle().Foreground(levelColour(v.Level())).Render("●") + title := fmt.Sprintf("%s %s %s", dot, v.Name, styleMuted.Render(v.Path)) + meta := styleMuted.Render(fmt.Sprintf("index %s · %s", + status.IndexLabel(v), status.OffloadLabel(v.Offload))) + if len(v.Targets) == 0 { + return title + "\n" + meta + "\n" + styleMuted.Render(" no targets configured") + } + rows := [][]string{{"TARGET", "ROLE", "LAST SYNC", "STATE", "DURABLE", "METHOD", "EVIDENCE"}} + for _, t := range v.Targets { rows = append(rows, []string{ - v.Name, - v.Path, - m.formatLast(v.ID, store.RunKindIndex), - m.formatLast(v.ID, store.RunKindSync), + t.Name, status.RoleLabel(t), status.LastSyncLabel(t), status.StateLabel(t), + status.DurableLabel(t), status.MethodLabel(t), status.EvidenceLabel(t), }) } - return header + "\n" + renderTable(rows, nil) + tbl := renderTableColoured(rows, nil, coverageCellColour(v.Targets)) + return title + "\n" + meta + "\n" + tbl +} + +// coverageCellColour paints the STATE column by each target's coverage +// level and the DURABLE column by its durability level, so the two "am I +// safe?" dimensions read at a glance without decoding the words. +func coverageCellColour(targets []status.TargetStatus) func(rowIdx, colIdx int) lipgloss.Color { + const stateCol, durableCol = 3, 4 + return func(rowIdx, colIdx int) lipgloss.Color { + if rowIdx == 0 || rowIdx > len(targets) { + return "" + } + t := targets[rowIdx-1] + switch colIdx { + case stateCol: + return levelColour(t.SyncLevel) + case durableCol: + if t.Durability != nil { + return levelColour(t.Durability.Level) + } + } + return "" + } +} + +// levelColour maps a status level onto the dashboard palette. Neutral gets +// no colour (the default foreground) so informational cells don't shout. +func levelColour(l status.Level) lipgloss.Color { + switch l { + case status.LevelOK: + return colourSuccess + case status.LevelWarn: + return colourWarning + case status.LevelCritical: + return colourFailure + default: + return "" + } } func (m *dashboardModel) renderRecentRuns() string { @@ -239,26 +309,12 @@ func (m *dashboardModel) volumeName(id sql.NullInt64) string { return fmt.Sprintf("vol#%d", id.Int64) } -func (m *dashboardModel) formatLast(volID int64, kind string) string { - byKind := m.data.latestByVol[volID] - if byKind == nil { - return styleMuted.Render("—") - } - r, ok := byKind[kind] - if !ok { - return styleMuted.Render("—") - } - ago := whenAgo(r.EndedAtNs, m.data.now) - statusGlyph := lipgloss.NewStyle().Foreground(statusColour(r.Status)).Render(glyphForStatus(r.Status)) - return fmt.Sprintf("%s %s", ago, statusGlyph) -} - func (m *dashboardModel) fetchData() tea.Cmd { return func() tea.Msg { // Use a tight per-fetch deadline so a stuck DB doesn't freeze the UI. ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - data, err := loadDashboardData(ctx, m.store) + data, err := loadDashboardData(ctx, m.store, m.cfg) return dashboardDataMsg{data: data, err: err} } } @@ -279,13 +335,14 @@ func (m *dashboardModel) probeAgent() tea.Cmd { } } -// loadDashboardData runs the SQL queries that back the dashboard. The +// loadDashboardData runs the queries that back the dashboard. The // recent-runs bucket comes from a bounded ListRuns scan (200 is plenty -// for "what happened today"); the per-(volume,kind) "last successful" -// table comes from its own helper that scans every run, so volumes -// whose last index sits beyond the recent window still surface -// correctly. -func loadDashboardData(ctx context.Context, s *store.Store) (dashboardData, error) { +// for "what happened today"); the coverage grid comes from the shared +// status query layer, which scans per (volume × target) so a target beyond +// the recent-runs window still surfaces correctly. The coverage build is +// skipped when no config is loaded — it needs sync_to / offload_requires / +// cadences — leaving the grid to render its own "no config" hint. +func loadDashboardData(ctx context.Context, s *store.Store, cfg *config.Config) (dashboardData, error) { now := time.Now() vols, err := s.ListVolumes(ctx) if err != nil { @@ -295,14 +352,16 @@ func loadDashboardData(ctx context.Context, s *store.Store) (dashboardData, erro if err != nil { return dashboardData{}, fmt.Errorf("list runs: %w", err) } - latestByVol, err := s.LatestSuccessfulRunsByVolumeAndKind(ctx) - if err != nil { - return dashboardData{}, fmt.Errorf("latest by volume: %w", err) - } alarms, err := s.ListDestinationAlarms(ctx) if err != nil { return dashboardData{}, fmt.Errorf("list alarms: %w", err) } + var coverage status.Report + if cfg != nil { + if coverage, err = status.Build(ctx, s, cfg); err != nil { + return dashboardData{}, fmt.Errorf("build coverage: %w", err) + } + } var active, recent []store.Run for _, r := range runs { if r.Status == store.RunStatusRunning { @@ -314,11 +373,11 @@ func loadDashboardData(ctx context.Context, s *store.Store) (dashboardData, erro } } return dashboardData{ - now: now, - volumes: vols, - activeRuns: active, - recentRuns: recent, - latestByVol: latestByVol, - alarms: alarms, + now: now, + volumes: vols, + activeRuns: active, + recentRuns: recent, + coverage: coverage, + alarms: alarms, }, nil } diff --git a/tui/dashboard_test.go b/tui/dashboard_test.go index c6f78b9..9a84838 100644 --- a/tui/dashboard_test.go +++ b/tui/dashboard_test.go @@ -36,8 +36,6 @@ func TestLoadDashboardDataPartitionsRuns(t *testing.T) { // a successful sync run. We expect: // - activeRuns: the one running sync // - recentRuns: every terminal run, newest first, capped at 10 - // - latestByVol[vol.ID]: index = most recent success, sync = the - // successful one (failed audit doesn't establish "last audit") finishedIdx1, _ := s.BeginIndexRun(ctx, store.RunKindIndex, vol.ID, false) if err := s.FinishRun(ctx, finishedIdx1, store.RunStatusSuccess, "", 100); err != nil { t.Fatalf("FinishRun: %v", err) @@ -57,7 +55,10 @@ func TestLoadDashboardDataPartitionsRuns(t *testing.T) { t.Fatalf("FinishRun: %v", err) } - data, err := loadDashboardData(ctx, s) + // cfg is nil here: this test pins the run-partitioning behaviour, which + // is independent of the config-driven coverage grid (that is exercised + // against the shared status layer in the status package's own tests). + data, err := loadDashboardData(ctx, s, nil) if err != nil { t.Fatalf("loadDashboardData: %v", err) } @@ -80,18 +81,8 @@ func TestLoadDashboardDataPartitionsRuns(t *testing.T) { t.Errorf("recentRuns[0] = %d, want %d (most recent)", data.recentRuns[0].ID, successSync) } - byKind := data.latestByVol[vol.ID] - if byKind == nil { - t.Fatalf("latestByVol missing entry for vol#%d", vol.ID) - } - if byKind[store.RunKindIndex].ID != finishedIdx2 { - t.Errorf("latest index = %d, want %d (the partial-but-newer one)", - byKind[store.RunKindIndex].ID, finishedIdx2) - } - if byKind[store.RunKindSync].ID != successSync { - t.Errorf("latest sync = %d, want %d", byKind[store.RunKindSync].ID, successSync) - } - if _, present := byKind[store.RunKindAudit]; present { - t.Errorf("latestByVol must not include the failed audit run") + // With no config, the coverage grid renders nothing. + if len(data.coverage.Volumes) != 0 { + t.Errorf("coverage.Volumes = %d, want 0 without config", len(data.coverage.Volumes)) } } diff --git a/tui/tui.go b/tui/tui.go index b11d895..0ccfee6 100644 --- a/tui/tui.go +++ b/tui/tui.go @@ -90,7 +90,7 @@ type rootModel struct { func newRootModel(s *store.Store, cfg *config.Config) *rootModel { client := newAgentClient(cfg) - dash := newDashboardModel(s) + dash := newDashboardModel(s, cfg) dash.attachClient(client) return &rootModel{ active: screenDashboard, From d0c5932332925bf16687f45381e00381a33e9109 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:58:12 +0000 Subject: [PATCH 04/14] status: adopt #157's 2-return RaiseDestinationAlarm in the alarm test Merging #157's latest brought its TOCTOU follow-up, which changed store.RaiseDestinationAlarm to return (raised bool, err error). The alarm Build test called the old single-return form; discard the raised bool it does not assert on. No production code called RaiseDestinationAlarm, so the merge is the only other change. go vet, go test ./..., and golangci-lint (2.12.2) all pass. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- status/build_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/status/build_test.go b/status/build_test.go index 2e1c352..7ae8de9 100644 --- a/status/build_test.go +++ b/status/build_test.go @@ -164,7 +164,7 @@ func TestBuildAlarm(t *testing.T) { s := setupStore(t) ctx := context.Background() root, idx := indexTree(t, s, "media") - if err := s.RaiseDestinationAlarm(ctx, "dest", store.AlarmKindVerifyMismatch, "checksum mismatch on object", idx.RunID); err != nil { + if _, err := s.RaiseDestinationAlarm(ctx, "dest", store.AlarmKindVerifyMismatch, "checksum mismatch on object", idx.RunID); err != nil { t.Fatalf("RaiseDestinationAlarm: %v", err) } cfg := cfgFor("media", root, []string{"dest"}, nil, []string{"dest"}, nil) From 66acddc02a3d6df0e3d96d2a8c758d97f06f819d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:58:14 +0000 Subject: [PATCH 05/14] conflicts: converge divergent edits via contested-freeze, notify both edges (#158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Divergent edits of the same path on two peers used to ping-pong forever: every cadence tick re-asserted and minted a fresh .squirrel-conflicts/run-N/ copy, the conflict store grew unboundedly, and no human was told (F27). A conflict now raises a contested_paths latch (schema v26, STRICT). While it stands the /plan classifier answers a divergent re-assertion from any peer with a new `contested` disposition — version-gated (ProtocolVersionContested) so older peers fall back to the legacy `conflict` — instead of minting another copy. The flip-flop stops at the first conflict; both versions stay preserved (winner live, loser under .squirrel-conflicts/). The frozen winner re-asserting its own bytes is still allowed through. Both initiators learn of it on their own machines, not just the hub: conflict and contested counts flow into SyncRunReport, the initiators' run rows (the CONFLICTS column), a scheduler.conflict warn line, and a "Contested paths" TUI badge. Every node mirrors the freeze into its own contested_paths latch. New question/change command pair: `squirrel conflicts` lists frozen paths (versions, preserved location, when, who); `squirrel conflicts resolve ` clears the latch so syncs flow again — the explicit human act (squirrel never resolves on its own). Resolve keeps the live version and points at the preserved copy; adopting the other version stays a deliberate restore, never a silent content move. Schema v26: migrateV25ToV26 adds the contested_paths table; snapshot regenerated. Closes F27. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- agent/agent.go | 15 +- agent/contested_test.go | 190 ++++++++++++++++++++++++ agent/scheduler.go | 12 ++ agent/sync.go | 116 +++++++++++++-- cmd/squirrel/agent.go | 8 +- cmd/squirrel/conflicts.go | 134 +++++++++++++++++ cmd/squirrel/conflicts_resolve.go | 96 ++++++++++++ cmd/squirrel/conflicts_test.go | 95 ++++++++++++ cmd/squirrel/root.go | 1 + cmd/squirrel/runs.go | 58 +++++++- cmd/squirrel/sync.go | 8 + design/friction-log.md | 16 ++ store/contested_paths.go | 238 ++++++++++++++++++++++++++++++ store/contested_paths_test.go | 143 ++++++++++++++++++ store/migrations.go | 67 ++++++++- store/runs_audit.go | 10 ++ store/schema.sql | 14 +- sync/node.go | 71 ++++++++- sync/node_test.go | 114 ++++++++++++++ sync/sync.go | 10 ++ syncproto/syncproto.go | 48 ++++++ tui/dashboard.go | 48 ++++++ 22 files changed, 1485 insertions(+), 27 deletions(-) create mode 100644 agent/contested_test.go create mode 100644 cmd/squirrel/conflicts.go create mode 100644 cmd/squirrel/conflicts_resolve.go create mode 100644 cmd/squirrel/conflicts_test.go create mode 100644 store/contested_paths.go create mode 100644 store/contested_paths_test.go diff --git a/agent/agent.go b/agent/agent.go index 5084e35..051846f 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -47,11 +47,18 @@ type SyncRunner func(ctx context.Context, vol *config.Volume, destName string) S // SyncRunReport is the SyncRunner result surfaced to the scheduler. // Kept minimal — anything richer belongs in the runs table the CLI -// inspects via `squirrel runs`, not on the agent's hot path. +// inspects via `squirrel runs`, not on the agent's hot path. Conflicts +// and Contested carry the peer-sync divergence counts so the scheduler +// logs a conflict signal on the initiating machine, not only the hub +// (#158, F27): Conflicts is how many paths this run resolved by +// preserving the loser and landing this node's bytes live, Contested how +// many the receiver refused because a prior freeze still stands. type SyncRunReport struct { - RunID int64 - Status string - Err error + RunID int64 + Status string + Err error + Conflicts int + Contested int } // Config configures one agent listener. Fields are validated by New; all diff --git a/agent/contested_test.go b/agent/contested_test.go new file mode 100644 index 0000000..ab1c477 --- /dev/null +++ b/agent/contested_test.go @@ -0,0 +1,190 @@ +package agent + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/mbertschler/squirrel/store" + "github.com/mbertschler/squirrel/syncproto" + "github.com/zeebo/blake3" +) + +// seedLocalWrite writes content to path and records a present index row +// attributed to a *local* index run (no peer linkage), so a divergent +// initiator hash classifies as a conflict — the precondition for a freeze. +func (f *preStageFixture) seedLocalWrite(t *testing.T, ctx context.Context, path string, content []byte) []byte { + t.Helper() + abs := filepath.Join(f.vol.Path, path) + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + t.Fatalf("mkdir for %s: %v", path, err) + } + if err := os.WriteFile(abs, content, 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + sum := blake3.Sum256(content) + if err := f.store.Upsert(ctx, store.FileRow{ + VolumeID: f.volID, Path: path, Blake3: sum[:], SizeBytes: int64(len(content)), + MtimeNs: store.NowNs(), Status: store.StatusPresent, + FirstSeenRunID: f.localRun, LastSeenRunID: f.localRun, IndexedAtNs: store.NowNs(), + }, nil); err != nil { + t.Fatalf("Upsert %s: %v", path, err) + } + return sum[:] +} + +// freshSession opens another receiver run and returns a session bound to +// it at the given protocol version, so a test can drive successive /plan +// exchanges the way successive cadence ticks would. +func (f *preStageFixture) freshSession(t *testing.T, protocol int) *peerSession { + t.Helper() + run, err := f.store.BeginPeerSyncRun(context.Background(), f.volID, f.peerID, 7, "peerA") + if err != nil { + t.Fatalf("BeginPeerSyncRun: %v", err) + } + sess := f.newSession() + sess.receiverRunID = run + sess.protocolVersion = protocol + return sess +} + +// countConflictRunDirs returns how many run-/ subdirectories exist +// under .squirrel-conflicts — one per distinct conflict-preserving run, so +// the F27 unbounded-growth regression shows up as this number climbing. +func countConflictRunDirs(t *testing.T, volPath string) int { + t.Helper() + entries, err := os.ReadDir(filepath.Join(volPath, ConflictsDirName)) + if err != nil { + if os.IsNotExist(err) { + return 0 + } + t.Fatalf("read conflicts dir: %v", err) + } + return len(entries) +} + +// TestContestedFreezeStopsPingPong is the #158 acceptance at the /plan +// level: the first conflict preserves the loser and freezes the path; a +// later divergent re-assertion is refused with `contested` (no second +// copy minted); and the frozen winner re-asserting is still allowed +// through so a dropped /close can re-land it. +func TestContestedFreezeStopsPingPong(t *testing.T) { + ctx := context.Background() + f := newPreStageFixture(t) + + contentX := []byte("homepc's version, indexed locally on the hub") + contentZ := []byte("laptop's divergent version — wins the first conflict") + contentW := []byte("homepc re-asserts yet another edit") + f.seedLocalWrite(t, ctx, "doc.md", contentX) + + // Tick 1: laptop's divergent bytes conflict with the local write. + sess1 := f.freshSession(t, syncproto.ProtocolVersionContested) + plan1, err := f.router.planSession(ctx, sess1, []syncproto.IndexEntry{ + {Path: "doc.md", Blake3Hex: blakeHex(contentZ), SizeBytes: int64(len(contentZ))}, + }) + if err != nil { + t.Fatalf("planSession tick 1: %v", err) + } + if d := sess1.dispositions["doc.md"].disposition; d != syncproto.DispositionConflict { + t.Fatalf("tick 1 disposition = %q, want conflict", d) + } + if len(plan1.Conflicts) != 1 { + t.Fatalf("tick 1 conflicts = %d, want 1", len(plan1.Conflicts)) + } + if got := countConflictRunDirs(t, f.vol.Path); got != 1 { + t.Fatalf("conflict run dirs after tick 1 = %d, want 1", got) + } + // The path is now frozen with laptop's bytes as the winner. + latch, err := f.store.GetContestedPath(ctx, f.volID, "doc.md") + if err != nil { + t.Fatalf("GetContestedPath: %v", err) + } + if blakeHex(contentZ) != hexOrEmpty(latch.LiveBlake3) { + t.Fatalf("latch winner = %s, want hash(Z) %s", hexOrEmpty(latch.LiveBlake3), blakeHex(contentZ)) + } + + // Tick 2: homepc re-asserts a third version. The freeze refuses it — + // no bytes move, no second copy is minted (the F27 fix). + sess2 := f.freshSession(t, syncproto.ProtocolVersionContested) + plan2, err := f.router.planSession(ctx, sess2, []syncproto.IndexEntry{ + {Path: "doc.md", Blake3Hex: blakeHex(contentW), SizeBytes: int64(len(contentW))}, + }) + if err != nil { + t.Fatalf("planSession tick 2: %v", err) + } + if d := sess2.dispositions["doc.md"].disposition; d != syncproto.DispositionContested { + t.Fatalf("tick 2 disposition = %q, want contested", d) + } + if len(plan2.Conflicts) != 0 { + t.Fatalf("tick 2 conflicts = %d, want 0 (frozen, not re-conflicted)", len(plan2.Conflicts)) + } + if len(plan2.Contested) != 1 || plan2.Contested[0].Path != "doc.md" { + t.Fatalf("tick 2 contested = %+v, want one doc.md entry", plan2.Contested) + } + if plan2.Contested[0].PreservedAtPath == "" || plan2.Contested[0].LiveBlake3Hex != blakeHex(contentZ) { + t.Fatalf("tick 2 contested detail = %+v, want winner Z + preserved path", plan2.Contested[0]) + } + if got := countConflictRunDirs(t, f.vol.Path); got != 1 { + t.Fatalf("conflict run dirs after tick 2 = %d, want 1 (no new copy minted)", got) + } + + // Tick 3: the frozen winner re-asserts its own bytes — allowed through + // (Transfer), so a /close that never committed can re-land the winner. + sess3 := f.freshSession(t, syncproto.ProtocolVersionContested) + if _, err := f.router.planSession(ctx, sess3, []syncproto.IndexEntry{ + {Path: "doc.md", Blake3Hex: blakeHex(contentZ), SizeBytes: int64(len(contentZ))}, + }); err != nil { + t.Fatalf("planSession tick 3: %v", err) + } + if d := sess3.dispositions["doc.md"].disposition; d != syncproto.DispositionTransfer { + t.Fatalf("tick 3 disposition = %q, want transfer (winner re-asserting)", d) + } +} + +// TestContestedFreezeVersionGated confirms the disposition is gated: on +// one and the same frozen state, a session negotiated below +// ProtocolVersionContested falls back to the legacy `conflict` it +// understands, while a contested-capable session gets the freeze. Older +// peers stay safe. +func TestContestedFreezeVersionGated(t *testing.T) { + ctx := context.Background() + f := newPreStageFixture(t) + contentX := f.seedLocalWrite(t, ctx, "doc.md", []byte("hub-local content")) + + // Freeze doc.md while its local-write bytes stay live (the winner), so + // both sessions below classify against an identical present row. + if err := f.store.RaiseContested(ctx, store.ContestedPath{ + VolumeID: f.volID, Path: "doc.md", LiveBlake3: contentX, RaisedRunID: f.localRun, + }); err != nil { + t.Fatalf("RaiseContested: %v", err) + } + divergent := blakeHex([]byte("a divergent edit")) + + // An older initiator (Merkle-walk protocol) must get the legacy + // conflict, not the contested disposition it can't interpret. + legacy := f.freshSession(t, syncproto.ProtocolVersionMerkleWalk) + plan, err := f.router.planSession(ctx, legacy, []syncproto.IndexEntry{ + {Path: "doc.md", Blake3Hex: divergent, SizeBytes: 16}, + }) + if err != nil { + t.Fatalf("planSession legacy: %v", err) + } + if d := legacy.dispositions["doc.md"].disposition; d != syncproto.DispositionConflict { + t.Fatalf("legacy disposition = %q, want conflict (version gate)", d) + } + if len(plan.Contested) != 0 { + t.Fatalf("legacy plan.Contested = %+v, want empty for an older peer", plan.Contested) + } + + // A contested-capable initiator, same frozen state, gets refused. + modern := f.freshSession(t, syncproto.ProtocolVersionContested) + if _, err := f.router.planSession(ctx, modern, []syncproto.IndexEntry{ + {Path: "doc.md", Blake3Hex: divergent, SizeBytes: 16}, + }); err != nil { + t.Fatalf("planSession modern: %v", err) + } + if d := modern.dispositions["doc.md"].disposition; d != syncproto.DispositionContested { + t.Fatalf("modern disposition = %q, want contested (version gate)", d) + } +} diff --git a/agent/scheduler.go b/agent/scheduler.go index 17d3883..2a3f39f 100644 --- a/agent/scheduler.go +++ b/agent/scheduler.go @@ -504,7 +504,19 @@ func (s *scheduler) runSync(ctx context.Context, vol *config.Volume, volumeID in "kind", "sync", "volume", vol.Name, "destination", destName, "run_id", rep.RunID, "status", status, "duration_ms", duration.Milliseconds(), + "conflicts", rep.Conflicts, "contested", rep.Contested, ) + // A conflict signal on this (initiating) machine, not just the hub: a + // divergent edit was preserved or a frozen path refused, and a human + // must eventually resolve it (#158, F27). Logged at warn so it stands + // out from the routine finished line in the scheduler's own output. + if rep.Conflicts > 0 || rep.Contested > 0 { + s.logger.Warn("scheduler.conflict", + "kind", "sync", "volume", vol.Name, "destination", destName, + "run_id", rep.RunID, + "conflicts", rep.Conflicts, "contested", rep.Contested, + ) + } if rep.Err != nil { s.logger.Error("scheduler.error", "kind", "sync", "volume", vol.Name, "destination", destName, diff --git a/agent/sync.go b/agent/sync.go index 29d6a7f..83a2fda 100644 --- a/agent/sync.go +++ b/agent/sync.go @@ -401,7 +401,7 @@ func (r *peerSyncRouter) finishBegin(ctx context.Context, body syncproto.BeginRe // know is clamped down to ProtocolVersionMerkleWalk rather than // rejected, so a partial rollout doesn't break syncs. func negotiateProtocol(requested int) int { - const receiverMax = syncproto.ProtocolVersionMerkleWalk + const receiverMax = syncproto.ProtocolVersionContested if requested <= 0 { return syncproto.ProtocolVersionFlat } @@ -574,9 +574,53 @@ func (r *peerSyncRouter) planSession(ctx context.Context, sess *peerSession, ent }) } resp.Conflicts = collectConflicts(sess) + contested, err := r.collectContested(ctx, sess, entries) + if err != nil { + return syncproto.PlanResponse{}, err + } + resp.Contested = contested return resp, nil } +// collectContested builds the wire-format contested list from the entries +// classified as `contested`, in the order the initiator sent them. Each +// path is frozen, so its contested_paths latch supplies the two versions +// and the preserved-loser location the initiator mirrors into its own +// marker and renders. A latch that vanished between classify and here +// (an operator resolved it mid-plan) is skipped rather than erroring — +// the next sync re-plans cleanly. +func (r *peerSyncRouter) collectContested(ctx context.Context, sess *peerSession, entries []syncproto.IndexEntry) ([]syncproto.ContestedDetail, error) { + var out []syncproto.ContestedDetail + for _, e := range entries { + if sess.dispositions[e.Path].disposition != syncproto.DispositionContested { + continue + } + latch, err := r.srv.store.GetContestedPath(ctx, sess.volumeID, e.Path) + if err != nil { + if store.IsNotFound(err) { + continue + } + return nil, fmt.Errorf("look up contested %s: %w", e.Path, err) + } + out = append(out, syncproto.ContestedDetail{ + Path: e.Path, + LiveBlake3Hex: hexOrEmpty(latch.LiveBlake3), + PreservedBlake3Hex: hexOrEmpty(latch.PreservedBlake3), + PreservedAtPath: latch.PreservedAtPath, + }) + } + return out, nil +} + +// hexOrEmpty renders a raw digest as lowercase hex, or "" for a nil digest +// (the "unknown winner" latch case), keeping the wire field omittable. +func hexOrEmpty(b []byte) string { + if len(b) == 0 { + return "" + } + return hex.EncodeToString(b) +} + // handlePlanFolders implements POST /v1/sync/plan-folders: look up the // receiver's folder rows for each requested path and return the // (shallow, deep, direct children) tuple. The endpoint is the @@ -804,21 +848,31 @@ func collectConflicts(sess *peerSession) []syncproto.ConflictDetail { // strategy is "off" (initiator opted out). func (r *peerSyncRouter) classify(ctx context.Context, sess *peerSession, relPath string, entry *sessionEntry) (string, error) { existing, err := r.srv.store.GetByPath(ctx, sess.volumeID, relPath) - if err != nil { - if store.IsNotFound(err) { - return r.classifyMissingPath(ctx, sess, entry) - } + liveFound := err == nil && existing.Status == store.StatusPresent + if err != nil && !store.IsNotFound(err) { return "", err } - if existing.Status != store.StatusPresent { - // A live row in 'missing' status is treated as "no file here"; - // the initiator's bytes are new content. Dedup still applies — - // the bytes may exist at another path. - return r.classifyMissingPath(ctx, sess, entry) - } - if bytesEqual(existing.Blake3, entry.blake3) { + if liveFound && bytesEqual(existing.Blake3, entry.blake3) { return syncproto.DispositionAlreadyCorrect, nil } + // The initiator's bytes differ from any live row here (or there is no + // live row). Before deciding Supersede/Conflict/Transfer, check the + // contested freeze: a path frozen by a prior conflict refuses a + // divergent re-assertion from any peer, so the flip-flop stops at the + // first conflict instead of minting a fresh copy every cadence tick. + // The frozen winner re-asserting its own digest is *not* refused — it + // falls through so a /close that never committed can re-establish it. + if contested, err := r.contestedRefusal(ctx, sess, relPath, entry); err != nil { + return "", err + } else if contested { + return syncproto.DispositionContested, nil + } + if !liveFound { + // No live row (or a 'missing'/'offloaded' one): the initiator's + // bytes are new content here. Dedup still applies — the bytes may + // exist at another path. + return r.classifyMissingPath(ctx, sess, entry) + } // Different blake3 at this path. Provenance decides the verdict // (Supersede / Conflict). Content at a path takes precedence over // a cross-path dedup — we don't let dedup paper over a divergence @@ -831,6 +885,28 @@ func (r *peerSyncRouter) classify(ctx context.Context, sess *peerSession, relPat return disp, nil } +// contestedRefusal reports whether relPath is frozen and this initiator's +// bytes should be refused with the `contested` disposition. It returns +// false — letting normal classification proceed — when the path isn't +// frozen, when the initiator's digest matches the frozen winner (the +// winner re-asserting, which must be allowed to re-land), or when the +// session negotiated a protocol older than ProtocolVersionContested (an +// initiator that can't interpret the disposition falls back to the legacy +// `conflict`, keeping older peers safe per the version gate). +func (r *peerSyncRouter) contestedRefusal(ctx context.Context, sess *peerSession, relPath string, entry *sessionEntry) (bool, error) { + if sess.protocolVersion < syncproto.ProtocolVersionContested { + return false, nil + } + liveBlake3, contested, err := r.srv.store.IsPathContested(ctx, sess.volumeID, relPath) + if err != nil { + return false, err + } + if !contested { + return false, nil + } + return !bytesEqual(entry.blake3, liveBlake3), nil +} + // classifyMissingPath is the no-live-row branch of classify: try to // satisfy the path from existing content elsewhere in the volume // (CopyFromExisting), or fall back to Transfer. Strategy "off" skips @@ -1188,6 +1264,22 @@ func (r *peerSyncRouter) preStageConflicts(ctx context.Context, sess *peerSessio return fmt.Errorf("record conflict pre-stage for %s: %w", path, err) } entry.preservedAtPath = preservedRel + // Freeze the path so the next cadence tick refuses a divergent + // re-assertion instead of minting another `.squirrel-conflicts/` + // copy (#158, F27). The winner is the initiator's bytes (landing + // live on /close); the loser is the just-preserved prior row. + // Idempotent: a path already frozen keeps its original latch. + if err := r.srv.store.RaiseContested(ctx, store.ContestedPath{ + VolumeID: sess.volumeID, + Path: path, + LiveBlake3: entry.blake3, + PreservedBlake3: entry.priorRow.Blake3, + PreservedAtPath: preservedRel, + PeerNodeID: sess.peerNodeID, + RaisedRunID: sess.receiverRunID, + }); err != nil { + return fmt.Errorf("raise contested for %s: %w", path, err) + } } return nil } diff --git a/cmd/squirrel/agent.go b/cmd/squirrel/agent.go index ad18923..d11d730 100644 --- a/cmd/squirrel/agent.go +++ b/cmd/squirrel/agent.go @@ -222,7 +222,13 @@ func buildSchedulerSyncRunner(cfg *config.Config, s *store.Store, rcl *sync.Rclo opts.Snapshot = sync.NewSnapshotter(s, rcl, snapshotConfig(cfg, s.Path())) } rep, runErr := sync.RunPair(ctx, s, tools, pair, opts) - return agent.SyncRunReport{RunID: rep.RunID, Status: rep.Status, Err: runErr} + return agent.SyncRunReport{ + RunID: rep.RunID, + Status: rep.Status, + Err: runErr, + Conflicts: len(rep.NodeConflicts), + Contested: len(rep.NodeContested), + } } } diff --git a/cmd/squirrel/conflicts.go b/cmd/squirrel/conflicts.go new file mode 100644 index 0000000..b4344a7 --- /dev/null +++ b/cmd/squirrel/conflicts.go @@ -0,0 +1,134 @@ +package main + +import ( + "database/sql" + "encoding/hex" + "fmt" + "io" + "text/tabwriter" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/store" +) + +// newConflictsCmd returns the `squirrel conflicts` command: the question +// half of the contested-freeze pair (#158, F27). It lists the paths this +// node knows are frozen by a peer-sync conflict — divergent edits that +// stopped ping-ponging once the receiver refused to keep minting +// `.squirrel-conflicts/` copies. Both versions stay preserved; the listing +// shows which is live, which is preserved and where, when the freeze +// landed, and which peer it involves. The change half — `conflicts +// resolve` — is the explicit human act that clears a freeze. +func newConflictsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "conflicts", + Short: "List unresolved contested paths (frozen peer-sync conflicts)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return runConflicts(cmd) + }, + } + cmd.AddCommand(newConflictsResolveCmd()) + return cmd +} + +func runConflicts(cmd *cobra.Command) error { + cfg, err := tryLoadConfig(cmd) + if err != nil { + return err + } + s, err := openStore(cmd, cfg) + if err != nil { + return err + } + defer s.Close() + + contested, err := s.ListContestedPaths(cmd.Context()) + if err != nil { + return fmt.Errorf("list contested paths: %w", err) + } + out := cmd.OutOrStdout() + if len(contested) == 0 { + fmt.Fprintln(out, "no contested paths — nothing frozen") + return nil + } + volumes, err := loadVolumeNames(cmd, s) + if err != nil { + return err + } + nodes := loadContestedNodeNames(cmd, s, contested) + return printContested(out, contested, volumes, nodes) +} + +// loadContestedNodeNames resolves the peer_node_id of every contested row +// to a name. A dropped or unattributed (zero) id is simply omitted; the +// renderer falls back to "—". Best-effort: a lookup error skips the id +// rather than failing the whole listing. +func loadContestedNodeNames(cmd *cobra.Command, s *store.Store, rows []store.ContestedPath) map[int64]string { + out := map[int64]string{} + for _, c := range rows { + if c.PeerNodeID == 0 { + continue + } + if _, ok := out[c.PeerNodeID]; ok { + continue + } + node, err := s.GetNodeByID(cmd.Context(), c.PeerNodeID) + if err != nil { + continue + } + out[c.PeerNodeID] = node.Name + } + return out +} + +func printContested(w io.Writer, rows []store.ContestedPath, volumes, nodes map[int64]string) error { + fmt.Fprintf(w, "%d contested path(s) — resolve each with `squirrel conflicts resolve `:\n\n", len(rows)) + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, "VOLUME\tPATH\tLIVE\tPRESERVED\tPRESERVED AT\tSINCE\tPEER") + for _, c := range rows { + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + volumeLabel(sql.NullInt64{Int64: c.VolumeID, Valid: true}, volumes), + c.Path, + shortDigest(c.LiveBlake3), + shortDigest(c.PreservedBlake3), + dashIfEmpty(c.PreservedAtPath), + formatStarted(c.RaisedAtNs), + contestedPeerLabel(c.PeerNodeID, nodes), + ) + } + return tw.Flush() +} + +// contestedPeerLabel names the peer a freeze involves, or "—" when +// unattributed (zero id) or the node row has vanished. +func contestedPeerLabel(peerNodeID int64, nodes map[int64]string) string { + if peerNodeID == 0 { + return "—" + } + if name, ok := nodes[peerNodeID]; ok { + return name + } + return fmt.Sprintf("id=%d", peerNodeID) +} + +// shortDigest renders the first 12 hex chars of a digest for a compact +// column, or "—" for an unknown (nil) digest. +func shortDigest(b []byte) string { + if len(b) == 0 { + return "—" + } + h := hex.EncodeToString(b) + if len(h) > 12 { + return h[:12] + } + return h +} + +func dashIfEmpty(s string) string { + if s == "" { + return "—" + } + return s +} diff --git a/cmd/squirrel/conflicts_resolve.go b/cmd/squirrel/conflicts_resolve.go new file mode 100644 index 0000000..2cd05a9 --- /dev/null +++ b/cmd/squirrel/conflicts_resolve.go @@ -0,0 +1,96 @@ +package main + +import ( + "encoding/hex" + "fmt" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/store" +) + +// newConflictsResolveCmd returns `squirrel conflicts resolve +// `: the explicit human act that clears a contested freeze (#158, +// F27). Resolution is deliberately a human decision — squirrel never +// resolves a conflict on its own ("irreversible acts remain human"). The +// command clears the standing freeze so syncs flow again; it does not +// itself move bytes between the two versions. Both remain preserved: the +// winner live at the path, the loser under `.squirrel-conflicts/`. To +// adopt the *other* version, restore the preserved copy first — the output +// names exactly where it is. +// +// Clearing loses no history: every conflict run survives in the runs +// table, the raise and this clear survive in runs_audit, and the preserved +// bytes stay on disk. If a peer still holds the divergent version, its +// next sync will re-freeze the path — a fresh, singly-preserved, human- +// notified event, not the unbounded ping-pong F27 described. +func newConflictsResolveCmd() *cobra.Command { + return &cobra.Command{ + Use: "resolve ", + Short: "Clear a contested freeze so syncs flow again (explicit human act)", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return runConflictsResolve(cmd, args[0], args[1]) + }, + } +} + +func runConflictsResolve(cmd *cobra.Command, volumeName, relPath string) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + s, err := openStore(cmd, cfg) + if err != nil { + return err + } + defer s.Close() + + vol, err := s.GetVolumeByName(cmd.Context(), volumeName) + if err != nil { + if store.IsNotFound(err) { + return fmt.Errorf("no volume named %q", volumeName) + } + return fmt.Errorf("lookup volume %q: %w", volumeName, err) + } + latch, err := s.GetContestedPath(cmd.Context(), vol.ID, relPath) + if err != nil { + if store.IsNotFound(err) { + return fmt.Errorf("path %q in volume %q is not contested", relPath, volumeName) + } + return fmt.Errorf("look up contested %q: %w", relPath, err) + } + cleared, err := s.ClearContested(cmd.Context(), vol.ID, relPath, ackOperator()) + if err != nil { + return err + } + if !cleared { + // Raced with another resolve between the read and the write — the + // freeze is gone either way, the outcome the operator asked for. + return fmt.Errorf("path %q in volume %q is not contested", relPath, volumeName) + } + printResolved(cmd, volumeName, relPath, latch) + return nil +} + +// printResolved confirms the clear and points the operator at both +// preserved versions so adopting the non-live one stays a deliberate, +// informed act rather than a surprise the resolve silently performed. +func printResolved(cmd *cobra.Command, volumeName, relPath string, latch store.ContestedPath) { + out := cmd.OutOrStdout() + fmt.Fprintf(out, "resolved %s %s: contested freeze cleared — syncs may flow again\n", volumeName, relPath) + fmt.Fprintf(out, " live version kept: %s\n", digestOrUnknown(latch.LiveBlake3)) + if latch.PreservedAtPath != "" { + fmt.Fprintf(out, " preserved version %s remains at %s (restore it to adopt that version instead)\n", + digestOrUnknown(latch.PreservedBlake3), latch.PreservedAtPath) + } +} + +// digestOrUnknown renders a digest as full hex, or a placeholder when the +// latch recorded no digest (an older or partial freeze record). +func digestOrUnknown(b []byte) string { + if len(b) == 0 { + return "(unknown)" + } + return hex.EncodeToString(b) +} diff --git a/cmd/squirrel/conflicts_test.go b/cmd/squirrel/conflicts_test.go new file mode 100644 index 0000000..4802550 --- /dev/null +++ b/cmd/squirrel/conflicts_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/mbertschler/squirrel/store" +) + +// raiseTestContested opens the CLI's DB directly and freezes one path, so +// the conflicts commands have a standing latch to read and clear without +// driving a whole peer sync. The store handle is closed before returning +// so the next CLI invocation has no concurrent connection from this +// process. +func raiseTestContested(t *testing.T, dbPath, volumeName, relPath string) { + t.Helper() + s, err := store.OpenWithOptions(dbPath, store.OpenOptions{}) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer s.Close() + ctx := context.Background() + vol, err := s.GetVolumeByName(ctx, volumeName) + if err != nil { + t.Fatalf("GetVolumeByName %q: %v", volumeName, err) + } + run, err := s.BeginIndexRun(ctx, store.RunKindIndex, vol.ID, false) + if err != nil { + t.Fatalf("BeginIndexRun: %v", err) + } + if err := s.RaiseContested(ctx, store.ContestedPath{ + VolumeID: vol.ID, + Path: relPath, + LiveBlake3: make([]byte, 32), + PreservedBlake3: make([]byte, 32), + PreservedAtPath: ".squirrel-conflicts/run-" + relPath, + RaisedRunID: run, + }); err != nil { + t.Fatalf("RaiseContested: %v", err) + } +} + +// TestCLIConflictsListAndResolve walks the question→change pair: an empty +// listing, a frozen path surfacing in the list, and an explicit resolve +// that clears it so the list empties again. +func TestCLIConflictsListAndResolve(t *testing.T) { + src := t.TempDir() + writeTestFile(t, filepath.Join(src, "a.txt"), "hello") + f := writeConfigFor(t, map[string]string{"src": src}) + runCLI(t, "--config", f.configPath, "index", "src") + + // Nothing frozen yet. + if out := runCLI(t, "--config", f.configPath, "conflicts"); !strings.Contains(out, "no contested paths") { + t.Fatalf("empty listing = %q, want 'no contested paths'", out) + } + + raiseTestContested(t, f.dbPath, "src", "a.txt") + + out := runCLI(t, "--config", f.configPath, "conflicts") + if !strings.Contains(out, "a.txt") || !strings.Contains(out, "1 contested path") { + t.Fatalf("listing = %q, want the frozen a.txt", out) + } + + // Resolve clears it and reports what was kept. + out = runCLI(t, "--config", f.configPath, "conflicts", "resolve", "src", "a.txt") + if !strings.Contains(out, "contested freeze cleared") { + t.Fatalf("resolve output = %q, want cleared confirmation", out) + } + + if out := runCLI(t, "--config", f.configPath, "conflicts"); !strings.Contains(out, "no contested paths") { + t.Fatalf("listing after resolve = %q, want empty", out) + } +} + +// TestCLIConflictsResolveUnknown: resolving a path that isn't contested is +// a clean error, not a silent no-op. +func TestCLIConflictsResolveUnknown(t *testing.T) { + src := t.TempDir() + writeTestFile(t, filepath.Join(src, "a.txt"), "hello") + f := writeConfigFor(t, map[string]string{"src": src}) + runCLI(t, "--config", f.configPath, "index", "src") + + _, err := runCLIExpectErr(t, "--config", f.configPath, "conflicts", "resolve", "src", "nope.txt") + if err == nil || !strings.Contains(err.Error(), "is not contested") { + t.Fatalf("error = %v, want 'is not contested'", err) + } + + // An unknown volume is a distinct, legible error. + _, err = runCLIExpectErr(t, "--config", f.configPath, "conflicts", "resolve", "ghost", "a.txt") + if err == nil || !strings.Contains(err.Error(), "no volume named") { + t.Fatalf("error = %v, want 'no volume named'", err) + } +} diff --git a/cmd/squirrel/root.go b/cmd/squirrel/root.go index 4bb9def..e6c4244 100644 --- a/cmd/squirrel/root.go +++ b/cmd/squirrel/root.go @@ -52,6 +52,7 @@ func newRootCmd() *cobra.Command { root.AddCommand(newAgentCmd()) root.AddCommand(newAuditCmd()) root.AddCommand(newVerifyCmd()) + root.AddCommand(newConflictsCmd()) root.AddCommand(newPeerSyncCmd()) root.AddCommand(newTUICmd()) root.AddCommand(newDBCmd()) diff --git a/cmd/squirrel/runs.go b/cmd/squirrel/runs.go index 67e96a0..650a00e 100644 --- a/cmd/squirrel/runs.go +++ b/cmd/squirrel/runs.go @@ -83,9 +83,31 @@ func runRuns(cmd *cobra.Command, volumeName string, limit int) error { return fmt.Errorf("list destination alarms: %w", err) } printActiveAlarms(cmd.OutOrStdout(), alarms) + contested, err := s.ListContestedPaths(cmd.Context()) + if err != nil { + return fmt.Errorf("list contested paths: %w", err) + } + printContestedReminder(cmd.OutOrStdout(), contested) return nil } +// printContestedReminder surfaces standing contested freezes beneath the +// runs listing (#158, F27). Like the alarms reminder, a single old +// conflict run scrolls away, so the audit surface repeats the standing +// condition until a human resolves it — and points at the dedicated +// `squirrel conflicts` question-command for the detail. +func printContestedReminder(w io.Writer, contested []store.ContestedPath) { + if len(contested) == 0 { + return + } + fmt.Fprintf(w, "\n%d contested path(s) frozen — inspect with `squirrel conflicts`, clear with `squirrel conflicts resolve`:\n", + len(contested)) + for _, c := range contested { + fmt.Fprintf(w, " CONTESTED volume=%d %s since %s\n", + c.VolumeID, c.Path, formatStarted(c.RaisedAtNs)) + } +} + // printActiveAlarms surfaces standing per-destination alarms beneath the // runs listing (#157, F30). A verify mismatch latches an alarm that must // stay visible until an operator clears it — a single old run row scrolls @@ -132,11 +154,22 @@ func loadNodeNames(cmd *cobra.Command, s *store.Store, runs []store.Run) (map[in return out, nil } -// loadConflictCounts derives the conflict count for every peer-sync -// run in the listing via one grouped query against `files`. The v6 -// schema doesn't carry the count on the run row itself; every -// conflict inserts one row under `.squirrel-conflicts/run-/`, so -// the prefix-count is the audit number we want. +// loadConflictCounts derives the conflict count for every peer-sync run in +// the listing. The schema doesn't carry the count on the run row itself, +// so it comes from two derivations merged per run: +// +// - Receiver side: every conflict inserts one row under +// `.squirrel-conflicts/run-/`, so the prefix-count is the number +// of losers this run preserved. +// - Initiator side: the conflict was frozen on a *remote* receiver, so +// no local `.squirrel-conflicts/` rows exist; instead the run raised +// local contested_paths latches (one per conflict + contested +// refusal), so the raised-by-run count is the initiator's signal +// (#158, F27). +// +// A run is one side of a sync, so at most one derivation is non-zero for +// it; taking the max yields the right figure without double-counting the +// rare case where both happen to fire on the same id. func loadConflictCounts(cmd *cobra.Command, s *store.Store, runs []store.Run) (map[int64]int, error) { var peerSyncIDs []int64 for _, r := range runs { @@ -147,10 +180,23 @@ func loadConflictCounts(cmd *cobra.Command, s *store.Store, runs []store.Run) (m if len(peerSyncIDs) == 0 { return map[int64]int{}, nil } - counts, err := s.CountFilesFirstSeenByRunWithPathPrefix(cmd.Context(), peerSyncIDs, sync.ConflictsDirName) + preserved, err := s.CountFilesFirstSeenByRunWithPathPrefix(cmd.Context(), peerSyncIDs, sync.ConflictsDirName) if err != nil { return nil, fmt.Errorf("conflict counts: %w", err) } + frozen, err := s.CountContestedRaisedByRun(cmd.Context(), peerSyncIDs) + if err != nil { + return nil, fmt.Errorf("contested counts: %w", err) + } + counts := make(map[int64]int, len(preserved)+len(frozen)) + for id, n := range preserved { + counts[id] = n + } + for id, n := range frozen { + if n > counts[id] { + counts[id] = n + } + } return counts, nil } diff --git a/cmd/squirrel/sync.go b/cmd/squirrel/sync.go index 9abcc6f..02d0e3b 100644 --- a/cmd/squirrel/sync.go +++ b/cmd/squirrel/sync.go @@ -223,6 +223,14 @@ func printSyncReport(w io.Writer, rep sync.Report, runErr error) { fmt.Fprintf(w, " preserved at %s:%s\n", rep.Destination, c.PreservedAtPath) } } + for _, c := range rep.NodeContested { + fmt.Fprintf(w, " contested %s: frozen by a prior conflict — bytes refused, not delivered\n", c.Path) + if c.PreservedAtPath != "" { + fmt.Fprintf(w, " versions live %s, preserved %s at %s:%s\n", + c.LiveBlake3Hex, c.PreservedBlake3Hex, rep.Destination, c.PreservedAtPath) + } + fmt.Fprintf(w, " resolve with: squirrel conflicts resolve %s %s\n", rep.Volume, c.Path) + } for _, ff := range r.FailedFiles { // Some rclone errors (auth, listing, fatal copy) have no Object. // Render those as a bare "error: ..." rather than "error : ...". diff --git a/design/friction-log.md b/design/friction-log.md index 5b563db..2816288 100644 --- a/design/friction-log.md +++ b/design/friction-log.md @@ -313,6 +313,22 @@ re-supersede a path whose live row lost a conflict since your last delivery without operator action), and a `squirrel conflicts` question-command to list unresolved ones. +*Resolved (#158, contested-freeze, option 2a).* A conflict now raises a +`contested_paths` latch on the receiver (schema v26); while it stands the +`/plan` classifier answers a divergent re-assertion from any peer with a +new `contested` disposition (version-gated for older peers) instead of +minting another `.squirrel-conflicts/run-N/` copy — the flip-flop stops at +the first conflict, preserved once. Both versions stay reachable (winner +live, loser under `.squirrel-conflicts/`). Conflict/contested counts flow +into `SyncRunReport`, the initiators' run rows (the CONFLICTS column), a +`scheduler.conflict` warn line, and a "Contested paths" TUI badge — each +node mirrors the freeze into its own latch, so the losing edge sees it too, +not only the hub. `squirrel conflicts` lists the frozen paths and +`squirrel conflicts resolve ` clears the latch (the explicit +human act that unfreezes; squirrel never resolves on its own). Adopting the +non-live version stays a deliberate `restore`, not something resolve does +silently. + **F29 · S1 — relayed offload against a cold archive is structurally unreachable, so no machine in the reference household can offload anything.** The endgame of the whole design — htpc/laptop dropping diff --git a/store/contested_paths.go b/store/contested_paths.go new file mode 100644 index 0000000..e5956a8 --- /dev/null +++ b/store/contested_paths.go @@ -0,0 +1,238 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// ContestedPath is one row of the contested_paths latch: a path frozen by +// a peer-sync conflict so divergent edits stop ping-ponging (#158, F27). +// While the row stands the receiver refuses to re-supersede the path (the +// `contested` disposition) and every node that knows about it surfaces a +// badge; both versions stay preserved — the winner live at Path, the loser +// under .squirrel-conflicts/ at PreservedAtPath. It is derived standing +// state, like DestinationAlarm: the permanent record of every conflict +// lives in runs/runs_audit and the preserved bytes on disk, so clearing +// the latch loses no history. +// +// LiveBlake3 is the frozen winner (nil when a node recorded the freeze +// without knowing the winning digest); PreservedBlake3 is the loser. +// PeerNodeID is the node that caused the freeze — the initiator peer on +// the receiver, or the destination peer on an initiator — and is 0 when +// unattributed. RaisedRunID references the run that first raised the +// latch; RaisedAtNs is stable across repeated conflicts so a surface can +// show "contested since". +type ContestedPath struct { + VolumeID int64 + Path string + LiveBlake3 []byte + PreservedBlake3 []byte + PreservedAtPath string + PeerNodeID int64 + RaisedRunID int64 + RaisedAtNs int64 +} + +// RaiseContested latches a contested freeze on (volume, path) when none is +// already active. The raise is idempotent: an existing latch keeps its +// original digests, timestamp, and run so a repeated conflict does not +// reset "contested since" or append a duplicate audit row — the freeze is +// recorded once, not every cadence tick (the F27 fix). A +// 'contested-raise' runs_audit entry is written against RaisedRunID in the +// same transaction as the insert, only when a fresh latch is created. +func (s *Store) RaiseContested(ctx context.Context, c ContestedPath) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin raise contested %q: %w", c.Path, err) + } + defer func() { _ = tx.Rollback() }() + + atNs := NowNs() + res, err := tx.ExecContext(ctx, ` + INSERT INTO contested_paths ( + volume_id, path, live_blake3, preserved_blake3, preserved_at_path, + peer_node_id, raised_run_id, raised_at_ns + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(volume_id, path) DO NOTHING + `, c.VolumeID, c.Path, nullBlob(c.LiveBlake3), nullBlob(c.PreservedBlake3), + nullString(c.PreservedAtPath), nullPeerID(c.PeerNodeID), c.RaisedRunID, atNs) + if err != nil { + return fmt.Errorf("raise contested %q: %w", c.Path, err) + } + inserted, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("raise contested %q rows affected: %w", c.Path, err) + } + if inserted == 0 { + // Already frozen: keep the original latch untouched. + return tx.Commit() + } + if err := appendRunAuditTx(ctx, tx, RunAuditEntry{ + RunID: c.RaisedRunID, Transition: TransitionContestedRaise, Note: c.Path, + }, atNs); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit raise contested %q: %w", c.Path, err) + } + return nil +} + +// ClearContested clears the active freeze on (volume, path), if any, and +// appends a 'contested-clear' runs_audit entry (tagged with operator) +// against the run that raised it. Returns whether a latch was active. +// Deleting the latch loses no history: the raise and this clear both +// survive in runs_audit, every conflict run survives in runs, and the +// preserved bytes stay on disk under .squirrel-conflicts/. +func (s *Store) ClearContested(ctx context.Context, volumeID int64, path, operator string) (bool, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return false, fmt.Errorf("begin clear contested %q: %w", path, err) + } + defer func() { _ = tx.Rollback() }() + + var raisedRunID int64 + err = tx.QueryRowContext(ctx, + `SELECT raised_run_id FROM contested_paths WHERE volume_id = ? AND path = ?`, + volumeID, path).Scan(&raisedRunID) + if IsNotFound(err) { + return false, tx.Commit() + } + if err != nil { + return false, fmt.Errorf("read contested %q: %w", path, err) + } + if _, err := tx.ExecContext(ctx, + `DELETE FROM contested_paths WHERE volume_id = ? AND path = ?`, + volumeID, path); err != nil { + return false, fmt.Errorf("clear contested %q: %w", path, err) + } + if err := appendRunAuditTx(ctx, tx, RunAuditEntry{ + RunID: raisedRunID, Transition: TransitionContestedClear, Operator: operator, Note: path, + }, NowNs()); err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("commit clear contested %q: %w", path, err) + } + return true, nil +} + +// IsPathContested reports whether (volume, path) is frozen and, if so, +// returns the frozen winner's digest so the classifier can distinguish the +// winner re-asserting (digest matches — allowed) from a divergent +// re-assertion (digest differs — refused). liveBlake3 is nil when the +// freeze was recorded without a known winner digest. +func (s *Store) IsPathContested(ctx context.Context, volumeID int64, path string) (liveBlake3 []byte, contested bool, err error) { + var digest []byte + err = s.db.QueryRowContext(ctx, + `SELECT live_blake3 FROM contested_paths WHERE volume_id = ? AND path = ?`, + volumeID, path).Scan(&digest) + if IsNotFound(err) { + return nil, false, nil + } + if err != nil { + return nil, false, fmt.Errorf("check contested %q: %w", path, err) + } + return digest, true, nil +} + +// GetContestedPath returns the active freeze on (volume, path), or +// sql.ErrNoRows (IsNotFound) when the path is not contested. +func (s *Store) GetContestedPath(ctx context.Context, volumeID int64, path string) (ContestedPath, error) { + row := s.db.QueryRowContext(ctx, + `SELECT `+contestedColumns+` FROM contested_paths WHERE volume_id = ? AND path = ?`, + volumeID, path) + return scanContestedPath(row.Scan) +} + +// ListContestedPaths returns every active freeze, ordered by (volume, path) +// so the listing is deterministic. An empty slice means nothing is frozen. +func (s *Store) ListContestedPaths(ctx context.Context) ([]ContestedPath, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT `+contestedColumns+` FROM contested_paths ORDER BY volume_id, path`) + if err != nil { + return nil, fmt.Errorf("list contested paths: %w", err) + } + defer rows.Close() + var out []ContestedPath + for rows.Next() { + c, err := scanContestedPath(rows.Scan) + if err != nil { + return nil, err + } + out = append(out, c) + } + return out, rows.Err() +} + +// CountContestedRaisedByRun returns a map from run id to the number of +// contested_paths latches that run raised, for the given run ids. Only +// runs with a non-zero count appear as keys. `squirrel runs` uses it to +// fill the CONFLICTS column for an initiator's own sync runs — where the +// conflict was frozen on a remote receiver, so the receiver-side +// `.squirrel-conflicts/run-N/` prefix count is zero on this node. +func (s *Store) CountContestedRaisedByRun(ctx context.Context, runIDs []int64) (map[int64]int, error) { + out := make(map[int64]int, len(runIDs)) + if len(runIDs) == 0 { + return out, nil + } + placeholders := strings.Repeat("?,", len(runIDs)-1) + "?" + args := make([]any, 0, len(runIDs)) + for _, id := range runIDs { + args = append(args, id) + } + rows, err := s.db.QueryContext(ctx, + `SELECT raised_run_id, COUNT(*) FROM contested_paths + WHERE raised_run_id IN (`+placeholders+`) + GROUP BY raised_run_id`, args...) + if err != nil { + return nil, fmt.Errorf("count contested by run: %w", err) + } + defer rows.Close() + for rows.Next() { + var id int64 + var n int + if err := rows.Scan(&id, &n); err != nil { + return nil, fmt.Errorf("scan contested count row: %w", err) + } + out[id] = n + } + return out, rows.Err() +} + +const contestedColumns = `volume_id, path, live_blake3, preserved_blake3, preserved_at_path, peer_node_id, raised_run_id, raised_at_ns` + +func scanContestedPath(scan func(...any) error) (ContestedPath, error) { + var ( + c ContestedPath + prsvd sql.NullString + peerID sql.NullInt64 + ) + if err := scan(&c.VolumeID, &c.Path, &c.LiveBlake3, &c.PreservedBlake3, + &prsvd, &peerID, &c.RaisedRunID, &c.RaisedAtNs); err != nil { + return ContestedPath{}, err + } + c.PreservedAtPath = prsvd.String + c.PeerNodeID = peerID.Int64 + return c, nil +} + +// nullBlob renders a digest for an INSERT bind: an empty slice becomes a +// NULL BLOB (the "digest unknown" convention), any other value binds as-is. +func nullBlob(b []byte) any { + if len(b) == 0 { + return nil + } + return b +} + +// nullPeerID maps a zero node id to a NULL peer_node_id (unattributed +// freeze) and any positive id to a valid one. +func nullPeerID(id int64) sql.NullInt64 { + if id == 0 { + return sql.NullInt64{} + } + return sql.NullInt64{Int64: id, Valid: true} +} diff --git a/store/contested_paths_test.go b/store/contested_paths_test.go new file mode 100644 index 0000000..672a8dd --- /dev/null +++ b/store/contested_paths_test.go @@ -0,0 +1,143 @@ +package store + +import ( + "bytes" + "context" + "testing" +) + +// TestContestedPathLifecycle exercises raise → idempotent re-raise → +// query → clear and the runs_audit trail each transition leaves. +func TestContestedPathLifecycle(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + volID := makeVolume(t, s, "/v") + first := makeRun(t, s, volID) + live := digest(0xAA) + loser := digest(0xBB) + + c := ContestedPath{ + VolumeID: volID, + Path: "doc.md", + LiveBlake3: live, + PreservedBlake3: loser, + PreservedAtPath: ".squirrel-conflicts/run-1/doc.md", + RaisedRunID: first, + } + if err := s.RaiseContested(ctx, c); err != nil { + t.Fatalf("RaiseContested: %v", err) + } + + got, err := s.GetContestedPath(ctx, volID, "doc.md") + if err != nil { + t.Fatalf("GetContestedPath: %v", err) + } + if !bytes.Equal(got.LiveBlake3, live) || !bytes.Equal(got.PreservedBlake3, loser) { + t.Fatalf("digests = live %x preserved %x, want %x / %x", got.LiveBlake3, got.PreservedBlake3, live, loser) + } + if got.PreservedAtPath != c.PreservedAtPath || got.RaisedRunID != first || got.RaisedAtNs == 0 { + t.Fatalf("latch = %+v, want preserved-path/run/timestamp set", got) + } + firstAt := got.RaisedAtNs + + // IsPathContested returns the frozen winner so the classifier can tell + // the winner re-asserting from a divergent re-assertion. + winner, contested, err := s.IsPathContested(ctx, volID, "doc.md") + if err != nil || !contested { + t.Fatalf("IsPathContested = (%x, %v, %v), want frozen", winner, contested, err) + } + if !bytes.Equal(winner, live) { + t.Fatalf("IsPathContested winner = %x, want %x", winner, live) + } + + // A second conflict on the same path keeps the original latch: same + // run, same timestamp — "contested since" never resets, and no second + // copy is minted (the F27 fix is preserved-once). + second := makeRun(t, s, volID) + if err := s.RaiseContested(ctx, ContestedPath{ + VolumeID: volID, Path: "doc.md", LiveBlake3: digest(0xCC), + PreservedBlake3: digest(0xDD), RaisedRunID: second, + }); err != nil { + t.Fatalf("re-raise: %v", err) + } + got, err = s.GetContestedPath(ctx, volID, "doc.md") + if err != nil { + t.Fatalf("GetContestedPath after re-raise: %v", err) + } + if got.RaisedRunID != first || got.RaisedAtNs != firstAt || !bytes.Equal(got.LiveBlake3, live) { + t.Fatalf("re-raise mutated the latch: %+v", got) + } + + // Exactly one contested-raise audit row against the first run; none + // against the second (the re-raise was a no-op). + if n := countTransition(t, s, first, TransitionContestedRaise); n != 1 { + t.Fatalf("first run contested-raise count = %d, want 1", n) + } + if n := countTransition(t, s, second, TransitionContestedRaise); n != 0 { + t.Fatalf("second run contested-raise count = %d, want 0 (idempotent)", n) + } + + // The run-row conflict column derivation counts one freeze against the + // raising run. + counts, err := s.CountContestedRaisedByRun(ctx, []int64{first, second}) + if err != nil { + t.Fatalf("CountContestedRaisedByRun: %v", err) + } + if counts[first] != 1 || counts[second] != 0 { + t.Fatalf("raised-by-run counts = %v, want {first:1}", counts) + } + + // Clear via resolve: recorded against the raising run, tagged operator. + cleared, err := s.ClearContested(ctx, volID, "doc.md", "alice") + if err != nil { + t.Fatalf("ClearContested: %v", err) + } + if !cleared { + t.Fatal("ClearContested reported no active freeze") + } + if _, err := s.GetContestedPath(ctx, volID, "doc.md"); !IsNotFound(err) { + t.Fatalf("latch still present after clear: %v", err) + } + if _, contested, _ := s.IsPathContested(ctx, volID, "doc.md"); contested { + t.Fatal("IsPathContested still reports frozen after clear") + } + if n := countTransition(t, s, first, TransitionContestedClear); n != 1 { + t.Fatalf("contested-clear count against raising run = %d, want 1", n) + } + + // Clearing an already-cleared path is a benign no-op, not an error. + cleared, err = s.ClearContested(ctx, volID, "doc.md", "alice") + if err != nil || cleared { + t.Fatalf("second ClearContested = (%v, %v), want (false, nil)", cleared, err) + } +} + +// TestContestedPathUnknownDigest covers a freeze recorded without a known +// winner digest (an initiator mirror that lost the hex): the NULL columns +// round-trip as nil, and IsPathContested still reports frozen. +func TestContestedPathUnknownDigest(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + volID := makeVolume(t, s, "/v") + run := makeRun(t, s, volID) + + if err := s.RaiseContested(ctx, ContestedPath{ + VolumeID: volID, Path: "a/b.txt", RaisedRunID: run, + }); err != nil { + t.Fatalf("RaiseContested: %v", err) + } + winner, contested, err := s.IsPathContested(ctx, volID, "a/b.txt") + if err != nil || !contested { + t.Fatalf("IsPathContested = (%x, %v, %v), want frozen", winner, contested, err) + } + if winner != nil { + t.Fatalf("winner = %x, want nil (unknown digest)", winner) + } + list, err := s.ListContestedPaths(ctx) + if err != nil { + t.Fatalf("ListContestedPaths: %v", err) + } + if len(list) != 1 || list[0].Path != "a/b.txt" || list[0].PreservedAtPath != "" { + t.Fatalf("list = %+v, want one row with empty preserved path", list) + } +} diff --git a/store/migrations.go b/store/migrations.go index 927ebb6..440fae5 100644 --- a/store/migrations.go +++ b/store/migrations.go @@ -10,7 +10,7 @@ import ( ) // SchemaVersion is the schema version this binary writes and reads. -const SchemaVersion = 25 +const SchemaVersion = 26 // freshSchemaBaseline is the version applied to a brand-new database. The // chain in `migrations` continues from here. v1 is no longer reachable from @@ -61,6 +61,7 @@ func buildMigrations(mctx migrationCtx) []migration { {version: 23, up: migrateV22ToV23}, {version: 24, up: migrateV23ToV24}, {version: 25, up: migrateV24ToV25}, + {version: 26, up: migrateV25ToV26}, } } @@ -2173,3 +2174,67 @@ func createDestinationAlarmsV25(ctx context.Context, tx *sql.Tx) error { } return nil } + +// --- v25 → v26 --- + +// migrateV25ToV26 adds the contested_paths standing-state latch (#158, +// F27): the per-(volume, path) freeze that stops divergent edits from +// ping-ponging forever. A conflict raises a row; while it stands, the +// receiver refuses to re-supersede the path (the `contested` disposition) +// and the initiators surface a badge, so both versions stay preserved +// exactly once instead of a fresh `.squirrel-conflicts/run-N/` copy every +// cadence tick. Like destination_alarms (v25) and peer_sync_state, it is +// derived standing state: the permanent forensic record of every conflict +// lives in runs/runs_audit and the preserved bytes on disk, so clearing +// the latch (an explicit `squirrel conflicts resolve`) loses no history. +// +// The table is written on every node. On the receiver a row records the +// authoritative freeze; on an initiator a row mirrors what the receiver +// reported so its own `squirrel conflicts` and TUI badge answer "am I +// safe?" from its local index (no fleet view needed). STRICT per the +// new-table convention; additive, so no rebuild of any existing table. +func migrateV25ToV26(ctx context.Context, db *sql.DB) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + if err := createContestedPathsV26(ctx, tx); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO schema_version (version) VALUES (26)`); err != nil { + return fmt.Errorf("record schema v26: %w", err) + } + return tx.Commit() +} + +// createContestedPathsV26 creates the contested-freeze latch. The PK is +// (volume_id, path): at most one active freeze per path, which the raise +// path keeps idempotent so "contested since" stays stable across repeated +// conflicts. live_blake3 is the frozen winner and preserved_blake3 the +// loser; the classifier compares an incoming digest against live_blake3 to +// tell the winner re-asserting (allowed) from a divergent re-assertion +// (refused). peer_node_id is nullable — the initiator that caused the +// freeze on the receiver, or the destination peer on an initiator — and +// raised_run_id is a real FK to the run that raised it. No secondary +// index: reads are PK lookups or a full scan of the handful of rows in +// alarm, so an index would go unused and violate the +// low-cardinality-index guidance. +func createContestedPathsV26(ctx context.Context, tx *sql.Tx) error { + const ddl = `CREATE TABLE contested_paths ( + volume_id INTEGER NOT NULL REFERENCES volumes(id), + path TEXT NOT NULL, + live_blake3 BLOB CHECK (live_blake3 IS NULL OR length(live_blake3) = 32), + preserved_blake3 BLOB CHECK (preserved_blake3 IS NULL OR length(preserved_blake3) = 32), + preserved_at_path TEXT, + peer_node_id INTEGER REFERENCES nodes(id), + raised_run_id INTEGER NOT NULL REFERENCES runs(id), + raised_at_ns INTEGER NOT NULL, + PRIMARY KEY (volume_id, path) + ) STRICT` + if _, err := tx.ExecContext(ctx, ddl); err != nil { + return fmt.Errorf("create contested_paths: %w", err) + } + return nil +} diff --git a/store/runs_audit.go b/store/runs_audit.go index 9a9d04e..371d8b9 100644 --- a/store/runs_audit.go +++ b/store/runs_audit.go @@ -47,6 +47,16 @@ const ( // NULL) or against the run that raised it when an operator acks // (operator set). The note carries the destination. TransitionAlarmClear = "alarm-clear" + // TransitionContestedRaise records a contested_paths latch being + // raised against the sync run that hit the conflict (#158, F27): the + // freeze that stops the divergent-edit ping-pong. The note carries the + // volume-relative path. Written once per distinct freeze episode. + TransitionContestedRaise = "contested-raise" + // TransitionContestedClear records a contested_paths latch being + // cleared by an explicit `squirrel conflicts resolve` (operator set), + // against the run that raised it. The note carries the path. This is + // the deliberate human act that unfreezes the path. + TransitionContestedClear = "contested-clear" ) // RunAudit is one row of the insert-only runs_audit log: a single diff --git a/store/schema.sql b/store/schema.sql index 692cc9d..0713a6a 100644 --- a/store/schema.sql +++ b/store/schema.sql @@ -1,6 +1,6 @@ -- Generated by `go test ./store -update-schema` — DO NOT EDIT. -- --- Flattened snapshot of the squirrel index schema at version 25, for humans +-- Flattened snapshot of the squirrel index schema at version 26, for humans -- and agents who want the current shape without replaying the migration -- chain in migrations.go. It is NOT used to create or migrate databases — -- a fresh DB is built by applyV5 plus the migration registry. The golden @@ -27,6 +27,18 @@ CREATE TRIGGER contents_no_update BEFORE UPDATE ON contents SELECT RAISE(ABORT, 'contents is append-only; supersede the files row and insert new content instead of updating'); END; +CREATE TABLE contested_paths ( + volume_id INTEGER NOT NULL REFERENCES volumes(id), + path TEXT NOT NULL, + live_blake3 BLOB CHECK (live_blake3 IS NULL OR length(live_blake3) = 32), + preserved_blake3 BLOB CHECK (preserved_blake3 IS NULL OR length(preserved_blake3) = 32), + preserved_at_path TEXT, + peer_node_id INTEGER REFERENCES nodes(id), + raised_run_id INTEGER NOT NULL REFERENCES runs(id), + raised_at_ns INTEGER NOT NULL, + PRIMARY KEY (volume_id, path) + ) STRICT; + CREATE TABLE destination_alarms ( destination TEXT NOT NULL PRIMARY KEY, kind TEXT NOT NULL, diff --git a/sync/node.go b/sync/node.go index ba92d79..4affac2 100644 --- a/sync/node.go +++ b/sync/node.go @@ -114,6 +114,11 @@ type nodeSyncDriver struct { // receiverRunID is filled in after the begin handshake; rclone + // verify + close reference it. receiverRunID int64 + // peerNodeID is the local nodes-row id of the destination peer, + // resolved at /begin. Recorded on any contested_paths latch this run + // raises locally so the initiator's `squirrel conflicts` can name the + // peer the freeze lives on. + peerNodeID int64 // protocolVersion is the plan-exchange version negotiated at // /begin. Defaults to ProtocolVersionFlat so a missing field in // the receiver's response (older agent) keeps today's behaviour. @@ -215,8 +220,10 @@ func (d *nodeSyncDriver) run() error { // Conflicts flow through the transfer path: the receiver has // already pre-staged each loser under .squirrel-conflicts/ and // the original path is empty, so rclone treats the entry like a - // fresh transfer. + // fresh transfer. Contested paths are frozen: the receiver refused + // them, no bytes move, and the initiator only surfaces the freeze. d.report.NodeConflicts = plan.Conflicts + d.report.NodeContested = plan.Contested if !d.opts.DryRun { advance, err := captureDurabilityAdvance(d.ctx, d.store, d.volID) if err != nil { @@ -233,9 +240,68 @@ func (d *nodeSyncDriver) run() error { if err := d.phaseClose(); err != nil { return fmt.Errorf("close: %w", err) } + d.recordContestedLocally() return nil } +// recordContestedLocally mirrors the receiver-reported conflicts and +// contested refusals into this node's own contested_paths latch, so the +// losing/edge machine answers "am I safe?" from its local index — a badge +// in its TUI and a row in its `squirrel conflicts`, not just a line on the +// hub (#158, F27). Both a conflict (this node's bytes won live, the prior +// version was preserved) and a contested refusal (a frozen path declined +// this node's divergent bytes) leave the household master contested until +// a human resolves it, so both raise a marker. Raises are idempotent and +// best-effort: a failure here is logged into the report's warnings and +// never flips the sync's status — the authoritative freeze already lives +// on the receiver. +func (d *nodeSyncDriver) recordContestedLocally() { + if d.opts.DryRun { + return + } + for _, c := range d.report.NodeConflicts { + d.raiseContestedMarker(c.Path, c.InitiatorBlake3Hex, c.ReceiverBlake3Hex, c.PreservedAtPath) + } + for _, c := range d.report.NodeContested { + d.raiseContestedMarker(c.Path, c.LiveBlake3Hex, c.PreservedBlake3Hex, c.PreservedAtPath) + } +} + +// raiseContestedMarker records one local freeze. liveHex is the winning +// version (this node's own bytes for a conflict, the frozen winner for a +// contested refusal) and preservedHex the loser; a bad hex digest is +// dropped to nil so a marker is still recorded rather than lost. Errors +// are surfaced as a report warning, never fatal. +func (d *nodeSyncDriver) raiseContestedMarker(path, liveHex, preservedHex, preservedAt string) { + err := d.store.RaiseContested(d.ctx, store.ContestedPath{ + VolumeID: d.volID, + Path: path, + LiveBlake3: decodeHexOrNil(liveHex), + PreservedBlake3: decodeHexOrNil(preservedHex), + PreservedAtPath: preservedAt, + PeerNodeID: d.peerNodeID, + RaisedRunID: d.report.RunID, + }) + if err != nil { + d.report.Warnings = append(d.report.Warnings, + fmt.Sprintf("record contested %s locally: %v", path, err)) + } +} + +// decodeHexOrNil decodes a 64-char lowercase-hex digest to raw bytes, +// returning nil for an empty or malformed value so the caller records a +// marker with an unknown-digest field rather than dropping the freeze. +func decodeHexOrNil(hexStr string) []byte { + if hexStr == "" { + return nil + } + b, err := hex.DecodeString(hexStr) + if err != nil || len(b) != 32 { + return nil + } + return b +} + // phaseBegin opens a session with the receiver. The initiator's own // runs row is inserted first so the (peer_node_id, correlated_run_id) // pair lands the right way around: the receiver's id becomes our @@ -248,6 +314,7 @@ func (d *nodeSyncDriver) phaseBegin() error { if err != nil { return fmt.Errorf("record peer node: %w", err) } + d.peerNodeID = peer.ID self, err := d.store.GetSelfNode(d.ctx) if err != nil { return fmt.Errorf("look up self node: %w", err) @@ -276,7 +343,7 @@ func (d *nodeSyncDriver) phaseBegin() error { InitiatorNodeName: self.Name, InitiatorRunID: runID, DedupStrategy: d.node.DedupStrategy, - ProtocolVersion: syncproto.ProtocolVersionMerkleWalk, + ProtocolVersion: syncproto.ProtocolVersionContested, }) if err != nil { return err diff --git a/sync/node_test.go b/sync/node_test.go index 429350d..2ef8d79 100644 --- a/sync/node_test.go +++ b/sync/node_test.go @@ -451,6 +451,120 @@ func TestNodeSyncResolvesConflictOnLocalWriteOnReceiver(t *testing.T) { } } +// countRecvConflictDirs returns how many run-/ subdirectories exist +// under the receiver volume's .squirrel-conflicts — the F27 regression +// (unbounded copies) shows up as this climbing past 1. +func countRecvConflictDirs(t *testing.T, f *nodeFixture) int { + t.Helper() + entries, err := os.ReadDir(filepath.Join(f.recvVol.Path, ConflictsDirName)) + if err != nil { + if os.IsNotExist(err) { + return 0 + } + t.Fatalf("read conflicts dir: %v", err) + } + return len(entries) +} + +// TestNodeSyncContestedFreezeEndToEnd is the #158 acceptance across two +// real nodes: a first conflict freezes the path and raises a badge on the +// initiator too (not just the hub); a later divergent push is refused with +// `contested` and mints no second copy; and an explicit resolve on the +// receiver unfreezes so the next push flows. +func TestNodeSyncContestedFreezeEndToEnd(t *testing.T) { + f := setupNodeFixture(t) + ctx := context.Background() + + // Receiver holds a local-write doc.md; the initiator diverges. + v, err := f.recvStore.CreateVolume(ctx, f.recvVol.Name, f.recvVol.Path) + if err != nil { + t.Fatalf("seed receiver volume: %v", err) + } + runID, err := f.recvStore.BeginIndexRun(ctx, store.RunKindIndex, v.ID, false) + if err != nil { + t.Fatalf("BeginIndexRun: %v", err) + } + _ = f.recvStore.FinishRun(ctx, runID, store.RunStatusSuccess, "", 1) + if err := os.WriteFile(filepath.Join(f.recvVol.Path, "doc.md"), []byte("recvr"), 0o644); err != nil { + t.Fatal(err) + } + receiverDigest := hashFile(t, filepath.Join(f.recvVol.Path, "doc.md")) + if err := f.recvStore.Upsert(ctx, store.FileRow{ + VolumeID: v.ID, Path: "doc.md", Blake3: receiverDigest, SizeBytes: 5, MtimeNs: 50, + Status: store.StatusPresent, FirstSeenRunID: runID, LastSeenRunID: runID, IndexedAtNs: 50, + }, nil); err != nil { + t.Fatalf("seed receiver file row: %v", err) + } + if err := os.WriteFile(filepath.Join(f.initVol.Path, "doc.md"), []byte("initr"), 0o644); err != nil { + t.Fatal(err) + } + f.indexInitiator(t) + + // Round 1: conflict. Initiator wins live; the loser is preserved once. + rep, err := SyncNode(ctx, f.initStore, f.rcl, f.initVol, f.node, Options{Shallow: true}) + if err != nil || len(rep.NodeConflicts) != 1 { + t.Fatalf("round 1: err=%v conflicts=%+v, want one conflict", err, rep.NodeConflicts) + } + if got := countRecvConflictDirs(t, f); got != 1 { + t.Fatalf("conflict dirs after round 1 = %d, want 1", got) + } + // The receiver froze the path... + if _, contested, _ := f.recvStore.IsPathContested(ctx, v.ID, "doc.md"); !contested { + t.Fatal("receiver did not freeze doc.md after the conflict") + } + // ...and the initiator raised its own local badge (F27: not only the hub). + initVolRow, err := f.initStore.GetVolumeByName(ctx, "pics") + if err != nil { + t.Fatalf("initiator GetVolumeByName: %v", err) + } + if _, contested, _ := f.initStore.IsPathContested(ctx, initVolRow.ID, "doc.md"); !contested { + t.Fatal("initiator did not raise a local contested marker") + } + + // Round 2: the initiator pushes yet another divergent version. The + // freeze refuses it — NodeContested, no bytes delivered, no new copy. + if err := os.WriteFile(filepath.Join(f.initVol.Path, "doc.md"), []byte("third"), 0o644); err != nil { + t.Fatal(err) + } + f.indexInitiator(t) + rep2, err := SyncNode(ctx, f.initStore, f.rcl, f.initVol, f.node, Options{Shallow: true}) + if err != nil { + t.Fatalf("round 2 SyncNode: %v", err) + } + if len(rep2.NodeContested) != 1 || rep2.NodeContested[0].Path != "doc.md" { + t.Fatalf("round 2 NodeContested = %+v, want one doc.md entry", rep2.NodeContested) + } + if len(rep2.NodeConflicts) != 0 { + t.Fatalf("round 2 NodeConflicts = %+v, want none (frozen, not re-conflicted)", rep2.NodeConflicts) + } + if got := countRecvConflictDirs(t, f); got != 1 { + t.Fatalf("conflict dirs after round 2 = %d, want 1 (no second copy)", got) + } + // The receiver's live bytes are unchanged — the divergent push was refused. + if live, _ := os.ReadFile(filepath.Join(f.recvVol.Path, "doc.md")); string(live) != "initr" { + t.Fatalf("live content after refusal = %q, want unchanged 'initr'", live) + } + + // Resolve on the receiver clears the freeze (the explicit human act). + cleared, err := f.recvStore.ClearContested(ctx, v.ID, "doc.md", "operator") + if err != nil || !cleared { + t.Fatalf("ClearContested = (%v, %v), want cleared", cleared, err) + } + + // Round 3: with the freeze lifted, the initiator's pending edit flows + // through as an ordinary supersede. + rep3, err := SyncNode(ctx, f.initStore, f.rcl, f.initVol, f.node, Options{Shallow: true}) + if err != nil { + t.Fatalf("round 3 SyncNode: %v", err) + } + if len(rep3.NodeContested) != 0 || rep3.Status != store.RunStatusSuccess { + t.Fatalf("round 3 status=%q contested=%+v, want clean success", rep3.Status, rep3.NodeContested) + } + if live, _ := os.ReadFile(filepath.Join(f.recvVol.Path, "doc.md")); string(live) != "third" { + t.Fatalf("live content after resolve+sync = %q, want 'third'", live) + } +} + // TestNodeSyncVerifyMismatchPartialStatus simulates rclone "succeeding" // but the on-disk content being wrong (the agent's re-hash will // catch it). We inject the mismatch by writing different content diff --git a/sync/sync.go b/sync/sync.go index 4fcf57f..a32c8ea 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -157,6 +157,16 @@ type Report struct { // and the new BLAKE3 plus the receiver-relative preserved path so // the CLI can render a meaningful "review at " pointer. NodeConflicts []syncproto.ConflictDetail + // NodeContested is non-empty when the receiver refused one or more + // paths with the `contested` disposition: paths frozen by a prior + // conflict whose divergent re-assertion is declined until an operator + // resolves it (#158, F27). The initiator's bytes for these paths did + // NOT land — both existing versions stay preserved on the receiver. + // Each record carries the frozen winner + preserved-loser digests and + // the receiver-relative preserved location so the initiator mirrors + // the freeze into its own contested marker and points the operator at + // both versions. + NodeContested []syncproto.ContestedDetail // SnapshotErr captures a failure to take the post-sync index // snapshot or to ride it along to the destination (#75). It is // strictly defense-in-depth: a snapshot failure must not flip a diff --git a/syncproto/syncproto.go b/syncproto/syncproto.go index 4c61fa0..4601851 100644 --- a/syncproto/syncproto.go +++ b/syncproto/syncproto.go @@ -67,6 +67,20 @@ const DispositionSupersede = "supersede" // declared content origin. const DispositionConflict = "conflict" +// DispositionContested — the path is frozen: a prior conflict at this +// path raised a contested latch on the receiver (see +// store.ContestedPath), and this initiator's bytes differ from the +// frozen winner. Rather than mint yet another `.squirrel-conflicts/` +// copy every cadence tick (the F27 ping-pong), the receiver refuses to +// re-supersede: no bytes move, no new row is written, and both existing +// versions stay preserved (the winner live at the path, the loser under +// `.squirrel-conflicts/`). The initiator surfaces the refusal to its +// operator; only an explicit `squirrel conflicts resolve` clears the +// latch and lets syncs flow again. The disposition is version-gated — +// only sent to an initiator that negotiated ProtocolVersionContested; +// an older initiator falls back to the legacy `conflict` disposition. +const DispositionContested = "contested" + // DispositionCopyFromExisting — receiver has no live row at this path // but holds the requested blake3 at a different path in the same volume. // Instead of forcing the initiator to re-transfer the bytes over the @@ -105,6 +119,16 @@ const ProtocolVersionFlat = 1 // scope reduction on the input, not a new disposition. const ProtocolVersionMerkleWalk = 2 +// ProtocolVersionContested adds the contested-freeze disposition (#158, +// F27): once a path is frozen, the receiver answers a divergent +// re-assertion with DispositionContested instead of minting another +// `.squirrel-conflicts/` copy. The version gate keeps an older initiator +// safe — it never learns the `contested` disposition it can't interpret, +// so the receiver falls back to the legacy `conflict` for that peer. The +// folder walk is unchanged from v2; this only widens the /plan verdict +// set for initiators that opt in. +const ProtocolVersionContested = 3 + // BeginRequest opens a peer-sync session. type BeginRequest struct { // Volume is the volume name (matched against the receiver's @@ -263,6 +287,15 @@ type PlanResponse struct { // has already pre-staged the loser under .squirrel-conflicts/ and // the initiator's bytes are still in scope for the rclone transfer. Conflicts []ConflictDetail `json:"conflicts,omitempty"` + // Contested captures the paths whose disposition was "contested": + // frozen by a prior conflict, so this initiator's divergent bytes + // were refused rather than superseding (no new copy minted). Each + // entry carries the frozen winner + preserved-loser digests and the + // receiver-relative location the loser is preserved at, so the + // initiator can raise its own local contested marker and its CLI can + // point the operator at both versions. Only populated for an + // initiator that negotiated ProtocolVersionContested. + Contested []ContestedDetail `json:"contested,omitempty"` } // PlanDisposition is the receiver's verdict on one path. @@ -298,6 +331,21 @@ type ConflictDetail struct { PreservedAtPath string `json:"preserved_at_path,omitempty"` } +// ContestedDetail names a path whose disposition was "contested" and +// surfaces the two frozen versions plus where the loser is preserved, so +// the initiator's CLI and its local contested marker describe the same +// state the receiver already froze. LiveBlake3Hex is the frozen winner +// (live at the path on the receiver); PreservedBlake3Hex is the loser +// held under .squirrel-conflicts/; PreservedAtPath is that loser's +// receiver-relative path. The initiator's own refused digest is the +// entry it sent, so it isn't echoed here. +type ContestedDetail struct { + Path string `json:"path"` + LiveBlake3Hex string `json:"live_blake3,omitempty"` + PreservedBlake3Hex string `json:"preserved_blake3,omitempty"` + PreservedAtPath string `json:"preserved_at_path,omitempty"` +} + // VerifyRequest tells the receiver "rclone said it finished, please // re-hash the affected paths now". type VerifyRequest struct { diff --git a/tui/dashboard.go b/tui/dashboard.go index 30bac28..0a007cd 100644 --- a/tui/dashboard.go +++ b/tui/dashboard.go @@ -50,6 +50,11 @@ type dashboardData struct { // mismatch latches one until cleared; the dashboard shows them so the // trust surface answers "am I safe?" with a red panel, not silence. alarms []store.DestinationAlarm + // contested are the standing contested-path freezes (#158, F27). A + // peer-sync conflict latches one until an operator resolves it; the + // dashboard shows them as a badge on *this* machine — the losing edge, + // not only the hub — so a silently diverging file surfaces. + contested []store.ContestedPath } type dashboardDataMsg struct { @@ -112,6 +117,9 @@ func (m *dashboardModel) View() string { if alarms := m.renderAlarms(); alarms != "" { sections = append(sections, alarms) } + if contested := m.renderContested(); contested != "" { + sections = append(sections, contested) + } sections = append(sections, m.renderActiveRuns(), m.renderVolumes(), @@ -144,6 +152,41 @@ func (m *dashboardModel) renderAlarms() string { return header + "\n" + renderTable(rows, colours) } +// renderContested shows standing contested-path freezes (#158, F27) as a +// badge just below the alarms, because a divergent edit frozen on this +// machine is part of the "am I safe?" answer the trust surface must not +// bury. Returns "" when nothing is frozen so the section is absent on a +// healthy install. The count in the header is the badge the losing edge +// needs — the hub is no longer the only place the conflict is visible. +func (m *dashboardModel) renderContested() string { + if len(m.data.contested) == 0 { + return "" + } + header := styleErr.Render(fmt.Sprintf("Contested paths (%d)", len(m.data.contested))) + rows := [][]string{{"VOLUME", "PATH", "PRESERVED AT", "SINCE"}} + for _, c := range m.data.contested { + rows = append(rows, []string{ + m.volumeName(sql.NullInt64{Int64: c.VolumeID, Valid: true}), + c.Path, + contestedPreservedCell(c.PreservedAtPath), + whenAgo(sql.NullInt64{Int64: c.RaisedAtNs, Valid: true}, m.data.now), + }) + } + colours := []lipgloss.Color{colourFailure, "", "", ""} + hint := styleMuted.Render("resolve with `squirrel conflicts resolve `") + return header + "\n" + renderTable(rows, colours) + "\n" + hint +} + +// contestedPreservedCell renders the preserved-loser location, or a dash +// when the freeze record carries none (an initiator that learned of the +// freeze without a preserved path). +func contestedPreservedCell(p string) string { + if p == "" { + return "—" + } + return p +} + func (m *dashboardModel) renderAgentBlock() string { header := styleHeader.Render("Agent") var body string @@ -303,6 +346,10 @@ func loadDashboardData(ctx context.Context, s *store.Store) (dashboardData, erro if err != nil { return dashboardData{}, fmt.Errorf("list alarms: %w", err) } + contested, err := s.ListContestedPaths(ctx) + if err != nil { + return dashboardData{}, fmt.Errorf("list contested paths: %w", err) + } var active, recent []store.Run for _, r := range runs { if r.Status == store.RunStatusRunning { @@ -320,5 +367,6 @@ func loadDashboardData(ctx context.Context, s *store.Store) (dashboardData, erro recentRuns: recent, latestByVol: latestByVol, alarms: alarms, + contested: contested, }, nil } From 080f0efeaca7b9d05a65ae7d7918b319d0184854 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:58:18 +0000 Subject: [PATCH 06/14] cli: output polish batch from the friction walk (#161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Small, independently-landable output fixes surfaced by the reference-setup walk. No behavioural change to what gets synced or stored — reporting only. F7 — sync summaries report already_correct so an in-sync no-op is distinguishable from an empty one (transferred=0 alone is ambiguous). For rclone bucket pushes and restores it is rclone's already-matching count; for peer syncs the handler derives it as present-total minus the paths the sync acted on (the Merkle walk never sends identical folders to /plan). F8 — a first index of an empty tree warns ("no files found — new volume, empty directory, or wrong mount?"); a wrong mount otherwise looks identical to a healthy no-op. index.Report.SawFiles() carries the decision. F11a — the index summary names the volume, so back-to-back runs are legible instead of three anonymous count lines. F11b — a kopia-only sync skips the rclone preamble; "rclone.conf updated" no longer prints when kopia never touches rclone. F11c — kopia's ANSI escape sequences are stripped before its stderr tail is folded into an error, so they don't leak into the structured log. F11d — the per-pair summary line is suppressed when a run never reached a terminal state (an error before its runs row was allocated), instead of printing "status= … run=0". Refused rows minted by #157 still print. F18 — peer-sync rows label direction (→ outbound / ← inbound) in `runs` and the TUI. Only the initiator records runs.shallow, so a set flag marks the pushing side; no schema change. F19 — `runs --failed` and `runs --changes` filters; TUI folds consecutive no-op rows behind a marker (f toggles); the `runs` help no longer claims to list only index runs. F24 — after a burst of new files following a long quiet gap, the index summary adds a catch-up note ("404 new files after 21 days"), so the moment squirrel earns the most trust isn't rendered as routine noise. Restore arrow — the restore summary flips to destination → volume, matching the byte-flow direction instead of the sync-shaped volume → destination. Stacked on #157 (PR #174). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/index.go | 69 ++++++++++++++- cmd/squirrel/polish_test.go | 159 +++++++++++++++++++++++++++++++++++ cmd/squirrel/restore.go | 4 +- cmd/squirrel/restore_test.go | 5 ++ cmd/squirrel/runs.go | 145 ++++++++++++++++++++++++++------ cmd/squirrel/sync.go | 108 +++++++++++++++++------- cmd/squirrel/sync_test.go | 21 +++++ index/index.go | 8 ++ store/files.go | 17 ++++ sync/kopia.go | 13 ++- sync/kopia_test.go | 17 ++++ sync/node.go | 24 ++++++ sync/sync.go | 13 +++ tui/runs.go | 119 +++++++++++++++++++++----- tui/runs_polish_test.go | 47 +++++++++++ 15 files changed, 687 insertions(+), 82 deletions(-) create mode 100644 cmd/squirrel/polish_test.go create mode 100644 tui/runs_polish_test.go diff --git a/cmd/squirrel/index.go b/cmd/squirrel/index.go index 1172c60..e17e50f 100644 --- a/cmd/squirrel/index.go +++ b/cmd/squirrel/index.go @@ -2,10 +2,13 @@ package main import ( "fmt" + "time" "github.com/spf13/cobra" + "github.com/mbertschler/squirrel/config" "github.com/mbertschler/squirrel/index" + "github.com/mbertschler/squirrel/store" ) // newIndexCmd returns the `squirrel index ` cobra command. The @@ -55,6 +58,11 @@ func runIndex(cmd *cobra.Command, volumeName string, progress bool, opts index.O } defer s.Close() + // Captured before the walk (so it reflects the prior run, not this + // one): its presence tells a first index from a re-index, and its + // timestamp feeds the catch-up note. + priorNs, hadPrior := priorIndexRun(cmd, s, vol.Name) + // Progress renders to stderr so a redirected stdout still captures only // the final summary line. The line is erased right after Index returns // (both paths) so any error list and the summary print on a clean line. @@ -79,10 +87,67 @@ func runIndex(cmd *cobra.Command, volumeName string, progress bool, opts index.O if opts.DryRun { prefix = "(dry-run) " } - fmt.Fprintf(cmd.OutOrStdout(), "%sadded=%d modified=%d unchanged=%d missing=%d errors=%d\n", - prefix, rep.Added, rep.Modified, rep.Unchanged, rep.Missing, rep.Errors) + // Naming the volume keeps back-to-back index runs legible instead of + // three anonymous count lines (friction F11a). + fmt.Fprintf(cmd.OutOrStdout(), "%s%s: added=%d modified=%d unchanged=%d missing=%d errors=%d\n", + prefix, vol.Name, rep.Added, rep.Modified, rep.Unchanged, rep.Missing, rep.Errors) + printIndexAdvisories(cmd, vol, rep, priorNs, hadPrior) if rep.Errors > 0 { return fmt.Errorf("encountered %d error(s) during walk", rep.Errors) } return nil } + +// printIndexAdvisories surfaces the two affirmative-feedback lines the +// friction walk asked for: an empty-path warning on a first index (F8) +// and a catch-up note after a burst of new files following a quiet gap +// (F24). Both are advisory — neither changes the exit code. +func printIndexAdvisories(cmd *cobra.Command, vol *config.Volume, rep index.Report, priorNs int64, hadPrior bool) { + if !hadPrior && rep.Errors == 0 && !rep.SawFiles() { + fmt.Fprintf(cmd.ErrOrStderr(), + "warning: volume %q at %s: no files found — new volume, empty directory, or wrong mount?\n", + vol.Name, vol.Path) + return + } + if hadPrior && rep.SawFiles() { + if note, ok := catchUpNote(rep.Added, time.Since(time.Unix(0, priorNs))); ok { + fmt.Fprintln(cmd.OutOrStdout(), note) + } + } +} + +// priorIndexRun returns the start time of the most recent successful index +// run of the volume, and whether one exists. Best-effort: any lookup miss +// (volume never indexed, store error) reports "no prior", which is the safe +// default for both advisories. +func priorIndexRun(cmd *cobra.Command, s *store.Store, volName string) (int64, bool) { + v, err := s.GetVolumeByName(cmd.Context(), volName) + if err != nil { + return 0, false + } + run, err := s.LatestSuccessfulIndexRun(cmd.Context(), v.ID) + if err != nil { + return 0, false + } + return run.StartedAtNs, true +} + +// Catch-up heuristic thresholds: a burst of new files after a long quiet +// gap is the moment squirrel earns the most trust (a trip's worth of +// photos landing in one tick), and it should not blend into routine churn. +// The bounds are deliberately conservative so ordinary cadence runs stay +// silent. +const ( + catchUpMinFiles = 25 + catchUpMinQuiet = 7 * 24 * time.Hour +) + +// catchUpNote returns a one-line "N new files after D days" summary when a +// run added enough files after a long enough quiet gap, and whether it +// applies (friction F24). +func catchUpNote(added int, quiet time.Duration) (string, bool) { + if added < catchUpMinFiles || quiet < catchUpMinQuiet { + return "", false + } + return fmt.Sprintf("catch-up: %d new files after %d days of quiet", added, int(quiet.Hours())/24), true +} diff --git a/cmd/squirrel/polish_test.go b/cmd/squirrel/polish_test.go new file mode 100644 index 0000000..627b0ae --- /dev/null +++ b/cmd/squirrel/polish_test.go @@ -0,0 +1,159 @@ +package main + +import ( + "database/sql" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/mbertschler/squirrel/store" +) + +// TestRunNeedsAttention pins the --failed predicate: only the non-clean +// terminal states qualify (F19). +func TestRunNeedsAttention(t *testing.T) { + cases := map[string]bool{ + store.RunStatusFailed: true, + store.RunStatusRefused: true, + store.RunStatusAborted: true, + store.RunStatusPartial: true, + store.RunStatusSuccess: false, + store.RunStatusRunning: false, + } + for status, want := range cases { + if got := runNeedsAttention(status); got != want { + t.Errorf("runNeedsAttention(%q) = %v, want %v", status, got, want) + } + } +} + +// TestRunIsInteresting pins the --changes predicate: clean successes that +// touched nothing fold; anything that moved files, resolved a conflict, or +// needs attention shows (F19). +func TestRunIsInteresting(t *testing.T) { + conflicts := map[int64]int{7: 2} + tests := []struct { + name string + r store.Run + want bool + }{ + {"clean no-op", store.Run{ID: 1, Status: store.RunStatusSuccess, FileCount: 0}, false}, + {"success moved files", store.Run{ID: 2, Status: store.RunStatusSuccess, FileCount: 5}, true}, + {"failed", store.Run{ID: 3, Status: store.RunStatusFailed}, true}, + {"refused", store.Run{ID: 4, Status: store.RunStatusRefused}, true}, + {"success with conflict", store.Run{ID: 7, Status: store.RunStatusSuccess, FileCount: 0}, true}, + } + for _, tc := range tests { + if got := runIsInteresting(tc.r, conflicts); got != tc.want { + t.Errorf("%s: runIsInteresting = %v, want %v", tc.name, got, tc.want) + } + } +} + +// TestFilterRuns exercises both filters end to end, including that the +// descending order is preserved. +func TestFilterRuns(t *testing.T) { + runs := []store.Run{ + {ID: 5, Status: store.RunStatusSuccess, FileCount: 0}, // no-op + {ID: 4, Status: store.RunStatusSuccess, FileCount: 3}, // change + {ID: 3, Status: store.RunStatusFailed}, // failed + {ID: 2, Status: store.RunStatusRefused}, // refused + {ID: 1, Status: store.RunStatusSuccess, FileCount: 0}, // no-op + } + conflicts := map[int64]int{} + + if ids := runIDs(filterRuns(runs, conflicts, true, false)); !sameIDs(ids, []int64{3, 2}) { + t.Fatalf("--failed ids = %v, want [3 2]", ids) + } + if ids := runIDs(filterRuns(runs, conflicts, false, true)); !sameIDs(ids, []int64{4, 3, 2}) { + t.Fatalf("--changes ids = %v, want [4 3 2]", ids) + } +} + +// TestPeerDirectionArrow / TestPeerLabelDirection cover F18: the initiator +// records runs.shallow, the receiver leaves it NULL, and the arrow follows. +func TestPeerDirectionArrow(t *testing.T) { + if got := peerDirectionArrow(store.Run{Shallow: sql.NullBool{Valid: true}}); got != "→" { + t.Errorf("initiator arrow = %q, want →", got) + } + if got := peerDirectionArrow(store.Run{Shallow: sql.NullBool{}}); got != "←" { + t.Errorf("receiver arrow = %q, want ←", got) + } +} + +func TestPeerLabelDirection(t *testing.T) { + nodes := map[int64]string{9: "htpc"} + out := peerLabel(store.Run{PeerNodeID: sql.NullInt64{Int64: 9, Valid: true}, Shallow: sql.NullBool{Valid: true}}, nodes) + if !strings.HasPrefix(out, "→ htpc") { + t.Errorf("outbound peer label = %q, want prefix %q", out, "→ htpc") + } + in := peerLabel(store.Run{PeerNodeID: sql.NullInt64{Int64: 9, Valid: true}}, nodes) + if !strings.HasPrefix(in, "← htpc") { + t.Errorf("inbound peer label = %q, want prefix %q", in, "← htpc") + } +} + +// TestCatchUpNote pins the F24 heuristic and its rendering. +func TestCatchUpNote(t *testing.T) { + if _, ok := catchUpNote(catchUpMinFiles, catchUpMinQuiet); !ok { + t.Fatal("expected catch-up at the thresholds") + } + note, ok := catchUpNote(404, 21*24*time.Hour) + if !ok || !strings.Contains(note, "404 new files") || !strings.Contains(note, "21 days") { + t.Fatalf("catch-up note = %q ok=%v", note, ok) + } + if _, ok := catchUpNote(catchUpMinFiles-1, catchUpMinQuiet); ok { + t.Error("below file threshold should not trigger") + } + if _, ok := catchUpNote(catchUpMinFiles, catchUpMinQuiet-time.Hour); ok { + t.Error("below quiet threshold should not trigger") + } +} + +// TestCLIIndexNamesVolume covers F11a: the index summary names the volume. +func TestCLIIndexNamesVolume(t *testing.T) { + src := t.TempDir() + writeTestFile(t, filepath.Join(src, "a.txt"), "hello") + f := writeConfigFor(t, map[string]string{"docs": src}) + out := runCLI(t, "--config", f.configPath, "index", "docs") + if !strings.Contains(out, "docs: added=1") { + t.Fatalf("index summary should name the volume: %q", out) + } +} + +// TestCLIIndexEmptyVolumeWarns covers F8: a first index of an empty tree +// warns, and a subsequent populated re-index does not. +func TestCLIIndexEmptyVolumeWarns(t *testing.T) { + src := t.TempDir() // empty + f := writeConfigFor(t, map[string]string{"docs": src}) + out := runCLI(t, "--config", f.configPath, "index", "docs") + if !strings.Contains(out, "no files found") { + t.Fatalf("empty first index should warn: %q", out) + } + writeTestFile(t, filepath.Join(src, "a.txt"), "hi") + out2 := runCLI(t, "--config", f.configPath, "index", "docs") + if strings.Contains(out2, "no files found") { + t.Fatalf("populated re-index should not warn: %q", out2) + } +} + +func runIDs(runs []store.Run) []int64 { + out := make([]int64, len(runs)) + for i, r := range runs { + out[i] = r.ID + } + return out +} + +func sameIDs(a, b []int64) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/cmd/squirrel/restore.go b/cmd/squirrel/restore.go index deab82e..2a6076d 100644 --- a/cmd/squirrel/restore.go +++ b/cmd/squirrel/restore.go @@ -100,7 +100,9 @@ func runRestore(cmd *cobra.Command, volumeName, fromName string, opts sync.Resto } rep, runErr := sync.Restore(cmd.Context(), s, rcl, vol, dest, opts) - printSyncReport(out, rep, runErr) + // Restore pulls destination → local, so flip the arrow: the sync-shaped + // "volume → destination" would misreport the byte-flow direction. + printSyncReport(out, rep, runErr, true) if runErr != nil || rep.Status != "success" { return fmt.Errorf("restore did not complete cleanly") } diff --git a/cmd/squirrel/restore_test.go b/cmd/squirrel/restore_test.go index b83b67f..5d6d25a 100644 --- a/cmd/squirrel/restore_test.go +++ b/cmd/squirrel/restore_test.go @@ -43,6 +43,11 @@ func TestCLIRestoreRoundTrip(t *testing.T) { if !strings.Contains(out, "status=success") { t.Fatalf("restore did not succeed:\n%s", out) } + // The restore arrow flips to destination → volume: bytes flow that + // way, unlike a sync's volume → destination (restore-arrow friction). + if !strings.Contains(out, "scratch → pics") { + t.Fatalf("restore arrow should read destination → volume:\n%s", out) + } for _, name := range []string{"a.txt", "b.txt"} { path := filepath.Join(f.volumeDir, name) body, err := os.ReadFile(path) diff --git a/cmd/squirrel/runs.go b/cmd/squirrel/runs.go index 67e96a0..544569c 100644 --- a/cmd/squirrel/runs.go +++ b/cmd/squirrel/runs.go @@ -15,29 +15,35 @@ import ( // newRunsCmd returns the `squirrel runs` cobra command. It lists rows from // the runs table, most-recent first, with optional volume filtering and a -// configurable cap. Runs of every status (running, success, failed, partial) -// are surfaced so the user can see in-flight or failed indexing alongside -// successful ones. +// configurable cap. Runs of every kind (index, sync, audit, restore, +// offload) and every status (running, success, failed, partial, refused, +// aborted) are surfaced so in-flight, failed, or refused work shows up +// alongside successful runs. --failed and --changes narrow steady-state +// noise (friction F19). func newRunsCmd() *cobra.Command { var ( - volumeName string - limit int + volumeName string + limit int + onlyFailed bool + onlyChanges bool ) cmd := &cobra.Command{ Use: "runs", - Short: "List index runs (most recent first)", + Short: "List runs of every kind (most recent first)", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - return runRuns(cmd, volumeName, limit) + return runRuns(cmd, volumeName, limit, onlyFailed, onlyChanges) }, } cmd.Flags().StringVar(&volumeName, "volume", "", "filter to runs against this volume name") cmd.Flags().IntVar(&limit, "limit", 20, "maximum number of runs to show (0 for no limit)") + cmd.Flags().BoolVar(&onlyFailed, "failed", false, "show only runs that need attention (failed, refused, aborted, or partial)") + cmd.Flags().BoolVar(&onlyChanges, "changes", false, "hide clean no-op runs; show only runs that moved content or need attention") cmd.AddCommand(newRunsFailCmd()) return cmd } -func runRuns(cmd *cobra.Command, volumeName string, limit int) error { +func runRuns(cmd *cobra.Command, volumeName string, limit int, onlyFailed, onlyChanges bool) error { cfg, err := tryLoadConfig(cmd) if err != nil { return err @@ -48,18 +54,7 @@ func runRuns(cmd *cobra.Command, volumeName string, limit int) error { } defer s.Close() - opts := store.ListRunsOpts{Limit: limit, Descending: true} - if volumeName != "" { - v, err := s.GetVolumeByName(cmd.Context(), volumeName) - if err != nil { - if store.IsNotFound(err) { - return fmt.Errorf("no volume named %q", volumeName) - } - return fmt.Errorf("lookup volume %q: %w", volumeName, err) - } - opts.VolumeID = &v.ID - } - runs, err := s.ListRuns(cmd.Context(), opts) + runs, conflicts, err := gatherRuns(cmd, s, volumeName, limit, onlyFailed, onlyChanges) if err != nil { return err } @@ -71,10 +66,6 @@ func runRuns(cmd *cobra.Command, volumeName string, limit int) error { if err != nil { return err } - conflicts, err := loadConflictCounts(cmd, s, runs) - if err != nil { - return err - } if err := printRuns(cmd.OutOrStdout(), runs, volumes, nodes, conflicts); err != nil { return err } @@ -104,6 +95,93 @@ func printActiveAlarms(w io.Writer, alarms []store.DestinationAlarm) { } } +// gatherRuns resolves the optional volume filter, fetches runs, derives +// conflict counts, and applies the --failed/--changes predicates. When a +// predicate is active the fetch is unbounded (opts.Limit=0) so the SQL +// LIMIT can't cap before the predicate runs and silently drop matches; the +// result is truncated to limit after filtering instead. +func gatherRuns(cmd *cobra.Command, s *store.Store, volumeName string, limit int, onlyFailed, onlyChanges bool) ([]store.Run, map[int64]int, error) { + filtering := onlyFailed || onlyChanges + opts := store.ListRunsOpts{Limit: limit, Descending: true} + if filtering { + opts.Limit = 0 + } + if volumeName != "" { + v, err := s.GetVolumeByName(cmd.Context(), volumeName) + if err != nil { + if store.IsNotFound(err) { + return nil, nil, fmt.Errorf("no volume named %q", volumeName) + } + return nil, nil, fmt.Errorf("lookup volume %q: %w", volumeName, err) + } + opts.VolumeID = &v.ID + } + runs, err := s.ListRuns(cmd.Context(), opts) + if err != nil { + return nil, nil, err + } + conflicts, err := loadConflictCounts(cmd, s, runs) + if err != nil { + return nil, nil, err + } + if filtering { + runs = filterRuns(runs, conflicts, onlyFailed, onlyChanges) + if limit > 0 && len(runs) > limit { + runs = runs[:limit] + } + } + return runs, conflicts, nil +} + +// filterRuns applies the --failed / --changes predicates in one pass, +// preserving the descending order the store returned. +func filterRuns(runs []store.Run, conflicts map[int64]int, onlyFailed, onlyChanges bool) []store.Run { + out := make([]store.Run, 0, len(runs)) + for _, r := range runs { + if onlyFailed && !runNeedsAttention(r.Status) { + continue + } + if onlyChanges && !runIsInteresting(r, conflicts) { + continue + } + out = append(out, r) + } + return out +} + +// runNeedsAttention reports whether a run ended in a state an operator +// should look at: a mid-flight failure, a preflight refusal, a reaped +// (aborted) run, or a partial completion. +func runNeedsAttention(status string) bool { + switch status { + case store.RunStatusFailed, store.RunStatusRefused, store.RunStatusAborted, store.RunStatusPartial: + return true + } + return false +} + +// runIsInteresting reports whether a run is worth showing under --changes: +// anything needing attention, still running, that touched files, or that +// resolved conflicts. A clean success that touched nothing is the routine +// no-op the filter hides. +// +// runs.file_count conflates transferred with already-correct for bucket and +// index runs (it is the total files considered, not the delta), so an +// in-sync bucket sync or an all-unchanged re-index still shows here — the +// filter deliberately errs toward showing rather than risk hiding a run +// that moved content. It reliably folds peer-sync no-ops, whose file_count +// is the receiver-verified (i.e. transferred) count and so is zero when +// nothing moved. +func runIsInteresting(r store.Run, conflicts map[int64]int) bool { + if r.Status != store.RunStatusSuccess { + return true + } + if r.FileCount > 0 { + return true + } + return conflicts[r.ID] > 0 +} + // loadNodeNames builds an id→name map for the peer_node_id values // referenced by the listing. Only peer-sync rows carry a non-NULL // peer_node_id, so this typically resolves to a small set of distinct @@ -200,10 +278,25 @@ func peerLabel(r store.Run, nodes map[int64]string) string { if !ok { name = fmt.Sprintf("id=%d", r.PeerNodeID.Int64) } + label := peerDirectionArrow(r) + " " + name if r.CorrelatedRunID.Valid { - return fmt.Sprintf("%s correlated=%d", name, r.CorrelatedRunID.Int64) + return fmt.Sprintf("%s correlated=%d", label, r.CorrelatedRunID.Int64) + } + return label +} + +// peerDirectionArrow labels which way a peer-sync ran so inbound and +// outbound are distinguishable in the audit trail (friction F18): the +// destination column holds a peer name in both directions. Only the +// initiator records runs.shallow — BeginSyncRunIfClear sets it, while the +// receiver's BeginPeerSyncRun leaves it NULL (see store.Run.Shallow) — so a +// set shallow flag marks the row as the side that pushed. "→" is outbound +// (we pushed to the peer), "←" inbound (the peer pushed to us). +func peerDirectionArrow(r store.Run) string { + if r.Shallow.Valid { + return "→" } - return name + return "←" } // conflictLabel formats the per-run conflict count for the listing. diff --git a/cmd/squirrel/sync.go b/cmd/squirrel/sync.go index 9abcc6f..7189767 100644 --- a/cmd/squirrel/sync.go +++ b/cmd/squirrel/sync.go @@ -74,11 +74,17 @@ func runSync(cmd *cobra.Command, volumeName, destinationName string, progress bo if opts.Shallow { fmt.Fprintln(out, shallowSyncWarning) } - if err := sync.EnsureMinVersion(cmd.Context(), rcl, out, sync.ShallowForPairs(pairs, opts.Shallow)); err != nil { - return err - } - if err := writeRcloneConfigLogged(out, rcl, cfg); err != nil { - return err + // Skip the rclone preamble entirely when every pair targets kopia: + // kopia never invokes rclone, so a version check is pointless and + // "rclone.conf updated" is misleading noise on a kopia-only sync + // (friction F11b). + if pairsNeedRclone(pairs) { + if err := sync.EnsureMinVersion(cmd.Context(), rcl, out, sync.ShallowForPairs(pairs, opts.Shallow)); err != nil { + return err + } + if err := writeRcloneConfigLogged(out, rcl, cfg); err != nil { + return err + } } // One snapshotter shared across every pair: the VACUUM INTO snapshot // is taken once per invocation and fanned out (decision #1). Disabled @@ -104,7 +110,7 @@ func runSync(cmd *cobra.Command, volumeName, destinationName string, progress bo if pp != nil { pp.clear() } - printSyncReport(out, rep, err) + printSyncReport(out, rep, err, false) if err != nil || rep.Status != "success" { anyFailed = true } @@ -146,6 +152,20 @@ func rcloneConfigPathFor(cfg *config.Config) string { return filepath.Join(filepath.Dir(cfg.Path), "rclone.conf") } +// pairsNeedRclone reports whether any pair in the batch drives rclone. +// Peer nodes (Destination nil) transfer bytes with rclone, as do every +// non-kopia bucket destination; only a batch composed entirely of kopia +// destinations skips rclone. Used to suppress the rclone preamble on a +// kopia-only sync (F11b). +func pairsNeedRclone(pairs []sync.Pair) bool { + for _, p := range pairs { + if p.Destination == nil || p.Destination.Type != "kopia" { + return true + } + } + return false +} + // writeRcloneConfigLogged renders the rclone.conf and logs a single line // when the file actually changed. An unexpected rewrite is worth // surfacing: the config is derived deterministically from squirrel's @@ -162,7 +182,10 @@ func writeRcloneConfigLogged(out io.Writer, rcl *sync.Rclone, cfg *config.Config return nil } -func printSyncReport(w io.Writer, rep sync.Report, runErr error) { +// printSyncReport renders one pair's outcome. reverse flips the arrow to +// destination → volume for restores, whose bytes flow the other way from +// a sync (the restore-arrow friction item). +func printSyncReport(w io.Writer, rep sync.Report, runErr error, reverse bool) { r := rep.RcloneResult for _, msg := range rep.Warnings { fmt.Fprintf(w, "warning: %s\n", msg) @@ -170,30 +193,19 @@ func printSyncReport(w io.Writer, rep sync.Report, runErr error) { for _, msg := range rep.NodePendingWarnings { fmt.Fprintf(w, "warning: peer reports %s\n", msg) } - switch rep.Verification.Method { - case sync.VerifyMethodKopia: - // Kopia pushes have no rclone counters; render the snapshot's - // own numbers instead. - fmt.Fprintf(w, "%s → %s status=%s files=%d bytes=%d snapshot=%s verified=%t run=%d\n", - rep.Volume, rep.Destination, rep.Status, - rep.Verification.Files, rep.Verification.Bytes, - rep.Verification.SnapshotID, rep.Verification.Verified(), rep.RunID, - ) - case sync.VerifyMethodPresenceSize: - // Content-addressed pushes count objects, with skipped = hashes - // the destination already recorded, entries = manifest segment - // lines, and fingerprints = provider checksums captured for the - // fresh uploads. - fmt.Fprintf(w, "%s → %s status=%s objects=%d skipped=%d errors=%d bytes=%d entries=%d fingerprints=%d run=%d\n", - rep.Volume, rep.Destination, rep.Status, - r.Transferred, r.Checked, r.DisplayErrors(), r.Bytes, - rep.Verification.Files, rep.Fingerprints, rep.RunID, - ) - default: - fmt.Fprintf(w, "%s → %s status=%s transferred=%d checked=%d errors=%d bytes=%d run=%d\n", - rep.Volume, rep.Destination, rep.Status, - r.Transferred, r.Checked, r.DisplayErrors(), r.Bytes, rep.RunID, - ) + src, dst := rep.Volume, rep.Destination + if reverse { + src, dst = rep.Destination, rep.Volume + } + // A run that never reached a terminal state — a preflight error before + // its runs row was allocated (unindexed volume, "already running") — + // carries an empty status and run=0; the counter line would read + // "status= … run=0", pure noise. Skip it and let the error below carry + // the message. Gates that mint a 'refused' row (#157) do have a status + // and run id, so they still print as their own condition (friction + // F11d). + if rep.Status != "" { + printSyncSummaryLine(w, rep, src, dst) } if rep.NodeReceiverRunID != 0 { fmt.Fprintf(w, " receiver_run=%d matched=%d mismatched=%d missing=%d conflicts=%d\n", @@ -247,3 +259,37 @@ func printSyncReport(w io.Writer, rep sync.Report, runErr error) { fmt.Fprintf(w, " %v\n", runErr) } } + +// printSyncSummaryLine renders the one-line counter summary for a pair, +// keyed off the handler's verification method. src/dst carry the already +// direction-resolved endpoints. The default (rclone bucket / peer / restore) +// line reports already_correct so an in-sync no-op is distinguishable from +// an empty one — transferred=0 alone is ambiguous (friction F7). +func printSyncSummaryLine(w io.Writer, rep sync.Report, src, dst string) { + r := rep.RcloneResult + switch rep.Verification.Method { + case sync.VerifyMethodKopia: + // Kopia pushes have no rclone counters; render the snapshot's + // own numbers instead. + fmt.Fprintf(w, "%s → %s status=%s files=%d bytes=%d snapshot=%s verified=%t run=%d\n", + src, dst, rep.Status, + rep.Verification.Files, rep.Verification.Bytes, + rep.Verification.SnapshotID, rep.Verification.Verified(), rep.RunID, + ) + case sync.VerifyMethodPresenceSize: + // Content-addressed pushes count objects, with skipped = hashes + // the destination already recorded, entries = manifest segment + // lines, and fingerprints = provider checksums captured for the + // fresh uploads. + fmt.Fprintf(w, "%s → %s status=%s objects=%d skipped=%d errors=%d bytes=%d entries=%d fingerprints=%d run=%d\n", + src, dst, rep.Status, + r.Transferred, r.Checked, r.DisplayErrors(), r.Bytes, + rep.Verification.Files, rep.Fingerprints, rep.RunID, + ) + default: + fmt.Fprintf(w, "%s → %s status=%s transferred=%d already_correct=%d errors=%d bytes=%d run=%d\n", + src, dst, rep.Status, + r.Transferred, rep.AlreadyCorrect, r.DisplayErrors(), r.Bytes, rep.RunID, + ) + } +} diff --git a/cmd/squirrel/sync_test.go b/cmd/squirrel/sync_test.go index 5902cbe..dcedf01 100644 --- a/cmd/squirrel/sync_test.go +++ b/cmd/squirrel/sync_test.go @@ -42,6 +42,22 @@ func TestCLISyncHappyPath(t *testing.T) { } } +// TestCLISyncAlreadyCorrect covers F7: a second sync of an unchanged +// volume reports already_correct so an in-sync no-op is distinguishable +// from an empty one (transferred=0 alone is ambiguous). +func TestCLISyncAlreadyCorrect(t *testing.T) { + requireRcloneCLI(t) + f := writeSyncFixture(t) + writeTestFile(t, filepath.Join(f.volumeDir, "a.txt"), "alpha") + writeTestFile(t, filepath.Join(f.volumeDir, "b.txt"), "beta") + runCLI(t, "--config", f.configPath, "index", f.volumeName) + runCLI(t, "--config", f.configPath, "sync", "pics") // first sync transfers + out := runCLI(t, "--config", f.configPath, "sync", "pics") + if !strings.Contains(out, "transferred=0") || !strings.Contains(out, "already_correct=2") { + t.Fatalf("second sync should report transferred=0 already_correct=2: %q", out) + } +} + func TestCLISyncUnknownDestinationFlag(t *testing.T) { f := writeSyncFixture(t) _, err := runCLIExpectErr(t, "--config", f.configPath, "sync", "pics", "--to", "ghost") @@ -105,6 +121,11 @@ sync_to = ["mirror"] if !strings.Contains(out, "snapshot=snap123") || !strings.Contains(out, "verified=true") { t.Fatalf("output missing the kopia snapshot summary:\n%s", out) } + // F11b: a kopia-only sync never touches rclone, so it must not print + // the "rclone.conf updated" line. + if strings.Contains(out, "rclone.conf updated") { + t.Fatalf("kopia-only sync should not log an rclone.conf update:\n%s", out) + } } func TestCLISyncRequiresConfig(t *testing.T) { diff --git a/index/index.go b/index/index.go index b9c43df..d219c5f 100644 --- a/index/index.go +++ b/index/index.go @@ -66,6 +66,14 @@ type Report struct { RunID int64 } +// SawFiles reports whether the walk observed any live file (added, +// modified, or unchanged). Zero across all three means the tree held no +// regular files this run — an empty or wrong-mounted volume path, which +// callers surface as a warning on first index since it is otherwise +// indistinguishable from a healthy no-op (friction F8). This package never +// writes to stderr, so the decision is returned rather than printed. +func (r Report) SawFiles() bool { return r.Added+r.Modified+r.Unchanged > 0 } + // ErrAlreadyRunning is returned from Index when store.BeginIndexRunIfClear // refuses to start because another index- or audit-kind run is already in // flight against the same volume. Callers can errors.As against this type diff --git a/store/files.go b/store/files.go index 107a2c6..9b355e9 100644 --- a/store/files.go +++ b/store/files.go @@ -922,6 +922,23 @@ func (s *Store) ListPresentPathsUnder(ctx context.Context, volumeID int64) (map[ return out, rows.Err() } +// CountPresentFilesInVolume returns the number of live (status='present') +// file rows in the volume. The peer-sync summary uses it to report the +// already-correct count: under the Merkle walk only differing folders +// reach /plan, so already-correct is derived as present-total minus the +// paths the sync acted on rather than counted from the (partial) +// disposition list. +func (s *Store) CountPresentFilesInVolume(ctx context.Context, volumeID int64) (int64, error) { + var n int64 + err := s.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM `+fileFromJoin+` + WHERE f.status = 'present' AND fo.volume_id = ?`, volumeID).Scan(&n) + if err != nil { + return 0, fmt.Errorf("count present files in volume %d: %w", volumeID, err) + } + return n, nil +} + // ListPresentFilesInFolder returns every present file row in one // folder, ordered by name ascending so the caller's downstream // processing is deterministic. Used by the Merkle walk's initiator diff --git a/sync/kopia.go b/sync/kopia.go index 9915ce8..15d4d64 100644 --- a/sync/kopia.go +++ b/sync/kopia.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strconv" "strings" @@ -15,6 +16,16 @@ import ( "github.com/mbertschler/squirrel/store" ) +// ansiEscapeRE matches ANSI CSI escape sequences (colour among them). +// kopia paints its progress and error output for a terminal; folding that +// raw into an error would leak escape sequences into squirrel's structured +// agent log and the runs.error column, so the stderr tail is stripped +// first (friction F11c). +var ansiEscapeRE = regexp.MustCompile("\x1b\\[[0-9;?]*[ -/]*[@-~]") + +// stripANSI removes ANSI escape sequences from s. +func stripANSI(s string) string { return ansiEscapeRE.ReplaceAllString(s, "") } + // Kopia is a configured kopia wrapper. Like rclone, kopia is treated as // an opaque child process: squirrel owns the argv, points every // invocation at a destination-scoped config file under ConfigDir @@ -71,7 +82,7 @@ func (k *Kopia) run(ctx context.Context, cfgFile, password string, args ...strin cmd.Stderr = &stderr if err := cmd.Run(); err != nil { verb := strings.Join(args[:min(2, len(args))], " ") - if msg := strings.TrimSpace(stderr.String()); msg != "" { + if msg := stripANSI(strings.TrimSpace(stderr.String())); msg != "" { return stdout.Bytes(), fmt.Errorf("kopia %s: %w: %s", verb, err, msg) } return stdout.Bytes(), fmt.Errorf("kopia %s: %w", verb, err) diff --git a/sync/kopia_test.go b/sync/kopia_test.go index bae2e13..46b9070 100644 --- a/sync/kopia_test.go +++ b/sync/kopia_test.go @@ -611,3 +611,20 @@ func TestKopiaAdvanceScopedToCapturedPresentSet(t *testing.T) { t.Fatalf("verify method = %q, want %q", vector[0].VerifyMethod, store.VerifyMethodKopia) } } + +// TestStripANSI covers F11c: kopia paints its output for a terminal, and +// those escape sequences must not leak into the error folded into +// squirrel's structured log / runs.error column. +func TestStripANSI(t *testing.T) { + cases := map[string]string{ + "\x1b[31merror:\x1b[0m disk full": "error: disk full", + "plain message": "plain message", + "\x1b[1;33mwarn\x1b[0m \x1b[2Kcleared": "warn cleared", + "": "", + } + for in, want := range cases { + if got := stripANSI(in); got != want { + t.Errorf("stripANSI(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/sync/node.go b/sync/node.go index ba92d79..3dc0b08 100644 --- a/sync/node.go +++ b/sync/node.go @@ -217,6 +217,7 @@ func (d *nodeSyncDriver) run() error { // the original path is empty, so rclone treats the entry like a // fresh transfer. d.report.NodeConflicts = plan.Conflicts + d.recordAlreadyCorrect(plan) if !d.opts.DryRun { advance, err := captureDurabilityAdvance(d.ctx, d.store, d.volID) if err != nil { @@ -236,6 +237,29 @@ func (d *nodeSyncDriver) run() error { return nil } +// recordAlreadyCorrect derives the count of paths the receiver already +// held correctly for the summary (F7). Under the Merkle walk only +// differing folders reach /plan, so the identical-folder files never +// appear as dispositions; already-correct is therefore present-total +// minus the paths the sync acted on (every non-already-correct +// disposition). Best-effort: a count error leaves the field zero rather +// than failing the sync over a cosmetic number. +func (d *nodeSyncDriver) recordAlreadyCorrect(plan syncproto.PlanResponse) { + actionable := 0 + for _, disp := range plan.Dispositions { + if disp.Disposition != syncproto.DispositionAlreadyCorrect { + actionable++ + } + } + present, err := d.store.CountPresentFilesInVolume(d.ctx, d.volID) + if err != nil { + return + } + if ac := present - int64(actionable); ac > 0 { + d.report.AlreadyCorrect = ac + } +} + // phaseBegin opens a session with the receiver. The initiator's own // runs row is inserted first so the (peer_node_id, correlated_run_id) // pair lands the right way around: the receiver's id becomes our diff --git a/sync/sync.go b/sync/sync.go index 4fcf57f..a625f48 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -120,6 +120,15 @@ type Report struct { RunID int64 RcloneResult RunResult Status string // success / partial / failed + // AlreadyCorrect counts the paths the destination already held + // correctly, so a no-op-because-in-sync is distinguishable from a + // no-op-because-empty in the summary (transferred=0 alone is + // ambiguous — friction F7). For rclone bucket pushes and restores it + // is rclone's already-matching count; for peer syncs the handler + // derives it as present-total minus the paths the sync acted on + // (the Merkle walk never sends identical folders to /plan, so it + // cannot be counted from the disposition list alone). + AlreadyCorrect int64 // Verification is the handler's typed durability report for this // push: which comparison backed it, what the tool counted, and — // via Verified() — whether the destination's copy was @@ -490,6 +499,10 @@ func beginRestoreRun(ctx context.Context, s *store.Store, dryRun bool, volID int // to the rclone outcome on this very run. func finishRun(ctx context.Context, s *store.Store, dryRun bool, runID int64, rep *Report) { rep.Status = deriveStatus(rep.RcloneResult) + // rclone's "checks" are the files it found already matching at the + // destination and did not transfer — exactly the already-correct + // count the summary reports (F7). Peer syncs set this themselves. + rep.AlreadyCorrect = rep.RcloneResult.Checked if dryRun || runID == 0 { return } diff --git a/tui/runs.go b/tui/runs.go index 43eabdd..73548b1 100644 --- a/tui/runs.go +++ b/tui/runs.go @@ -28,9 +28,14 @@ type runsModel struct { rows []store.Run volumesByID map[int64]store.Volume filterKind string // empty = no filter + fold bool // collapse consecutive no-op rows (F19) + foldedCount int // no-op rows hidden by the current fold loaded bool loadErr error selectedDetail *store.Run + // displayRuns aligns 1:1 with the table's rows: the run behind each + // row, or nil for a fold-marker row (which has no detail view). + displayRuns []*store.Run } type runsDataMsg struct { @@ -55,7 +60,7 @@ func newRunsModel(s *store.Store) *runsModel { Background(colourAccent). Bold(false) t.SetStyles(style) - return &runsModel{store: s, table: t} + return &runsModel{store: s, table: t, fold: true} } func (m *runsModel) Init() tea.Cmd { return m.fetch() } @@ -102,13 +107,11 @@ func (m *runsModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } switch msg.String() { case "enter": - if len(m.rows) == 0 { - return m, nil - } + // A fold-marker row maps to nil in displayRuns and has no + // detail view; ignore enter on it. idx := m.table.Cursor() - filtered := m.filteredRows() - if idx >= 0 && idx < len(filtered) { - r := filtered[idx] + if idx >= 0 && idx < len(m.displayRuns) && m.displayRuns[idx] != nil { + r := *m.displayRuns[idx] m.selectedDetail = &r } return m, nil @@ -127,6 +130,10 @@ func (m *runsModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case "A": m.setFilter("") return m, nil + case "f": + m.fold = !m.fold + m.applyFilter() + return m, nil } } var cmd tea.Cmd @@ -144,9 +151,18 @@ func (m *runsModel) View() string { if m.selectedDetail != nil { return m.renderDetail(*m.selectedDetail) } - header := styleHeader.Render("Runs") + " " + m.renderFilterChips() + header := styleHeader.Render("Runs") + " " + m.renderFilterChips() + m.renderFoldChip() return header + "\n" + m.table.View() + "\n" + - styleMuted.Render("i index · s sync · a audit · r restore · A all · enter detail") + styleMuted.Render("i index · s sync · a audit · r restore · A all · f fold · enter detail") +} + +// renderFoldChip notes how many no-op rows the fold is hiding, so a folded +// view never looks like runs simply went missing. +func (m *runsModel) renderFoldChip() string { + if !m.fold || m.foldedCount == 0 { + return "" + } + return " " + styleMuted.Render(fmt.Sprintf("(%d no-op rows folded — f to expand)", m.foldedCount)) } // renderFilterChips renders the active-filter pill, or muted text when no @@ -238,23 +254,84 @@ func (m *runsModel) applyFilter() { filtered := m.filteredRows() now := time.Now() rows := make([]table.Row, 0, len(filtered)) - for _, r := range filtered { - rows = append(rows, table.Row{ - fmt.Sprintf("#%d", r.ID), - r.Kind, - m.volumeName(r.VolumeID), - nullStr(r.Destination), - r.Status, - whenAgo(r.EndedAtNs, now), - runDuration(r), - fmt.Sprintf("%d", r.FileCount), - shallowGlyph(r.Shallow), - }) + display := make([]*store.Run, 0, len(filtered)) + m.foldedCount = 0 + for i := 0; i < len(filtered); { + // Collapse a maximal run of consecutive no-op rows into one marker, + // but only when there is more than one — a lone no-op isn't worth + // a fold line (F19). + if m.fold && runIsNoOp(filtered[i]) { + j := i + for j < len(filtered) && runIsNoOp(filtered[j]) { + j++ + } + if n := j - i; n > 1 { + rows = append(rows, foldMarkerRow(n)) + display = append(display, nil) + m.foldedCount += n + i = j + continue + } + } + r := filtered[i] + rows = append(rows, m.runTableRow(r, now)) + display = append(display, &r) + i++ } + m.displayRuns = display m.table.SetRows(rows) m.resizeColumns() } +// runTableRow renders one run as a table row. +func (m *runsModel) runTableRow(r store.Run, now time.Time) table.Row { + return table.Row{ + fmt.Sprintf("#%d", r.ID), + r.Kind, + m.volumeName(r.VolumeID), + destinationCell(r), + r.Status, + whenAgo(r.EndedAtNs, now), + runDuration(r), + fmt.Sprintf("%d", r.FileCount), + shallowGlyph(r.Shallow), + } +} + +// foldMarkerRow is the placeholder shown in place of a collapsed block of +// no-op rows. It carries the same column count as a real row so the table +// layout stays aligned. +func foldMarkerRow(n int) table.Row { + return table.Row{"⋯", "", fmt.Sprintf("%d no-op runs folded", n), "", "", "", "", "", ""} +} + +// runIsNoOp reports whether a run is routine no-op noise — a clean success +// that touched no files. Mirrors the CLI's `runs --changes` fold rule; a +// peer-sync no-op has file_count 0, so those collapse, while bucket/index +// no-ops (file_count counts files considered, not moved) stay visible. +func runIsNoOp(r store.Run) bool { + return r.Status == store.RunStatusSuccess && r.FileCount == 0 +} + +// destinationCell renders the DESTINATION column, prefixing a peer-sync row +// with its initiation-direction arrow so inbound and outbound are +// distinguishable (F18): "→" outbound (we pushed), "←" inbound (peer +// pushed). Only the initiator records runs.shallow, so a set flag marks the +// pushing side (see store.Run.Shallow). +func destinationCell(r store.Run) string { + if !r.Destination.Valid { + return nullStr(r.Destination) + } + if !r.PeerNodeID.Valid { + return r.Destination.String + } + arrow := "→" + if !r.Shallow.Valid { + arrow = "←" + } + return arrow + " " + r.Destination.String +} + func (m *runsModel) resizeColumns() { cols := []table.Column{ {Title: "ID", Width: 6}, diff --git a/tui/runs_polish_test.go b/tui/runs_polish_test.go new file mode 100644 index 0000000..09792d2 --- /dev/null +++ b/tui/runs_polish_test.go @@ -0,0 +1,47 @@ +package tui + +import ( + "database/sql" + "testing" + + "github.com/mbertschler/squirrel/store" +) + +// TestDestinationCell covers the F18 direction arrow in the runs table: +// bucket rows render the plain name, peer rows prefix the initiation arrow +// (→ outbound / ← inbound, keyed off runs.shallow). +func TestDestinationCell(t *testing.T) { + bucket := store.Run{Destination: sql.NullString{String: "s3archive", Valid: true}} + if got := destinationCell(bucket); got != "s3archive" { + t.Errorf("bucket cell = %q, want %q", got, "s3archive") + } + outbound := store.Run{ + Destination: sql.NullString{String: "htpc", Valid: true}, + PeerNodeID: sql.NullInt64{Int64: 3, Valid: true}, + Shallow: sql.NullBool{Valid: true}, + } + if got := destinationCell(outbound); got != "→ htpc" { + t.Errorf("outbound cell = %q, want %q", got, "→ htpc") + } + inbound := store.Run{ + Destination: sql.NullString{String: "laptop", Valid: true}, + PeerNodeID: sql.NullInt64{Int64: 4, Valid: true}, + } + if got := destinationCell(inbound); got != "← laptop" { + t.Errorf("inbound cell = %q, want %q", got, "← laptop") + } +} + +// TestRunIsNoOp covers the F19 fold rule: a clean success that touched no +// files is a no-op; anything else is not. +func TestRunIsNoOp(t *testing.T) { + if !runIsNoOp(store.Run{Status: store.RunStatusSuccess, FileCount: 0}) { + t.Error("clean 0-file success should be a no-op") + } + if runIsNoOp(store.Run{Status: store.RunStatusSuccess, FileCount: 3}) { + t.Error("success that touched files is not a no-op") + } + if runIsNoOp(store.Run{Status: store.RunStatusFailed}) { + t.Error("failed run is not a no-op") + } +} From 97481ee750329a97a3a08b4f15ffadacae1166c3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:58:38 +0000 Subject: [PATCH 07/14] agent: per-destination sync workers + rclone transfer timeouts (#160) The scheduler was a single serial worker with no per-run time bound: a hung rclone froze every destination, every volume, the peer syncs, and the index cadences until an operator killed the process by hand (F25). Split sync execution off the tick goroutine into a per-destination dispatcher. The concurrency unit is the destination: at most one sync per destination is in flight (two volumes targeting the same endpoint serialise on it), different destinations run concurrently, and an overall semaphore (default 4) bounds total parallelism. A slow or wedged destination is now confined to its own worker and can never delay the tick loop, index runs, peer syncs, or LAN pairs. The per-pair `skipped reason="in-flight sync run"` discipline and the pre-sync index ordering are preserved. Bound every automatic transfer two ways. rclone carries explicit --contimeout/--timeout on each invocation. A squirrel-side no-progress guard (stallGuard) watches rclone's per-second stats and, if transferred+checked+bytes stops advancing for StallTimeout (default 10m), kills the child and fails the run with a diagnosable "stalled" error that composes with #157's stderr capture. Checks count as progress so a long verification pass is never mistaken for a stall; a slow-but-moving transfer of a large volume is never killed. The guard is off for the foreground CLI (a human can interrupt) and set by the agent. In-flight visibility is already per-pair via the runs table (the TUI's active-runs block); the timeouts make those rows truthful by failing a stuck run instead of leaving it "running" forever. No schema change. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- agent/dispatcher.go | 148 ++++++++++++++++++++++++++++ agent/dispatcher_test.go | 128 ++++++++++++++++++++++++ agent/scheduler.go | 61 +++++------- agent/scheduler_test.go | 19 +++- cmd/squirrel/agent.go | 4 + sync/rclone.go | 207 +++++++++++++++++++++++++++++++++++---- sync/rclone_test.go | 59 +++++++++-- 7 files changed, 560 insertions(+), 66 deletions(-) create mode 100644 agent/dispatcher.go create mode 100644 agent/dispatcher_test.go diff --git a/agent/dispatcher.go b/agent/dispatcher.go new file mode 100644 index 0000000..70c1053 --- /dev/null +++ b/agent/dispatcher.go @@ -0,0 +1,148 @@ +package agent + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" +) + +// defaultMaxParallelSyncs bounds how many automatic syncs run at once +// across all destinations. Per-destination serialisation already caps +// parallelism at the number of configured destinations; this is the +// overall ceiling so a host with many destinations can't spawn one rclone +// per destination simultaneously and thrash its uplink and CPU. Four +// covers the reference household — a NAS pushing photos/docs/media to +// cloudbox + s3archive + kopia-mirror + htpc — without a LAN pair ever +// queueing behind a slow offsite, while still bounding a pathological +// config. +const defaultMaxParallelSyncs = 4 + +// syncDispatcher runs (volume, destination) syncs off the scheduler's tick +// goroutine so a slow or wedged destination can never delay the tick loop, +// index runs, peer syncs, or syncs to other destinations (#160, F25). The +// concurrency unit is the destination: at most one sync per destination is +// in flight at a time (inflight), and an overall semaphore (sem) bounds how +// many run concurrently across destinations. +// +// The pre-sync index ordering the scheduler guarantees is unaffected: the +// tick body still runs a volume's index to completion before it hands that +// volume's due syncs to the dispatcher. +type syncDispatcher struct { + run SyncRunner + logger *slog.Logger + now func() time.Time + + sem chan struct{} + + mu sync.Mutex + inflight map[string]bool + wg sync.WaitGroup +} + +func newSyncDispatcher(run SyncRunner, logger *slog.Logger, now func() time.Time, maxParallel int) *syncDispatcher { + if maxParallel <= 0 { + maxParallel = defaultMaxParallelSyncs + } + return &syncDispatcher{ + run: run, + logger: logger, + now: now, + sem: make(chan struct{}, maxParallel), + inflight: make(map[string]bool), + } +} + +// dispatch launches the sync for (vol, destName) on destName's worker, +// unless a sync to that destination is already in flight (a per-pair skip +// log, matching the CLI wording) or no runner is configured. It never +// blocks on the transfer: the rclone call runs in a goroutine bounded by +// the semaphore, so the caller — the tick loop — returns at once. +func (d *syncDispatcher) dispatch(ctx context.Context, vol *config.Volume, destName string) { + if d.run == nil { + d.logger.Info("scheduler.skipped", + "kind", "sync", "volume", vol.Name, "destination", destName, + "reason", "sync runner not configured") + return + } + if !d.claim(destName) { + d.logger.Info("scheduler.skipped", + "kind", "sync", "volume", vol.Name, "destination", destName, + "reason", "in-flight sync run") + return + } + d.logger.Info("scheduler.kicked", + "kind", "sync", "volume", vol.Name, "destination", destName) + d.wg.Add(1) + go d.runOne(ctx, vol, destName) +} + +// runOne is the per-destination worker body: take an overall slot (or bail +// if the scheduler is shutting down), run the sync, and always release both +// the slot and the destination claim. +func (d *syncDispatcher) runOne(ctx context.Context, vol *config.Volume, destName string) { + defer d.wg.Done() + defer d.release(destName) + select { + case d.sem <- struct{}{}: + defer func() { <-d.sem }() + case <-ctx.Done(): + return + } + if ctx.Err() != nil { + return + } + d.execute(ctx, vol, destName) +} + +// execute invokes the sync runner and emits the finished/error logs, +// mirroring the scheduler's kicked/finished/error discipline. A sync that +// stalls is failed by the runner's transfer timeout (rclone is killed); +// its diagnosable error lands here and in the run row. +func (d *syncDispatcher) execute(ctx context.Context, vol *config.Volume, destName string) { + start := d.now() + rep := d.run(ctx, vol, destName) + duration := d.now().Sub(start) + status := rep.Status + if status == "" { + status = store.RunStatusFailed + } + d.logger.Info("scheduler.finished", + "kind", "sync", "volume", vol.Name, "destination", destName, + "run_id", rep.RunID, "status", status, + "duration_ms", duration.Milliseconds(), + ) + if rep.Err != nil { + d.logger.Error("scheduler.error", + "kind", "sync", "volume", vol.Name, "destination", destName, + "run_id", rep.RunID, "err", rep.Err.Error()) + } +} + +// claim marks destName in flight, returning false when a sync to it is +// already running. release clears the mark. +func (d *syncDispatcher) claim(destName string) bool { + d.mu.Lock() + defer d.mu.Unlock() + if d.inflight[destName] { + return false + } + d.inflight[destName] = true + return true +} + +func (d *syncDispatcher) release(destName string) { + d.mu.Lock() + defer d.mu.Unlock() + delete(d.inflight, destName) +} + +// wait blocks until every dispatched sync has finished. The scheduler +// calls it during shutdown, after the run context is cancelled (which +// kills any in-flight rclone child), so it returns promptly. +func (d *syncDispatcher) wait() { + d.wg.Wait() +} diff --git a/agent/dispatcher_test.go b/agent/dispatcher_test.go new file mode 100644 index 0000000..c520775 --- /dev/null +++ b/agent/dispatcher_test.go @@ -0,0 +1,128 @@ +package agent + +import ( + "bytes" + "context" + "log/slog" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" +) + +func discardLogger() *slog.Logger { return slog.New(slog.DiscardHandler) } + +// TestSyncDispatcherRunsDestinationsConcurrently proves the core F25 fix: +// two destinations run at the same time, so a slow offsite can never delay +// a LAN pair. Both runner invocations must be in flight before either is +// released. +func TestSyncDispatcherRunsDestinationsConcurrently(t *testing.T) { + var active, maxActive atomic.Int32 + entered := make(chan struct{}, 2) + release := make(chan struct{}) + run := func(ctx context.Context, vol *config.Volume, destName string) SyncRunReport { + n := active.Add(1) + for { + old := maxActive.Load() + if n <= old || maxActive.CompareAndSwap(old, n) { + break + } + } + entered <- struct{}{} + <-release + active.Add(-1) + return SyncRunReport{Status: store.RunStatusSuccess} + } + d := newSyncDispatcher(run, discardLogger(), time.Now, 4) + vol := &config.Volume{Name: "photos"} + d.dispatch(context.Background(), vol, "cloudbox") + d.dispatch(context.Background(), vol, "htpc") + <-entered // both must be running before we release either + <-entered + close(release) + d.wait() + if got := maxActive.Load(); got != 2 { + t.Fatalf("max concurrent syncs = %d; want 2 (different destinations run concurrently)", got) + } +} + +// TestSyncDispatcherOneInFlightPerDestination proves the concurrency unit +// is the destination: a second volume targeting a destination that already +// has a sync in flight is skipped with the per-pair "in-flight sync run" +// log, and its runner never fires while the first is running. +func TestSyncDispatcherOneInFlightPerDestination(t *testing.T) { + var calls atomic.Int32 + entered := make(chan struct{}, 1) + release := make(chan struct{}) + run := func(ctx context.Context, vol *config.Volume, destName string) SyncRunReport { + calls.Add(1) + entered <- struct{}{} + <-release + return SyncRunReport{Status: store.RunStatusSuccess} + } + buf := &bytes.Buffer{} + logger := slog.New(slog.NewTextHandler(buf, nil)) + d := newSyncDispatcher(run, logger, time.Now, 4) + + d.dispatch(context.Background(), &config.Volume{Name: "photos"}, "cloudbox") + <-entered // the first sync has claimed cloudbox and is running + d.dispatch(context.Background(), &config.Volume{Name: "docs"}, "cloudbox") + + if got := calls.Load(); got != 1 { + t.Fatalf("runner called %d times; want 1 (second dispatch to a busy destination must skip)", got) + } + log := buf.String() + if !strings.Contains(log, `reason="in-flight sync run"`) || !strings.Contains(log, "volume=docs") { + t.Fatalf("expected a per-pair in-flight skip for docs→cloudbox, got:\n%s", log) + } + close(release) + d.wait() +} + +// TestSyncDispatcherBoundsOverallParallelism proves the semaphore ceiling: +// with a single slot, a second destination waits for the first to free the +// slot rather than running immediately. +func TestSyncDispatcherBoundsOverallParallelism(t *testing.T) { + var bCalled atomic.Bool + aEntered := make(chan struct{}) + release := make(chan struct{}) + run := func(ctx context.Context, vol *config.Volume, destName string) SyncRunReport { + if destName == "a" { + close(aEntered) + <-release + } else { + bCalled.Store(true) + } + return SyncRunReport{Status: store.RunStatusSuccess} + } + d := newSyncDispatcher(run, discardLogger(), time.Now, 1) + vol := &config.Volume{Name: "v"} + d.dispatch(context.Background(), vol, "a") + <-aEntered // a holds the only slot + d.dispatch(context.Background(), vol, "b") + time.Sleep(20 * time.Millisecond) // give b's worker time to reach the slot wait + if bCalled.Load() { + t.Fatal("b ran while the single parallelism slot was held by a") + } + close(release) + d.wait() + if !bCalled.Load() { + t.Fatal("b never ran after the slot was freed") + } +} + +// TestSyncDispatcherNilRunnerSkips covers the index-only host: no runner +// wired, so a dispatch logs a skip and starts no worker. +func TestSyncDispatcherNilRunnerSkips(t *testing.T) { + buf := &bytes.Buffer{} + logger := slog.New(slog.NewTextHandler(buf, nil)) + d := newSyncDispatcher(nil, logger, time.Now, 4) + d.dispatch(context.Background(), &config.Volume{Name: "v"}, "dest") + d.wait() + if !strings.Contains(buf.String(), "sync runner not configured") { + t.Fatalf("expected a nil-runner skip log, got:\n%s", buf.String()) + } +} diff --git a/agent/scheduler.go b/agent/scheduler.go index 17d3883..0394688 100644 --- a/agent/scheduler.go +++ b/agent/scheduler.go @@ -30,7 +30,7 @@ const DefaultSchedulerTick = 30 * time.Second type scheduler struct { store *store.Store volumes map[string]*config.Volume - syncRun SyncRunner + dispatch *syncDispatcher logger *slog.Logger locks lockHolder tickEvery time.Duration @@ -60,7 +60,7 @@ func newScheduler(srv *Server, tickEvery time.Duration, now func() time.Time) *s return &scheduler{ store: srv.store, volumes: srv.cfg.Volumes, - syncRun: srv.cfg.SyncRunner, + dispatch: newSyncDispatcher(srv.cfg.SyncRunner, srv.cfg.Logger, now, defaultMaxParallelSyncs), logger: srv.cfg.Logger, locks: srv.router, tickEvery: tickEvery, @@ -103,9 +103,12 @@ func (s *scheduler) run(ctx context.Context) { for { select { case <-ctx.Done(): - // Drain in-flight hooks before returning so Serve's shutdown - // wait doesn't race a hook goroutine writing its outcome. + // Drain in-flight hooks and per-destination sync workers before + // returning so Serve's shutdown wait doesn't race a goroutine + // writing its outcome. ctx is already cancelled, so any rclone + // child has been killed and the workers return promptly. s.hooks.wait() + s.dispatch.wait() return case <-t.C: s.tick(ctx) @@ -460,17 +463,21 @@ func indexRunStatus(rep index.Report, err error) (string, bool) { return store.RunStatusSuccess, true } -// runSync kicks one (volume, destination) sync via the configured -// SyncRunner. The pre-check against the runs table is the in-flight -// detection the issue specifies; the SyncRunner's downstream call -// (sync.RunPair) adds its own atomic BeginSyncRunIfClear gate, so a -// race that sneaks past the pre-check still produces a clean skip -// rather than a duplicate run. +// runSync evaluates the per-pair in-flight gate, then hands the sync to +// the per-destination dispatcher (#160). The DB check catches a sync of +// this exact pair already running — a concurrent CLI `squirrel sync`, or a +// stale row not yet reaped — producing the same clean per-pair skip the +// serial scheduler did; the dispatcher additionally serialises different +// volumes that target the same destination and confines a slow transfer to +// its own worker. The SyncRunner's downstream call (sync.RunPair) adds its +// own atomic BeginSyncRunIfClear gate, so a race that sneaks past the +// pre-check still produces a clean skip rather than a duplicate run. // -// A nil SyncRunner surfaces as a clean skip rather than an error so -// an agent running pure index-only schedules can omit the sync wiring -// (and the rclone dependency that comes with it) without the -// scheduler logging at error level on each tick. +// Dispatch is non-blocking: a slow or wedged destination is confined to its +// own worker and never stalls the tick loop or syncs to other destinations, +// which is the freeze F25 recorded. A nil SyncRunner surfaces as a clean +// skip inside the dispatcher so an agent running pure index-only schedules +// can omit the sync wiring without logging at error level each tick. func (s *scheduler) runSync(ctx context.Context, vol *config.Volume, volumeID int64, destName string) { running, err := s.store.HasRunningRun(ctx, store.RunKindSync, volumeID, destName) if err != nil { @@ -485,29 +492,5 @@ func (s *scheduler) runSync(ctx context.Context, vol *config.Volume, volumeID in "reason", "in-flight sync run") return } - if s.syncRun == nil { - s.logger.Info("scheduler.skipped", - "kind", "sync", "volume", vol.Name, "destination", destName, - "reason", "sync runner not configured") - return - } - s.logger.Info("scheduler.kicked", - "kind", "sync", "volume", vol.Name, "destination", destName) - start := s.now() - rep := s.syncRun(ctx, vol, destName) - duration := s.now().Sub(start) - status := rep.Status - if status == "" { - status = store.RunStatusFailed - } - s.logger.Info("scheduler.finished", - "kind", "sync", "volume", vol.Name, "destination", destName, - "run_id", rep.RunID, "status", status, - "duration_ms", duration.Milliseconds(), - ) - if rep.Err != nil { - s.logger.Error("scheduler.error", - "kind", "sync", "volume", vol.Name, "destination", destName, - "run_id", rep.RunID, "err", rep.Err.Error()) - } + s.dispatch.dispatch(ctx, vol, destName) } diff --git a/agent/scheduler_test.go b/agent/scheduler_test.go index 9a4a8d6..2d03553 100644 --- a/agent/scheduler_test.go +++ b/agent/scheduler_test.go @@ -174,7 +174,7 @@ func (f *schedulerFixture) scheduler() *scheduler { return &scheduler{ store: f.srv.store, volumes: f.srv.cfg.Volumes, - syncRun: f.srv.cfg.SyncRunner, + dispatch: newSyncDispatcher(f.srv.cfg.SyncRunner, f.srv.cfg.Logger, f.clock.Now, defaultMaxParallelSyncs), logger: f.srv.cfg.Logger, locks: f.srv.router, tickEvery: time.Second, @@ -310,6 +310,7 @@ func TestSchedulerSyncRunsPreSyncIndexFirst(t *testing.T) { ctx := context.Background() sch.tick(ctx) + sch.dispatch.wait() // syncs run on per-destination workers; drain them runs, err := f.store.ListRuns(ctx, store.ListRunsOpts{}) if err != nil { @@ -437,6 +438,7 @@ func TestSchedulerSkipsWhenInFlightSyncRun(t *testing.T) { sch := f.scheduler() sch.tick(context.Background()) + sch.dispatch.wait() if got := len(f.syncLog.Calls()); got != 0 { t.Fatalf("sync invoked %d times despite running sync row; want 0", got) @@ -517,6 +519,7 @@ func TestSchedulerCadenceUsesLastFinished(t *testing.T) { // Tick 1: nothing prior → fires. sch.tick(ctx) + sch.dispatch.wait() if got := len(f.syncLog.Calls()); got != 1 { t.Fatalf("after tick 1: sync calls = %d; want 1", got) } @@ -524,6 +527,7 @@ func TestSchedulerCadenceUsesLastFinished(t *testing.T) { // Advance 30min, still under cadence → does not fire. f.clock.Add(30 * time.Minute) sch.tick(ctx) + sch.dispatch.wait() if got := len(f.syncLog.Calls()); got != 1 { t.Fatalf("after tick 2 (under cadence): sync calls = %d; want 1", got) } @@ -531,6 +535,7 @@ func TestSchedulerCadenceUsesLastFinished(t *testing.T) { // Advance another hour, past cadence → fires again. f.clock.Add(time.Hour) sch.tick(ctx) + sch.dispatch.wait() if got := len(f.syncLog.Calls()); got != 2 { t.Fatalf("after tick 3 (over cadence): sync calls = %d; want 2", got) } @@ -562,6 +567,7 @@ func TestSchedulerFailedSyncStillConsumesCadence(t *testing.T) { ctx := context.Background() sch.tick(ctx) + sch.dispatch.wait() if calls.Load() != 1 { t.Fatalf("tick 1 sync calls = %d; want 1", calls.Load()) } @@ -569,6 +575,7 @@ func TestSchedulerFailedSyncStillConsumesCadence(t *testing.T) { // Within cadence → no re-fire even though the prior run failed. f.clock.Add(10 * time.Minute) sch.tick(ctx) + sch.dispatch.wait() if calls.Load() != 1 { t.Fatalf("under-cadence tick after failure called sync %d times; want 1 (no special retry)", calls.Load()) @@ -577,6 +584,7 @@ func TestSchedulerFailedSyncStillConsumesCadence(t *testing.T) { // Past cadence → re-fires (regardless of prior failure). f.clock.Add(time.Hour) sch.tick(ctx) + sch.dispatch.wait() if calls.Load() != 2 { t.Fatalf("past-cadence tick after failure called sync %d times; want 2", calls.Load()) } @@ -598,6 +606,7 @@ func TestSchedulerBothCadencesShareIndexRun(t *testing.T) { ctx := context.Background() sch.tick(ctx) + sch.dispatch.wait() runs, _ := f.store.ListRuns(ctx, store.ListRunsOpts{}) var indexCount int @@ -627,11 +636,13 @@ func TestSchedulerStandaloneIndexAfterSyncCadenceWaits(t *testing.T) { ctx := context.Background() sch.tick(ctx) // fires sync + pre-sync index. + sch.dispatch.wait() // 10min later → under index_every from the pre-sync index, no // standalone fires. f.clock.Add(10 * time.Minute) sch.tick(ctx) + sch.dispatch.wait() runs, _ := f.store.ListRuns(ctx, store.ListRunsOpts{}) var indexCount int for _, r := range runs { @@ -647,6 +658,7 @@ func TestSchedulerStandaloneIndexAfterSyncCadenceWaits(t *testing.T) { // due yet because sync_every is 1h). f.clock.Add(10 * time.Minute) // total 20min since pre-sync. sch.tick(ctx) + sch.dispatch.wait() runs, _ = f.store.ListRuns(ctx, store.ListRunsOpts{}) indexCount = 0 for _, r := range runs { @@ -792,9 +804,12 @@ func TestSchedulerDropsSyncWhenRunnerNil(t *testing.T) { }) f.seedFile() sch := f.scheduler() - sch.syncRun = nil + // A dispatcher with no runner is the index-only-host case (no rclone + // wired): sync-triggering paths log a skip and create no sync rows. + sch.dispatch = newSyncDispatcher(nil, f.srv.cfg.Logger, f.clock.Now, defaultMaxParallelSyncs) sch.tick(context.Background()) + sch.dispatch.wait() if !f.containsLogLine("scheduler.skipped", "kind=sync", "sync runner not configured") { t.Fatalf("missing nil-runner skip log:\n%s", f.logBuf.String()) diff --git a/cmd/squirrel/agent.go b/cmd/squirrel/agent.go index ad18923..872c5df 100644 --- a/cmd/squirrel/agent.go +++ b/cmd/squirrel/agent.go @@ -169,6 +169,10 @@ func resolveSchedulerRclone(cmd *cobra.Command, cfg *config.Config) (*sync.Rclon if err != nil { return nil, fmt.Errorf("scheduler needs rclone for scheduled syncs: %w", err) } + // Bound every automatic transfer by the no-progress guard so a wedged + // endpoint fails its own run instead of hanging forever (#160, F25). + // Foreground `squirrel sync` leaves this unset — a human can interrupt. + rcl.StallTimeout = sync.DefaultStallTimeout pairs, err := sync.PairsFor(cfg, "", "") if err != nil { return nil, fmt.Errorf("scheduler rclone preflight: %w", err) diff --git a/sync/rclone.go b/sync/rclone.go index 9a4ab86..8995da5 100644 --- a/sync/rclone.go +++ b/sync/rclone.go @@ -21,6 +21,8 @@ import ( "strconv" "strings" "sync" + "sync/atomic" + "time" "github.com/mbertschler/squirrel/config" "github.com/mbertschler/squirrel/runevents" @@ -31,6 +33,26 @@ import ( // --hash blake3 flag used by sync would be rejected by rclone. var MinRcloneVersion = Version{Major: 1, Minor: 66} +// DefaultStallTimeout is the no-progress bound the agent scheduler applies +// to every automatic rclone transfer (see Rclone.StallTimeout). Ten +// minutes is comfortably longer than the longest legitimate gap between +// progress reports — hashing one very large file for a --checksum +// comparison surfaces no transfer/check advance until it completes — while +// bounding a wedged endpoint far below the "indefinite" hang F25 recorded. +// A dark endpoint usually fails sooner via rclone's own --contimeout / +// --timeout; the guard covers the live-but-stuck case those don't. +const DefaultStallTimeout = 10 * time.Minute + +// rcloneConnectTimeout and rcloneIOTimeout are passed as --contimeout and +// --timeout on every streaming transfer so a dead or unresponsive endpoint +// self-terminates instead of hanging. They match rclone's own defaults but +// are set explicitly so the bound is guaranteed applied and lives in one +// tunable place (#160, F25). +const ( + rcloneConnectTimeout = "1m" + rcloneIOTimeout = "5m" +) + // Rclone is a configured rclone wrapper. Binary is the resolved executable // path; Config is the path to the squirrel-managed rclone.conf written by // WriteRcloneConfig. All Run invocations pass --config Config so the user's @@ -38,6 +60,17 @@ var MinRcloneVersion = Version{Major: 1, Minor: 66} type Rclone struct { Binary string Config string + // StallTimeout, when > 0, bounds a streaming transfer (RunWithProgress) + // by progress rather than by total wall-clock: if rclone reports no + // advance in transferred+checked+bytes for this long, the child is + // killed and the run fails with a diagnosable "stalled" error. It is the + // squirrel-side backstop for a connection that stays alive but stops + // making progress — the storage-full S3 hang of F25 that tripped neither + // --contimeout nor --timeout. Zero (the default, used by the foreground + // CLI where a human can interrupt) disables the guard; the agent + // scheduler sets it to DefaultStallTimeout so an unattended cadence can + // never wedge forever. + StallTimeout time.Duration } // Find locates the rclone binary on PATH. The returned Rclone has Config @@ -308,23 +341,18 @@ func (r *Rclone) RunWithProgress(ctx context.Context, onProgress func(runevents. if r.Config == "" { return RunResult{}, errors.New("rclone wrapper: Config not set (call WriteRcloneConfig first)") } - full := append([]string{ - "--config", r.Config, - "--use-json-log", - "--log-level", "INFO", - "--stats", "1s", - }, args...) - cmd := exec.CommandContext(ctx, r.Binary, full...) - stderr, err := cmd.StderrPipe() - if err != nil { - return RunResult{}, fmt.Errorf("rclone stderr pipe: %w", err) + // A per-run cancel backs the no-progress guard: rclone's own timeouts + // bound a dead connection, the guard bounds a live-but-wedged one (F25). + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + var guard *stallGuard + if r.StallTimeout > 0 { + guard = newStallGuard(runCtx, r.StallTimeout, cancel) } - stdout, err := cmd.StdoutPipe() + + cmd, stderr, stdout, err := r.startRclone(runCtx, args) if err != nil { - return RunResult{}, fmt.Errorf("rclone stdout pipe: %w", err) - } - if err := cmd.Start(); err != nil { - return RunResult{}, fmt.Errorf("rclone start: %w", err) + return RunResult{}, err } var ( @@ -334,7 +362,7 @@ func (r *Rclone) RunWithProgress(ctx context.Context, onProgress func(runevents. wg.Add(2) go func() { defer wg.Done() - parseJSONLog(stderr, &result, onProgress) + parseJSONLog(stderr, &result, onProgress, guard.advance) }() go func() { defer wg.Done() @@ -345,13 +373,136 @@ func (r *Rclone) RunWithProgress(ctx context.Context, onProgress func(runevents. wg.Wait() waitErr := cmd.Wait() + cancel() + guard.wait() if waitErr != nil { result.FatalError = result.Errors == 0 - return result, rcloneExitError(waitErr, result.Stderr) + return result, rcloneRunError(waitErr, result.Stderr, guard, r.StallTimeout) } return result, nil } +// startRclone assembles the full argument list — config, JSON logging, +// per-second stats, and the connect/IO timeouts every transfer carries +// (#160, F25) — and starts the child on runCtx, returning its stderr and +// stdout pipes. runCtx backs both the caller's cancellation and the stall +// guard's kill. +func (r *Rclone) startRclone(runCtx context.Context, args []string) (*exec.Cmd, io.Reader, io.Reader, error) { + full := append([]string{ + "--config", r.Config, + "--use-json-log", + "--log-level", "INFO", + "--stats", "1s", + "--contimeout", rcloneConnectTimeout, + "--timeout", rcloneIOTimeout, + }, args...) + cmd := exec.CommandContext(runCtx, r.Binary, full...) + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, nil, nil, fmt.Errorf("rclone stderr pipe: %w", err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, nil, fmt.Errorf("rclone stdout pipe: %w", err) + } + if err := cmd.Start(); err != nil { + return nil, nil, nil, fmt.Errorf("rclone start: %w", err) + } + return cmd, stderr, stdout, nil +} + +// stallGuard kills a running rclone child when a streaming transfer stops +// making progress for longer than timeout. rclone's --contimeout/--timeout +// bound a dead connection; the guard is the squirrel-side backstop for a +// connection that stays alive yet stops advancing — the storage-full hang +// of F25 that tripped neither. It is driven off rclone's periodic --stats +// events (via notifyAdvance): any growth in transferred+checked+bytes +// resets the timer, so a slow-but-moving transfer of a large volume is +// never killed while a wedged one is. +type stallGuard struct { + timeout time.Duration + cancel context.CancelFunc + poke chan struct{} + done chan struct{} + fired atomic.Bool +} + +func newStallGuard(ctx context.Context, timeout time.Duration, cancel context.CancelFunc) *stallGuard { + g := &stallGuard{ + timeout: timeout, + cancel: cancel, + poke: make(chan struct{}, 1), + done: make(chan struct{}), + } + go g.watch(ctx) + return g +} + +// advance signals progress from the stats-reader goroutine. Non-blocking: +// a pending poke already carries the reset, so a dropped signal is +// harmless. Safe on a nil guard so the disabled path needs no branch. +func (g *stallGuard) advance() { + if g == nil { + return + } + select { + case g.poke <- struct{}{}: + default: + } +} + +// watch resets its timer on every progress poke and, when the timer +// expires with no intervening progress, records the stall and cancels the +// run context — which makes exec.CommandContext kill the rclone child. It +// returns when ctx is done (the run finished or was cancelled elsewhere). +func (g *stallGuard) watch(ctx context.Context) { + defer close(g.done) + timer := time.NewTimer(g.timeout) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-g.poke: + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(g.timeout) + case <-timer.C: + g.fired.Store(true) + g.cancel() + return + } + } +} + +// wait blocks until the watch goroutine has returned so the caller can read +// fired without a race. Safe on a nil guard. +func (g *stallGuard) wait() { + if g == nil { + return + } + <-g.done +} + +// rcloneRunError builds the error for a non-zero rclone exit. When the +// stall guard fired, the exit is a kill squirrel caused, so we say so +// plainly (folding in the captured stderr tail, #157) rather than +// surfacing a bare "signal: killed"; otherwise it defers to +// rcloneExitError. +func rcloneRunError(waitErr error, stderrTail string, guard *stallGuard, timeout time.Duration) error { + if guard != nil && guard.fired.Load() { + if tail := strings.TrimSpace(stderrTail); tail != "" { + return fmt.Errorf("rclone stalled: no progress for %s, killed: %w: %s", timeout, waitErr, tail) + } + return fmt.Errorf("rclone stalled: no progress for %s, killed: %w", timeout, waitErr) + } + return rcloneExitError(waitErr, stderrTail) +} + // rcloneExitError wraps a non-zero rclone exit, folding in the captured // stderr tail when present so the scheduler log and the run row carry the // actual reason (auth, host key, path) rather than a bare "exit status 1" @@ -554,9 +705,10 @@ func isErrorLevel(level string) bool { // that never reaches the JSON log still carries its reason (#157, // F6/F15). onProgress, if non-nil, is invoked once per stats event so // callers can drive a live UI. -func parseJSONLog(r io.Reader, result *RunResult, onProgress func(runevents.Progress)) { +func parseJSONLog(r io.Reader, result *RunResult, onProgress func(runevents.Progress), onAdvance func()) { scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + var lastProgressTotal int64 for scanner.Scan() { line := scanner.Bytes() if len(line) == 0 { @@ -579,6 +731,7 @@ func parseJSONLog(r io.Reader, result *RunResult, onProgress func(runevents.Prog } if ev.Stats != nil { applyStatsEvent(result, ev.Stats, onProgress) + notifyAdvance(result, &lastProgressTotal, onAdvance) continue } if isErrorLevel(ev.Level) && !isRetrySummary(ev.Msg) { @@ -596,6 +749,24 @@ func parseJSONLog(r io.Reader, result *RunResult, onProgress func(runevents.Prog } } +// notifyAdvance pokes the stall guard whenever the running total of +// transferred+checked+bytes has grown since the last stats event, so a +// transfer that is slow but genuinely moving keeps resetting the guard's +// timer while a wedged one — rclone still emitting per-second stats, but +// with flat counters — lets it fire. Checks count as progress so a long +// verification pass over an already-populated destination is not mistaken +// for a stall. +func notifyAdvance(result *RunResult, last *int64, onAdvance func()) { + if onAdvance == nil { + return + } + total := result.Transferred + result.Checked + result.Bytes + if total > *last { + *last = total + onAdvance() + } +} + // applyStatsEvent folds one rclone stats event into result and drives the // optional progress callback. Errors is taken as the running maximum so a // per-attempt stats line that reports fewer errors than a prior one (a diff --git a/sync/rclone_test.go b/sync/rclone_test.go index efb52e1..0b252ac 100644 --- a/sync/rclone_test.go +++ b/sync/rclone_test.go @@ -7,6 +7,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -14,6 +15,51 @@ import ( "github.com/mbertschler/squirrel/runevents" ) +// TestStallGuardFiresWithoutProgress proves the no-progress guard cancels +// the run — and records that it fired — when no advance arrives within the +// timeout. This is the F25 wedge: rclone alive but transferring nothing. +func TestStallGuardFiresWithoutProgress(t *testing.T) { + var cancelled atomic.Bool + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + g := newStallGuard(ctx, 30*time.Millisecond, func() { + cancelled.Store(true) + cancel() + }) + g.wait() // blocks until watch returns; with no poke it fires first + if !g.fired.Load() { + t.Fatal("guard did not fire after the stall timeout elapsed with no progress") + } + if !cancelled.Load() { + t.Fatal("guard fired but never cancelled the run context") + } +} + +// TestStallGuardResetsOnProgress proves a transfer that keeps advancing is +// never killed: pokes arriving well inside the timeout keep resetting it, +// even across a span longer than a single timeout window. +func TestStallGuardResetsOnProgress(t *testing.T) { + var cancelled atomic.Bool + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + g := newStallGuard(ctx, 100*time.Millisecond, func() { + cancelled.Store(true) + cancel() + }) + for i := 0; i < 8; i++ { // 160ms total, gaps of 20ms << the 100ms bound + time.Sleep(20 * time.Millisecond) + g.advance() + } + if g.fired.Load() { + t.Fatal("guard fired while progress was still arriving") + } + cancel() // finishing the run stops the guard cleanly + g.wait() + if cancelled.Load() { + t.Fatal("guard cancelled the run despite steady progress") + } +} + // requireRclone skips the test if rclone is not on PATH. The wrapper tests // exercise the real binary against a local-filesystem destination; if a // developer wants to run them they need rclone installed. @@ -58,7 +104,7 @@ func TestParseJSONLogCapturesObjectlessErrors(t *testing.T) { `{"stats":{"errors":1,"fatalError":true,"totalTransfers":0,"totalChecks":0,"bytes":0}}`, }, "\n") var r RunResult - parseJSONLog(strings.NewReader(stream), &r, nil) + parseJSONLog(strings.NewReader(stream), &r, nil, nil) if len(r.FailedFiles) != 2 { t.Fatalf("FailedFiles = %+v, want 2 (auth + reading; retry summaries dropped)", r.FailedFiles) @@ -85,7 +131,7 @@ func TestParseJSONLogCapturesFatalLevelAndRawStderr(t *testing.T) { `{"stats":{"errors":0,"fatalError":true,"totalTransfers":0,"totalChecks":0,"bytes":0}}`, }, "\n") var r RunResult - parseJSONLog(strings.NewReader(stream), &r, nil) + parseJSONLog(strings.NewReader(stream), &r, nil, nil) if len(r.FailedFiles) != 1 || !strings.Contains(r.FailedFiles[0].Message, "NoCredentialProviders") { t.Fatalf("FailedFiles = %+v, want the fatal-level message captured", r.FailedFiles) @@ -105,8 +151,7 @@ func TestParseJSONLogStderrBounded(t *testing.T) { lines = append(lines, fmt.Sprintf("stderr line %04d", i)) } var r RunResult - parseJSONLog(strings.NewReader(strings.Join(lines, "\n")), &r, nil) - + parseJSONLog(strings.NewReader(strings.Join(lines, "\n")), &r, nil, nil) if len(r.Stderr) == 0 { t.Fatal("Stderr empty, want the tail of the stream") } @@ -153,7 +198,7 @@ func TestParseJSONLogDetectsHashFallback(t *testing.T) { `{"stats":{"errors":0,"fatalError":false,"totalTransfers":2,"totalChecks":0,"bytes":10}}`, }, "\n") var r RunResult - parseJSONLog(strings.NewReader(stream), &r, nil) + parseJSONLog(strings.NewReader(stream), &r, nil, nil) if !r.HashFallback { t.Fatalf("HashFallback = false, want true (no-common-hash notice should be detected)") @@ -168,7 +213,7 @@ func TestParseJSONLogDetectsHashFallback(t *testing.T) { func TestParseJSONLogNoFalseHashFallback(t *testing.T) { stream := `{"stats":{"errors":0,"fatalError":false,"totalTransfers":2,"totalChecks":1,"bytes":10}}` var r RunResult - parseJSONLog(strings.NewReader(stream), &r, nil) + parseJSONLog(strings.NewReader(stream), &r, nil, nil) if r.HashFallback { t.Fatalf("HashFallback = true on a clean run, want false") } @@ -187,7 +232,7 @@ func TestParseJSONLogEmitsProgressWithByteTotal(t *testing.T) { var events []runevents.Progress parseJSONLog(strings.NewReader(stream), &r, func(p runevents.Progress) { events = append(events, p) - }) + }, nil) if len(events) != 2 { t.Fatalf("progress events = %d, want 2", len(events)) } From 792ce7acba820f8f69234b70d1bdd082ca2b053e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 12:07:26 +0000 Subject: [PATCH 08/14] status: address Copilot review on #177 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five review comments, all in one round: 1. PARTIAL terminal outcome (status/labels.go StateLabel, status/build.go syncLevel): a 'partial' latest terminal run is terminal and degraded, but both sites only special-cased 'failed' and fell through to freshness (ok/late). A latest 'partial' now renders "partial"/amber in both — it is more recent than any success, so freshness must not mask it. 2. Never-indexed volume offload policy (status/build.go buildVolume): the never-indexed early return left Offload.Applicable=false, so a volume that declares offload_requires printed "offload: no policy". Applicable is now set from the config policy up front (present offload_requires), independent of whether a tally ran. 3. Single-pass readiness (offload/readiness.go): Readiness built and sorted a full candidate slice via selectCandidates and then iterated it again; ordering is irrelevant to totals. It now walks the index map once, applying the same present + not-under-reserved-subtree predicate and the same gate.check. tallyReadiness removed. 4. TUI readiness cost (tui/dashboard.go): the 1s dashboard tick called status.Build → offload.Readiness (whole-index gate pass) every second, risking deadline misses on large volumes. Added status.Options {SkipOffloadReadiness} + BuildWithOptions; the CLI keeps the full Build. The dashboard rebuilds the coverage/durability grid every tick with readiness skipped and refreshes the offload tally on a slower cadence (readinessRefreshTicks) into a per-volume cache, overlaid at render, so "N offloadable now" stays visible without the per-tick cost. Tests: partial in StateLabel + a syncLevel terminal-outcome table; never- indexed-reports-policy and skip-vs-full readiness Build tests; dashboard test updated for the new loadDashboardData arity. go vet, go test ./..., golangci-lint (2.12.2) all green. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- offload/readiness.go | 19 +++++------- status/build.go | 59 +++++++++++++++++++++++++++-------- status/build_test.go | 64 ++++++++++++++++++++++++++++++++++++++ status/labels.go | 8 +++-- status/logic_test.go | 22 ++++++++++++++ tui/dashboard.go | 71 +++++++++++++++++++++++++++++++++++++------ tui/dashboard_test.go | 2 +- 7 files changed, 210 insertions(+), 35 deletions(-) diff --git a/offload/readiness.go b/offload/readiness.go index a3e251d..bbc1a09 100644 --- a/offload/readiness.go +++ b/offload/readiness.go @@ -48,8 +48,10 @@ type ReadinessReport struct { // read-only introspection surface at any time. // // The gate is evaluated exactly as Offload's dry-run path evaluates it — -// same loadGate, same candidate selection, same per-file check — so the -// totals match what an offload would actually move. +// same loadGate, same present-and-not-reserved candidate predicate, same +// per-file check — so the totals match what an offload would actually move. +// Totals are order-independent, so this walks the index map in a single +// pass rather than building and sorting an intermediate candidate slice. func Readiness(ctx context.Context, s *store.Store, opts ReadinessOptions) (ReadinessReport, error) { if len(opts.Require) == 0 { return ReadinessReport{Applicable: false}, nil @@ -63,19 +65,14 @@ func Readiness(ctx context.Context, s *store.Store, opts ReadinessOptions) (Read if err != nil { return ReadinessReport{}, err } - candidates, _ := selectCandidates(rows, nil, 0, now) - return tallyReadiness(ctx, g, candidates) -} - -// tallyReadiness runs the gate over every candidate and accumulates the -// present and offloadable totals. Split from Readiness so the loop body -// stays small and the context-cancellation check has one home. -func tallyReadiness(ctx context.Context, g *gate, candidates []store.FileRow) (ReadinessReport, error) { rep := ReadinessReport{Applicable: true} - for _, row := range candidates { + for p, row := range rows { if err := ctx.Err(); err != nil { return ReadinessReport{}, err } + if row.Status != store.StatusPresent || underReservedSubtree(p) { + continue + } rep.PresentFiles++ rep.PresentBytes += row.SizeBytes failures, err := g.check(ctx, row) diff --git a/status/build.go b/status/build.go index ca37431..918cc09 100644 --- a/status/build.go +++ b/status/build.go @@ -19,12 +19,34 @@ import ( // destination that has effectively stopped (red). const lateFactor = 3 -// Build produces the status Report for this node: every config-declared -// volume with its per-target coverage and durability. It is read-only — -// no run rows, no disk reads, no mutation — so it is safe to call from any -// introspection surface at any time. cfg is the source of truth for which -// volumes and targets should exist; the store supplies the observations. +// Options tunes Build. +type Options struct { + // SkipOffloadReadiness omits the per-volume offload-readiness tally. + // offload.Readiness loads the whole volume index and runs the gate over + // every present file, so on large volumes it is the report's dominant + // cost. The TUI dashboard sets it on its one-second tick (and refreshes + // the tally on a slower cadence into its own cache) so the coverage and + // durability grid stays responsive; the CLI `squirrel status` leaves it + // off so the "N offloadable now" figure is always current. When set, a + // volume's Offload still reports Applicable from its config policy, just + // with zero counts. + SkipOffloadReadiness bool +} + +// Build produces the status Report for this node with readiness computed — +// the CLI's view and the default. See BuildWithOptions for the tunable +// form the TUI uses. func Build(ctx context.Context, s *store.Store, cfg *config.Config) (Report, error) { + return BuildWithOptions(ctx, s, cfg, Options{}) +} + +// BuildWithOptions produces the status Report for this node: every +// config-declared volume with its per-target coverage and durability. It is +// read-only — no run rows, no disk reads, no mutation — so it is safe to +// call from any introspection surface at any time. cfg is the source of +// truth for which volumes and targets should exist; the store supplies the +// observations. +func BuildWithOptions(ctx context.Context, s *store.Store, cfg *config.Config, opts Options) (Report, error) { rep := Report{Now: time.Now()} self, err := s.GetSelfNode(ctx) if err != nil { @@ -36,7 +58,7 @@ func Build(ctx context.Context, s *store.Store, cfg *config.Config) (Report, err } alarmByDest := indexAlarms(alarms) for _, name := range sortedVolumeNames(cfg) { - vs, err := buildVolume(ctx, s, cfg, self, alarmByDest, cfg.Volumes[name], rep.Now) + vs, err := buildVolume(ctx, s, cfg, self, alarmByDest, cfg.Volumes[name], rep.Now, opts) if err != nil { return Report{}, err } @@ -70,8 +92,11 @@ func sortedVolumeNames(cfg *config.Config) []string { // buildVolume assembles one volume's status. A volume with no index row // (declared but never indexed) still lists its planned targets so the // grid shows the configured coverage rather than a blank line. -func buildVolume(ctx context.Context, s *store.Store, cfg *config.Config, self store.Node, alarmByDest map[string]store.DestinationAlarm, vol *config.Volume, now time.Time) (VolumeStatus, error) { +func buildVolume(ctx context.Context, s *store.Store, cfg *config.Config, self store.Node, alarmByDest map[string]store.DestinationAlarm, vol *config.Volume, now time.Time, opts Options) (VolumeStatus, error) { vs := VolumeStatus{Name: vol.Name, Path: vol.Path} + // Applicable reflects the config policy, not whether a tally ran, so a + // never-indexed or skip-readiness volume still reports "has a policy". + vs.Offload.Applicable = len(vol.OffloadRequires) > 0 dbVol, err := s.GetVolumeByName(ctx, vol.Name) if store.IsNotFound(err) { vs.Targets, err = buildTargets(ctx, s, cfg, self, alarmByDest, vol, 0, false, 0, now) @@ -93,6 +118,9 @@ func buildVolume(ctx context.Context, s *store.Store, cfg *config.Config, self s if err != nil { return VolumeStatus{}, err } + if opts.SkipOffloadReadiness { + return vs, nil + } readiness, err := offload.Readiness(ctx, s, offload.ReadinessOptions{ VolumeID: dbVol.ID, Require: vol.OffloadRequires, MaxEvidenceAge: vol.OffloadMaxEvidenceAge, }) @@ -200,9 +228,11 @@ func isBootstrapRefusal(errVal sql.NullString) bool { return errVal.Valid && strings.Contains(errVal.String, "--init") } -// syncLevel scores the coverage dimension: standing states first, then a -// failed transfer, then freshness relative to the pair's own cadence. A -// relayed target (not pushed to by this node) has no coverage dimension — +// syncLevel scores the coverage dimension: standing states first, then the +// latest terminal outcome (a failed run is red, a partial run at least +// amber — both are more recent than any success, so they must not be +// masked by freshness), then freshness relative to the pair's own cadence. +// A relayed target (not pushed to by this node) has no coverage dimension — // its safety is entirely its durability — so it reads neutral. func syncLevel(ts TargetStatus, lastTerminal *store.Run) Level { switch ts.Standing { @@ -211,8 +241,13 @@ func syncLevel(ts TargetStatus, lastTerminal *store.Run) Level { case StandingNeedsBootstrap: return LevelWarn } - if lastTerminal != nil && lastTerminal.Status == store.RunStatusFailed { - return LevelCritical + if lastTerminal != nil { + switch lastTerminal.Status { + case store.RunStatusFailed: + return LevelCritical + case store.RunStatusPartial: + return LevelWarn + } } if !ts.SyncTarget { return LevelNeutral diff --git a/status/build_test.go b/status/build_test.go index 7ae8de9..798cca8 100644 --- a/status/build_test.go +++ b/status/build_test.go @@ -201,6 +201,70 @@ func TestBuildNeverIndexed(t *testing.T) { if vs.Level() != LevelWarn { t.Errorf("volume level = %v, want amber", vs.Level()) } + // No offload policy configured here, so it must not claim one. + if vs.Offload.Applicable { + t.Errorf("offload should not be applicable without a policy") + } +} + +// TestBuildNeverIndexedReportsPolicy: a never-indexed volume that DOES +// declare offload_requires must report Applicable so the surface shows its +// policy state, not "no policy" (Copilot review on #177). +func TestBuildNeverIndexedReportsPolicy(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + cfg := cfgFor("photos", "/does/not/matter", []string{"nas"}, []string{"nas", "s3archive"}, nil, []string{"nas"}) + rep, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build: %v", err) + } + vs := rep.Volumes[0] + if vs.Indexed { + t.Errorf("volume should not be indexed") + } + if !vs.Offload.Applicable { + t.Errorf("offload should be applicable — the config declares offload_requires") + } + if vs.Offload.OffloadableFiles != 0 || vs.Offload.PresentFiles != 0 { + t.Errorf("never-indexed volume should report zero counts: %+v", vs.Offload) + } +} + +// TestBuildSkipOffloadReadiness: the TUI tick path skips the whole-index +// gate pass but still reports the policy flag; the CLI path computes the +// counts. Both see the same coverage/durability grid. +func TestBuildSkipOffloadReadiness(t *testing.T) { + s := setupStore(t) + ctx := context.Background() + root, idx := indexTree(t, s, "photos") + v, _ := s.GetVolumeByName(ctx, "photos") + self, _ := s.GetSelfNode(ctx) + if err := s.UpsertDestinationRunIDVerified(ctx, v.ID, "dest", self.ID, idx.RunID, store.VerifyMethodBlake3, false); err != nil { + t.Fatalf("seed vector: %v", err) + } + recordSuccessfulSync(t, s, v.ID, "dest") + cfg := cfgFor("photos", root, []string{"dest"}, []string{"dest"}, []string{"dest"}, nil) + + skipped, err := BuildWithOptions(ctx, s, cfg, Options{SkipOffloadReadiness: true}) + if err != nil { + t.Fatalf("BuildWithOptions(skip): %v", err) + } + so := skipped.Volumes[0].Offload + if !so.Applicable || so.OffloadableFiles != 0 || so.PresentFiles != 0 { + t.Errorf("skip-readiness offload = %+v, want applicable with zero counts", so) + } + full, err := Build(ctx, s, cfg) + if err != nil { + t.Fatalf("Build(full): %v", err) + } + fo := full.Volumes[0].Offload + if !fo.Applicable || fo.OffloadableFiles != 3 || fo.PresentFiles != 3 { + t.Errorf("full offload = %+v, want 3 offloadable of 3 present", fo) + } + // The coverage grid is identical regardless of the readiness option. + if len(skipped.Volumes[0].Targets) != len(full.Volumes[0].Targets) { + t.Errorf("target count differs between skip and full builds") + } } // TestBuildRelayedRequiredTarget: a target named only in offload_requires diff --git a/status/labels.go b/status/labels.go index 50b3c70..790d4d9 100644 --- a/status/labels.go +++ b/status/labels.go @@ -66,7 +66,8 @@ func LastSyncLabel(t TargetStatus) string { } // StateLabel names the coverage state, most-specific first: standing states -// outrank a failed transfer, which outranks freshness. +// outrank the latest terminal outcome (a failed or partial run — both more +// recent than any success), which outranks freshness. func StateLabel(t TargetStatus) string { switch t.Standing { case StandingAlarm: @@ -76,8 +77,11 @@ func StateLabel(t TargetStatus) string { case StandingNeedsBootstrap: return "needs-init" } - if t.LastOutcome == "failed" { + switch t.LastOutcome { + case "failed": return "failed" + case "partial": + return "partial" } if !t.SyncTarget { return "—" diff --git a/status/logic_test.go b/status/logic_test.go index 88c6131..09c3ab2 100644 --- a/status/logic_test.go +++ b/status/logic_test.go @@ -98,6 +98,27 @@ func TestEvidenceStale(t *testing.T) { } } +func TestSyncLevelTerminalOutcome(t *testing.T) { + age := func(d time.Duration) *time.Duration { return &d } + // A fresh success would be OK, but a more-recent failed/partial terminal + // run must not be masked by that freshness. + fresh := TargetStatus{SyncTarget: true, LastSyncAgo: age(time.Minute), Cadence: time.Hour} + cases := []struct { + name string + terminal *store.Run + want Level + }{ + {"failed outranks fresh success", &store.Run{Status: store.RunStatusFailed}, LevelCritical}, + {"partial is at least amber", &store.Run{Status: store.RunStatusPartial}, LevelWarn}, + {"success uses freshness", &store.Run{Status: store.RunStatusSuccess}, LevelOK}, + } + for _, c := range cases { + if got := syncLevel(fresh, c.terminal); got != c.want { + t.Errorf("%s: syncLevel = %v, want %v", c.name, got, c.want) + } + } +} + func TestLevelExitCode(t *testing.T) { cases := map[Level]int{LevelNeutral: 0, LevelOK: 0, LevelWarn: 1, LevelCritical: 2} for l, want := range cases { @@ -149,6 +170,7 @@ func TestStateLabel(t *testing.T) { {"alarm", TargetStatus{Standing: StandingAlarm}, "alarm"}, {"needs-init", TargetStatus{Standing: StandingNeedsBootstrap}, "needs-init"}, {"failed", TargetStatus{LastOutcome: "failed", SyncTarget: true}, "failed"}, + {"partial", TargetStatus{LastOutcome: "partial", SyncTarget: true, LastSyncAgo: age(time.Minute), SyncLevel: LevelWarn}, "partial"}, {"relayed", TargetStatus{SyncTarget: false, Required: true}, "—"}, {"never", TargetStatus{SyncTarget: true}, "never-synced"}, {"ok", TargetStatus{SyncTarget: true, LastSyncAgo: age(time.Minute), SyncLevel: LevelOK}, "ok"}, diff --git a/tui/dashboard.go b/tui/dashboard.go index f6a94a0..862a183 100644 --- a/tui/dashboard.go +++ b/tui/dashboard.go @@ -37,8 +37,22 @@ type dashboardModel struct { loaded bool loadErr error agentStatus agentStatus + + // readinessByVol caches the last computed offload-readiness tally per + // volume name. The coverage/durability grid rebuilds every one-second + // tick with readiness skipped (its dominant cost — a whole-index gate + // pass); the tally is refreshed on the slower readinessRefreshTicks + // cadence and overlaid onto the grid at render time, so "N offloadable + // now" stays visible without paying for it every tick. + readinessByVol map[string]status.OffloadReadiness + ticksSinceReadiness int } +// readinessRefreshTicks is how many one-second ticks pass between +// offload-readiness recomputations. Coverage and durability still refresh +// every tick; only the expensive gate-pass tally is throttled. +const readinessRefreshTicks = 15 + // dashboardData is the snapshot rendered by the dashboard. Built fresh on // each tick — there is no incremental update path, since the queries are // trivial and the resulting struct is small. @@ -61,6 +75,10 @@ type dashboardData struct { type dashboardDataMsg struct { data dashboardData err error + // readinessFresh is true when this fetch recomputed the offload tally + // (a slow-cadence tick), so the receiver should refresh its cache from + // data.coverage; a fast tick leaves it false and the cache stands. + readinessFresh bool } type agentStatus struct { @@ -81,7 +99,10 @@ func newDashboardModel(s *store.Store, cfg *config.Config) *dashboardModel { func (m *dashboardModel) attachClient(c *agentClient) { m.client = c } func (m *dashboardModel) Init() tea.Cmd { - return tea.Batch(m.fetchData(), m.probeAgent()) + // Recompute readiness on (re)activation so the tally is current the + // moment the user opens the dashboard, then throttle on the tick path. + m.ticksSinceReadiness = 0 + return tea.Batch(m.fetchData(true), m.probeAgent()) } func (m *dashboardModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { @@ -90,12 +111,20 @@ func (m *dashboardModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.width, m.height = msg.Width, msg.Height return m, nil case tickMsg: - return m, tea.Batch(m.fetchData(), m.probeAgent()) + m.ticksSinceReadiness++ + refresh := m.ticksSinceReadiness >= readinessRefreshTicks + if refresh { + m.ticksSinceReadiness = 0 + } + return m, tea.Batch(m.fetchData(refresh), m.probeAgent()) case dashboardDataMsg: m.loaded = true m.loadErr = msg.err if msg.err == nil { m.data = msg.data + if msg.readinessFresh { + m.cacheReadiness(msg.data.coverage) + } } return m, nil case agentStatusMsg: @@ -213,12 +242,19 @@ func (m *dashboardModel) renderCoverage() string { return strings.Join(blocks, "\n\n") } -// renderVolumeCoverage renders one volume's coverage block. +// renderVolumeCoverage renders one volume's coverage block. The +// offload-readiness figure comes from the throttled cache (the grid itself +// rebuilds every tick with readiness skipped), falling back to the volume's +// own tally when the cache has no entry yet. func (m *dashboardModel) renderVolumeCoverage(v status.VolumeStatus) string { dot := lipgloss.NewStyle().Foreground(levelColour(v.Level())).Render("●") title := fmt.Sprintf("%s %s %s", dot, v.Name, styleMuted.Render(v.Path)) + offload := v.Offload + if cached, ok := m.readinessByVol[v.Name]; ok { + offload = cached + } meta := styleMuted.Render(fmt.Sprintf("index %s · %s", - status.IndexLabel(v), status.OffloadLabel(v.Offload))) + status.IndexLabel(v), status.OffloadLabel(offload))) if len(v.Targets) == 0 { return title + "\n" + meta + "\n" + styleMuted.Render(" no targets configured") } @@ -309,14 +345,29 @@ func (m *dashboardModel) volumeName(id sql.NullInt64) string { return fmt.Sprintf("vol#%d", id.Int64) } -func (m *dashboardModel) fetchData() tea.Cmd { +// fetchData loads the dashboard snapshot. withReadiness selects whether the +// coverage build pays for the offload-readiness tally: false on the +// one-second tick (the grid still refreshes; the cached tally is overlaid +// at render), true on the slow cadence and on activation. +func (m *dashboardModel) fetchData(withReadiness bool) tea.Cmd { return func() tea.Msg { // Use a tight per-fetch deadline so a stuck DB doesn't freeze the UI. ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - data, err := loadDashboardData(ctx, m.store, m.cfg) - return dashboardDataMsg{data: data, err: err} + data, err := loadDashboardData(ctx, m.store, m.cfg, !withReadiness) + return dashboardDataMsg{data: data, err: err, readinessFresh: withReadiness && err == nil} + } +} + +// cacheReadiness refreshes the per-volume offload tally from a report that +// was built with readiness computed. Fast-tick reports carry only the +// policy flag (zero counts) and must not overwrite this cache. +func (m *dashboardModel) cacheReadiness(rep status.Report) { + cache := make(map[string]status.OffloadReadiness, len(rep.Volumes)) + for _, v := range rep.Volumes { + cache[v.Name] = v.Offload } + m.readinessByVol = cache } func (m *dashboardModel) probeAgent() tea.Cmd { @@ -342,7 +393,9 @@ func (m *dashboardModel) probeAgent() tea.Cmd { // the recent-runs window still surfaces correctly. The coverage build is // skipped when no config is loaded — it needs sync_to / offload_requires / // cadences — leaving the grid to render its own "no config" hint. -func loadDashboardData(ctx context.Context, s *store.Store, cfg *config.Config) (dashboardData, error) { +// skipReadiness omits the expensive offload-readiness tally (see fetchData +// and readinessRefreshTicks). +func loadDashboardData(ctx context.Context, s *store.Store, cfg *config.Config, skipReadiness bool) (dashboardData, error) { now := time.Now() vols, err := s.ListVolumes(ctx) if err != nil { @@ -358,7 +411,7 @@ func loadDashboardData(ctx context.Context, s *store.Store, cfg *config.Config) } var coverage status.Report if cfg != nil { - if coverage, err = status.Build(ctx, s, cfg); err != nil { + if coverage, err = status.BuildWithOptions(ctx, s, cfg, status.Options{SkipOffloadReadiness: skipReadiness}); err != nil { return dashboardData{}, fmt.Errorf("build coverage: %w", err) } } diff --git a/tui/dashboard_test.go b/tui/dashboard_test.go index 9a84838..29bdea2 100644 --- a/tui/dashboard_test.go +++ b/tui/dashboard_test.go @@ -58,7 +58,7 @@ func TestLoadDashboardDataPartitionsRuns(t *testing.T) { // cfg is nil here: this test pins the run-partitioning behaviour, which // is independent of the config-driven coverage grid (that is exercised // against the shared status layer in the status package's own tests). - data, err := loadDashboardData(ctx, s, nil) + data, err := loadDashboardData(ctx, s, nil, false) if err != nil { t.Fatalf("loadDashboardData: %v", err) } From c4b50b43891320af6351dce898a5edee98d59b20 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 12:10:46 +0000 Subject: [PATCH 09/14] =?UTF-8?q?conflicts:=20address=20PR=20#180=20review?= =?UTF-8?q?=20=E2=80=94=20record=20freeze=20pre-transfer,=20quote=20hint,?= =?UTF-8?q?=20scope=20reminder=20(#158)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FIX 1 (observability): the initiator now mirrors the freeze into its own contested_paths latch via a deferred call armed right after /plan, so the losing edge's badge/`squirrel conflicts` signal shows even when the transfer, verify, or close later fails — the receiver already froze the path during /plan pre-stage, so the conflict is real regardless of whether our bytes land. New test drives a conflict then a forced transfer failure (bogus rclone binary, no rclone needed) and asserts the local latch is set. FIX 2: the `conflicts resolve` hint printed after a contested sync now quotes the volume and path (%q), so a contested path containing whitespace stays a single shell word when copy-pasted. FIX 3: the contested reminder under `squirrel runs` renders volumes by name (via the shared label map, never the internal id) and, under --volume, scopes to that volume instead of listing unrelated freezes. New CLI test covers both. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/conflicts_test.go | 42 ++++++++++++++++++++++++++ cmd/squirrel/runs.go | 29 +++++++++++++----- cmd/squirrel/sync.go | 5 +++- sync/node.go | 9 +++++- sync/node_test.go | 55 ++++++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 10 deletions(-) diff --git a/cmd/squirrel/conflicts_test.go b/cmd/squirrel/conflicts_test.go index 4802550..dd49e19 100644 --- a/cmd/squirrel/conflicts_test.go +++ b/cmd/squirrel/conflicts_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "os" "path/filepath" "strings" "testing" @@ -74,6 +75,47 @@ func TestCLIConflictsListAndResolve(t *testing.T) { } } +// TestCLIRunsContestedReminder covers the reminder beneath `squirrel runs`: +// it names volumes (never the internal id) and, under --volume, scopes to +// that volume rather than listing freezes on unrelated ones. +func TestCLIRunsContestedReminder(t *testing.T) { + tmp := t.TempDir() + dirA := filepath.Join(tmp, "a") + dirB := filepath.Join(tmp, "b") + for _, d := range []string{dirA, dirB} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(d, "f.txt"), "content") + } + f := writeConfigFor(t, map[string]string{"a": dirA, "b": dirB}) + runCLI(t, "--config", f.configPath, "index", "a") + runCLI(t, "--config", f.configPath, "index", "b") + raiseTestContested(t, f.dbPath, "a", "f.txt") + raiseTestContested(t, f.dbPath, "b", "f.txt") + + // Unfiltered: both volumes appear, rendered by name not numeric id. + out := runCLI(t, "--config", f.configPath, "runs") + if !strings.Contains(out, "2 contested path(s)") { + t.Fatalf("unfiltered reminder = %q, want both freezes", out) + } + if !strings.Contains(out, "volume=a ") || !strings.Contains(out, "volume=b ") { + t.Fatalf("reminder did not name both volumes: %q", out) + } + if strings.Contains(out, "volume=1") || strings.Contains(out, "volume=2") { + t.Fatalf("reminder leaked a numeric volume id: %q", out) + } + + // Filtered to volume a: only a's freeze is reminded. + out = runCLI(t, "--config", f.configPath, "runs", "--volume", "a") + if !strings.Contains(out, "1 contested path(s)") || !strings.Contains(out, "volume=a ") { + t.Fatalf("filtered reminder = %q, want only volume a", out) + } + if strings.Contains(out, "volume=b ") { + t.Fatalf("filtered reminder leaked volume b: %q", out) + } +} + // TestCLIConflictsResolveUnknown: resolving a path that isn't contested is // a clean error, not a silent no-op. func TestCLIConflictsResolveUnknown(t *testing.T) { diff --git a/cmd/squirrel/runs.go b/cmd/squirrel/runs.go index 650a00e..2336db4 100644 --- a/cmd/squirrel/runs.go +++ b/cmd/squirrel/runs.go @@ -87,7 +87,7 @@ func runRuns(cmd *cobra.Command, volumeName string, limit int) error { if err != nil { return fmt.Errorf("list contested paths: %w", err) } - printContestedReminder(cmd.OutOrStdout(), contested) + printContestedReminder(cmd.OutOrStdout(), contested, opts.VolumeID, volumes) return nil } @@ -95,16 +95,29 @@ func runRuns(cmd *cobra.Command, volumeName string, limit int) error { // runs listing (#158, F27). Like the alarms reminder, a single old // conflict run scrolls away, so the audit surface repeats the standing // condition until a human resolves it — and points at the dedicated -// `squirrel conflicts` question-command for the detail. -func printContestedReminder(w io.Writer, contested []store.ContestedPath) { - if len(contested) == 0 { +// `squirrel conflicts` question-command for the detail. When the listing +// was scoped with --volume (filterVolumeID non-nil), the reminder is +// scoped to match rather than reporting freezes on unrelated volumes. +// Volumes render by name via the shared label map, never by internal id. +func printContestedReminder(w io.Writer, contested []store.ContestedPath, filterVolumeID *int64, volumes map[int64]string) { + scoped := contested + if filterVolumeID != nil { + scoped = scoped[:0:0] + for _, c := range contested { + if c.VolumeID == *filterVolumeID { + scoped = append(scoped, c) + } + } + } + if len(scoped) == 0 { return } fmt.Fprintf(w, "\n%d contested path(s) frozen — inspect with `squirrel conflicts`, clear with `squirrel conflicts resolve`:\n", - len(contested)) - for _, c := range contested { - fmt.Fprintf(w, " CONTESTED volume=%d %s since %s\n", - c.VolumeID, c.Path, formatStarted(c.RaisedAtNs)) + len(scoped)) + for _, c := range scoped { + fmt.Fprintf(w, " CONTESTED volume=%s %s since %s\n", + volumeLabel(sql.NullInt64{Int64: c.VolumeID, Valid: true}, volumes), + c.Path, formatStarted(c.RaisedAtNs)) } } diff --git a/cmd/squirrel/sync.go b/cmd/squirrel/sync.go index 02d0e3b..ab065ce 100644 --- a/cmd/squirrel/sync.go +++ b/cmd/squirrel/sync.go @@ -229,7 +229,10 @@ func printSyncReport(w io.Writer, rep sync.Report, runErr error) { fmt.Fprintf(w, " versions live %s, preserved %s at %s:%s\n", c.LiveBlake3Hex, c.PreservedBlake3Hex, rep.Destination, c.PreservedAtPath) } - fmt.Fprintf(w, " resolve with: squirrel conflicts resolve %s %s\n", rep.Volume, c.Path) + // Quote the args: a contested path may contain whitespace + // (validateRelPath allows it), which would otherwise split into + // several shell words when the operator copy-pastes the hint. + fmt.Fprintf(w, " resolve with: squirrel conflicts resolve %q %q\n", rep.Volume, c.Path) } for _, ff := range r.FailedFiles { // Some rclone errors (auth, listing, fatal copy) have no Object. diff --git a/sync/node.go b/sync/node.go index 4affac2..2a73de8 100644 --- a/sync/node.go +++ b/sync/node.go @@ -224,6 +224,14 @@ func (d *nodeSyncDriver) run() error { // them, no bytes move, and the initiator only surfaces the freeze. d.report.NodeConflicts = plan.Conflicts d.report.NodeContested = plan.Contested + // Mirror the freeze into this node's own contested_paths latch as + // soon as /plan has classified it, deferred so it still runs when a + // later phase (transfer, verify, close) fails. The receiver already + // pre-staged the loser and raised its own latch during /plan, so the + // conflict is real regardless of whether our bytes land — recording + // only after a successful close would hide the badge on the losing + // edge exactly when a sync failed mid-flight (#158, F27). + defer d.recordContestedLocally() if !d.opts.DryRun { advance, err := captureDurabilityAdvance(d.ctx, d.store, d.volID) if err != nil { @@ -240,7 +248,6 @@ func (d *nodeSyncDriver) run() error { if err := d.phaseClose(); err != nil { return fmt.Errorf("close: %w", err) } - d.recordContestedLocally() return nil } diff --git a/sync/node_test.go b/sync/node_test.go index 2ef8d79..176f62c 100644 --- a/sync/node_test.go +++ b/sync/node_test.go @@ -565,6 +565,61 @@ func TestNodeSyncContestedFreezeEndToEnd(t *testing.T) { } } +// TestNodeSyncContestedMirroredOnTransferFailure guards the observability +// fix: the initiator must mirror a freeze into its own contested_paths +// latch even when the sync fails *after* /plan. The receiver already +// pre-staged the loser and froze the path during /plan, so recording the +// badge only on a successful close would hide it on the losing edge +// exactly when a sync broke mid-flight (#158, F27). A bogus rclone binary +// fails the transfer deterministically without needing rclone installed — +// begin + plan run over the in-process HTTP receiver. +func TestNodeSyncContestedMirroredOnTransferFailure(t *testing.T) { + f, root := buildNodeFixture(t) + f.rcl = &Rclone{Binary: filepath.Join(root, "no-such-rclone-binary")} + ctx := context.Background() + + // Receiver holds a local-write doc.md; the initiator diverges → conflict. + v, err := f.recvStore.CreateVolume(ctx, f.recvVol.Name, f.recvVol.Path) + if err != nil { + t.Fatalf("seed receiver volume: %v", err) + } + runID, err := f.recvStore.BeginIndexRun(ctx, store.RunKindIndex, v.ID, false) + if err != nil { + t.Fatalf("BeginIndexRun: %v", err) + } + _ = f.recvStore.FinishRun(ctx, runID, store.RunStatusSuccess, "", 1) + if err := os.WriteFile(filepath.Join(f.recvVol.Path, "doc.md"), []byte("recvr"), 0o644); err != nil { + t.Fatal(err) + } + receiverDigest := hashFile(t, filepath.Join(f.recvVol.Path, "doc.md")) + if err := f.recvStore.Upsert(ctx, store.FileRow{ + VolumeID: v.ID, Path: "doc.md", Blake3: receiverDigest, SizeBytes: 5, MtimeNs: 50, + Status: store.StatusPresent, FirstSeenRunID: runID, LastSeenRunID: runID, IndexedAtNs: 50, + }, nil); err != nil { + t.Fatalf("seed receiver file row: %v", err) + } + if err := os.WriteFile(filepath.Join(f.initVol.Path, "doc.md"), []byte("initr"), 0o644); err != nil { + t.Fatal(err) + } + f.indexInitiator(t) + + // The transfer must fail (bogus binary), so the sync returns an error. + _, err = SyncNode(ctx, f.initStore, f.rcl, f.initVol, f.node, Options{Shallow: true}) + if err == nil { + t.Fatal("SyncNode succeeded, want a transfer failure") + } + + // Despite the failure, the initiator mirrored the freeze locally — the + // losing edge's badge / `squirrel conflicts` signal is present. + initVolRow, err := f.initStore.GetVolumeByName(ctx, "pics") + if err != nil { + t.Fatalf("initiator GetVolumeByName: %v", err) + } + if _, contested, err := f.initStore.IsPathContested(ctx, initVolRow.ID, "doc.md"); err != nil || !contested { + t.Fatalf("initiator IsPathContested = (%v, %v), want frozen after a post-plan failure", contested, err) + } +} + // TestNodeSyncVerifyMismatchPartialStatus simulates rclone "succeeding" // but the on-disk content being wrong (the agent's re-hash will // catch it). We inject the mismatch by writing different content From 24035a4c8abbe4895f52c0c26d291bc93675d280 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 12:10:48 +0000 Subject: [PATCH 10/14] runs: bound the conflict-count query for --failed/--changes (Copilot #179) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `gatherRuns` fetched an unbounded run slice under --failed/--changes and counted conflicts across the whole thing, feeding every peer-sync id into one IN (...) query — on a large history that overflows SQLite's bound-parameter cap ("too many SQL variables"), and --failed never needed the counts at all. - Conflict counts are now loaded only where needed: --failed consults none for filtering (an attention-worthy run is kept unconditionally); --changes counts only the ambiguous rows (a clean 0-file success); and the CONFLICTS-column counts are loaded for just the displayed rows, after filtering and truncation, not the unbounded fetch. - CountFilesFirstSeenByRunWithPathPrefix batches the id set (chunks of 400, each query binding chunk+2 params) so no caller can overflow the cap regardless of history size. Added a store test that passes 5000+ ids. filterRuns stays a pure predicate (unit-tested); the load strategy lives in gatherRuns/conflictCountsForFilter. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/runs.go | 50 +++++++++++++++++++++++++++++++++++--------- store/runs.go | 30 +++++++++++++++++++++----- store/store_test.go | 36 +++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 15 deletions(-) diff --git a/cmd/squirrel/runs.go b/cmd/squirrel/runs.go index 544569c..c21456e 100644 --- a/cmd/squirrel/runs.go +++ b/cmd/squirrel/runs.go @@ -95,11 +95,18 @@ func printActiveAlarms(w io.Writer, alarms []store.DestinationAlarm) { } } -// gatherRuns resolves the optional volume filter, fetches runs, derives -// conflict counts, and applies the --failed/--changes predicates. When a -// predicate is active the fetch is unbounded (opts.Limit=0) so the SQL -// LIMIT can't cap before the predicate runs and silently drop matches; the -// result is truncated to limit after filtering instead. +// gatherRuns resolves the optional volume filter, fetches runs, applies the +// --failed/--changes predicates, and returns the rows to display plus the +// conflict counts for their CONFLICTS column. When a predicate is active the +// fetch is unbounded (opts.Limit=0) so the SQL LIMIT can't cap before the +// predicate runs and silently drop matches; the result is truncated to limit +// after filtering instead. +// +// Conflict counts are loaded only where they are actually needed, never +// across the whole unbounded fetch: --failed needs none for filtering, and +// the CONFLICTS column only covers the displayed rows. This keeps the id set +// handed to the count query small (and the query itself batches ids), so a +// large peer-sync history can't overflow SQLite's bound-parameter cap. func gatherRuns(cmd *cobra.Command, s *store.Store, volumeName string, limit int, onlyFailed, onlyChanges bool) ([]store.Run, map[int64]int, error) { filtering := onlyFailed || onlyChanges opts := store.ListRunsOpts{Limit: limit, Descending: true} @@ -120,19 +127,42 @@ func gatherRuns(cmd *cobra.Command, s *store.Store, volumeName string, limit int if err != nil { return nil, nil, err } - conflicts, err := loadConflictCounts(cmd, s, runs) - if err != nil { - return nil, nil, err - } if filtering { - runs = filterRuns(runs, conflicts, onlyFailed, onlyChanges) + filterConflicts, err := conflictCountsForFilter(cmd, s, runs, onlyFailed) + if err != nil { + return nil, nil, err + } + runs = filterRuns(runs, filterConflicts, onlyFailed, onlyChanges) if limit > 0 && len(runs) > limit { runs = runs[:limit] } } + // Load the CONFLICTS-column counts for exactly the displayed rows. + conflicts, err := loadConflictCounts(cmd, s, runs) + if err != nil { + return nil, nil, err + } return runs, conflicts, nil } +// conflictCountsForFilter loads only the conflict counts the active filter +// needs. --failed consults none — an attention-worthy run is unconditionally +// kept — so it returns an empty map. --changes needs them only for the +// ambiguous rows (a clean success that touched no files), so it counts just +// those rather than the whole unbounded fetch. +func conflictCountsForFilter(cmd *cobra.Command, s *store.Store, runs []store.Run, onlyFailed bool) (map[int64]int, error) { + if onlyFailed { + return map[int64]int{}, nil + } + var candidates []store.Run + for _, r := range runs { + if r.Status == store.RunStatusSuccess && r.FileCount == 0 { + candidates = append(candidates, r) + } + } + return loadConflictCounts(cmd, s, candidates) +} + // filterRuns applies the --failed / --changes predicates in one pass, // preserving the descending order the store returned. func filterRuns(runs []store.Run, conflicts map[int64]int, onlyFailed, onlyChanges bool) []store.Run { diff --git a/store/runs.go b/store/runs.go index 027e4ac..98b51c5 100644 --- a/store/runs.go +++ b/store/runs.go @@ -400,11 +400,31 @@ func (s *Store) GetRun(ctx context.Context, id int64) (Run, error) { // from the sync-package directory naming convention. func (s *Store) CountFilesFirstSeenByRunWithPathPrefix(ctx context.Context, runIDs []int64, pathPrefix string) (map[int64]int, error) { out := make(map[int64]int, len(runIDs)) + // Batch the id set: each query binds len(chunk)+2 parameters, so an + // unbatched call over a large peer-sync history would overflow SQLite's + // bound-parameter cap (SQLITE_MAX_VARIABLE_NUMBER) and fail with "too + // many SQL variables". 400 keeps every query well under any cap. + const idsPerQuery = 400 + for start := 0; start < len(runIDs); start += idsPerQuery { + end := start + idsPerQuery + if end > len(runIDs) { + end = len(runIDs) + } + if err := s.countFilesFirstSeenChunk(ctx, runIDs[start:end], pathPrefix, out); err != nil { + return nil, err + } + } + return out, nil +} + +// countFilesFirstSeenChunk runs the grouped count for one id batch and folds +// the results into out. Non-zero counts only; absent keys mean zero. +func (s *Store) countFilesFirstSeenChunk(ctx context.Context, runIDs []int64, pathPrefix string, out map[int64]int) error { if len(runIDs) == 0 { - return out, nil + return nil } placeholders := strings.Repeat("?,", len(runIDs)-1) + "?" - args := make([]any, 0, len(runIDs)+1) + args := make([]any, 0, len(runIDs)+2) for _, id := range runIDs { args = append(args, id) } @@ -416,18 +436,18 @@ func (s *Store) CountFilesFirstSeenByRunWithPathPrefix(ctx context.Context, runI AND (fo.path = ? OR fo.path LIKE ? ESCAPE '\') GROUP BY f.first_seen_run_id`, args...) if err != nil { - return nil, fmt.Errorf("count files by run: %w", err) + return fmt.Errorf("count files by run: %w", err) } defer rows.Close() for rows.Next() { var id int64 var n int if err := rows.Scan(&id, &n); err != nil { - return nil, fmt.Errorf("scan count row: %w", err) + return fmt.Errorf("scan count row: %w", err) } out[id] = n } - return out, rows.Err() + return rows.Err() } // escapeLikePrefix escapes %, _ and \ in s so it can be embedded into a diff --git a/store/store_test.go b/store/store_test.go index 6eefdd0..477f997 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -2523,6 +2523,42 @@ func TestCountFilesFirstSeenByRunWithPathPrefix(t *testing.T) { } } +// TestCountFilesFirstSeenBatchesLargeIDSet guards the Copilot #179 fix: an +// id set far larger than SQLite's bound-parameter cap must be batched, not +// fail with "too many SQL variables". A real matching row buried among the +// synthetic ids confirms the per-batch results still merge correctly. +func TestCountFilesFirstSeenBatchesLargeIDSet(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer s.Close() + ctx := context.Background() + volID := makeVolume(t, s, "/v") + run := makeRun(t, s, volID) + if err := s.Upsert(ctx, FileRow{ + VolumeID: volID, Path: ".squirrel-conflicts/run-1/a", Blake3: digest(0x42), + SizeBytes: 1, MtimeNs: 1, Status: StatusPresent, + FirstSeenRunID: run, LastSeenRunID: run, IndexedAtNs: 1, + }, nil); err != nil { + t.Fatalf("upsert: %v", err) + } + // Far more ids than any SQLITE_MAX_VARIABLE_NUMBER a single query could + // bind; the real run id is appended after 5000 synthetic non-matches. + ids := make([]int64, 0, 5001) + for i := int64(1); i <= 5000; i++ { + ids = append(ids, i*1000) + } + ids = append(ids, run) + counts, err := s.CountFilesFirstSeenByRunWithPathPrefix(ctx, ids, ".squirrel-conflicts") + if err != nil { + t.Fatalf("large id set errored (batching regressed?): %v", err) + } + if counts[run] != 1 { + t.Fatalf("counts[run] = %d, want 1", counts[run]) + } +} + // TestListPresentByOrigin pins the two filter modes: valid nodeID // returns rows attributed to that peer, NULL nodeID returns rows // without provenance (local writes). Superseded and missing rows From 62722ebb7eed3ccbb1f38872eae153771eb0fb9f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 12:19:59 +0000 Subject: [PATCH 11/14] agent: real per-destination FIFO queue; loosen stall-guard test timing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Copilot review on #181. Fix 1 (starvation): the dispatcher skipped a sync when its destination was already busy. Because the tick iterates volumes by name, two volumes due to the same destination on one tick meant the later one was skipped every tick forever — never a run row, never consuming its cadence, just skip-log spam. In the reference setup this is the common case (photos and docs both push to cloudbox / s3archive / kopia-mirror), and it defeated the per-destination QUEUES that #160 option 4b specified. Replace skip-on-busy with a real per-destination FIFO queue: an idle destination starts its worker at once, a busy one takes the pair onto its queue (deduped so re-evaluation can't queue it twice) and runs it when the current transfer finishes. At most one sync per destination, overall parallelism still bounded, pre-sync-index ordering and per-pair semantics preserved. The genuine same-pair in-flight skip stays (the DB HasRunningRun check in the scheduler). Fix 2/3 (flaky tests): the stallGuard tests used tight real-time bounds. Loosen them — the fire test waits for the guard rather than racing a deadline; the reset test drives advance() off a ticker at 20x the bound's resolution across a span twice the stall window, so scheduler jitter can't fake a stall. Tests: TestSyncDispatcherSerializesSameDestination (two volumes, one destination: both run, FIFO, max one concurrent) and TestSchedulerTwoVolumesSameDestinationBothRun (two volumes due to the same destination on one tick both produce a sync run — neither starved). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- agent/dispatcher.go | 152 ++++++++++++++++++++++++--------------- agent/dispatcher_test.go | 65 +++++++++++------ agent/scheduler_test.go | 48 +++++++++++-- sync/rclone_test.go | 28 ++++++-- 4 files changed, 202 insertions(+), 91 deletions(-) diff --git a/agent/dispatcher.go b/agent/dispatcher.go index 70c1053..b8f0cf8 100644 --- a/agent/dispatcher.go +++ b/agent/dispatcher.go @@ -23,10 +23,15 @@ const defaultMaxParallelSyncs = 4 // syncDispatcher runs (volume, destination) syncs off the scheduler's tick // goroutine so a slow or wedged destination can never delay the tick loop, -// index runs, peer syncs, or syncs to other destinations (#160, F25). The -// concurrency unit is the destination: at most one sync per destination is -// in flight at a time (inflight), and an overall semaphore (sem) bounds how -// many run concurrently across destinations. +// index runs, peer syncs, or syncs to other destinations (#160, F25). +// +// The concurrency unit is the destination: each destination has a FIFO +// queue drained by a single worker, so at most one sync per destination is +// in flight while a second volume due to the same destination (photos and +// docs both push to cloudbox in the reference setup) waits its turn instead +// of being skipped and starved — issue #160 option 4b is per-destination +// queues, not skip-on-busy. An overall semaphore bounds how many +// destinations transfer at once. // // The pre-sync index ordering the scheduler guarantees is unaffected: the // tick body still runs a volume's index to completion before it hands that @@ -38,9 +43,20 @@ type syncDispatcher struct { sem chan struct{} - mu sync.Mutex - inflight map[string]bool - wg sync.WaitGroup + mu sync.Mutex + dests map[string]*destQueue + wg sync.WaitGroup +} + +// destQueue is one destination's serial pipeline: at most one sync active, +// the rest waiting in FIFO order. queued holds every volume name active or +// pending on this destination so a re-dispatch of a pair already in the +// pipeline — the scheduler re-evaluates a still-unfinished pair on every +// tick — is a quiet no-op rather than a duplicate enqueue. +type destQueue struct { + pending []*config.Volume + queued map[string]bool + active bool } func newSyncDispatcher(run SyncRunner, logger *slog.Logger, now func() time.Time, maxParallel int) *syncDispatcher { @@ -48,19 +64,20 @@ func newSyncDispatcher(run SyncRunner, logger *slog.Logger, now func() time.Time maxParallel = defaultMaxParallelSyncs } return &syncDispatcher{ - run: run, - logger: logger, - now: now, - sem: make(chan struct{}, maxParallel), - inflight: make(map[string]bool), + run: run, + logger: logger, + now: now, + sem: make(chan struct{}, maxParallel), + dests: make(map[string]*destQueue), } } -// dispatch launches the sync for (vol, destName) on destName's worker, -// unless a sync to that destination is already in flight (a per-pair skip -// log, matching the CLI wording) or no runner is configured. It never -// blocks on the transfer: the rclone call runs in a goroutine bounded by -// the semaphore, so the caller — the tick loop — returns at once. +// dispatch enqueues the sync for (vol, destName) on destName's FIFO queue. +// An idle destination starts its worker immediately; a busy one takes the +// pair onto its queue to run when the current transfer finishes (deduped so +// re-evaluation on later ticks can't queue the same pair twice). It never +// blocks on the transfer — the tick loop returns at once, so a slow +// destination cannot delay any other. func (d *syncDispatcher) dispatch(ctx context.Context, vol *config.Volume, destName string) { if d.run == nil { d.logger.Info("scheduler.skipped", @@ -68,24 +85,67 @@ func (d *syncDispatcher) dispatch(ctx context.Context, vol *config.Volume, destN "reason", "sync runner not configured") return } - if !d.claim(destName) { - d.logger.Info("scheduler.skipped", - "kind", "sync", "volume", vol.Name, "destination", destName, - "reason", "in-flight sync run") + d.mu.Lock() + q := d.dests[destName] + if q == nil { + q = &destQueue{queued: make(map[string]bool)} + d.dests[destName] = q + } + if q.queued[vol.Name] { + d.mu.Unlock() // already active or queued for this destination return } - d.logger.Info("scheduler.kicked", - "kind", "sync", "volume", vol.Name, "destination", destName) + q.queued[vol.Name] = true + if q.active { + q.pending = append(q.pending, vol) + d.mu.Unlock() + return + } + q.active = true + d.mu.Unlock() d.wg.Add(1) - go d.runOne(ctx, vol, destName) + go d.worker(ctx, destName, vol) } -// runOne is the per-destination worker body: take an overall slot (or bail -// if the scheduler is shutting down), run the sync, and always release both -// the slot and the destination claim. -func (d *syncDispatcher) runOne(ctx context.Context, vol *config.Volume, destName string) { +// worker drains destName's queue one sync at a time, so the destination +// never has two transfers in flight. It exits — freeing the destination — +// when the queue empties; on shutdown (ctx cancelled) it drains without +// running, leaving the unfinished pairs to be re-evaluated next start. +func (d *syncDispatcher) worker(ctx context.Context, destName string, vol *config.Volume) { defer d.wg.Done() - defer d.release(destName) + for vol != nil { + if ctx.Err() == nil { + d.runOne(ctx, vol, destName) + } + vol = d.next(destName, vol.Name) + } +} + +// next removes the just-finished volume from destName's pipeline and returns +// the next queued volume, or nil (dropping the now-idle destination) when +// none remain. +func (d *syncDispatcher) next(destName, done string) *config.Volume { + d.mu.Lock() + defer d.mu.Unlock() + q := d.dests[destName] + if q == nil { + return nil + } + delete(q.queued, done) + if len(q.pending) == 0 { + delete(d.dests, destName) + return nil + } + next := q.pending[0] + q.pending = q.pending[1:] + return next +} + +// runOne takes an overall parallelism slot (or bails on shutdown), runs the +// sync, and emits the kicked/finished/error logs. A sync that stalls is +// failed by the runner's transfer timeout (rclone is killed); its +// diagnosable error lands here and in the run row. +func (d *syncDispatcher) runOne(ctx context.Context, vol *config.Volume, destName string) { select { case d.sem <- struct{}{}: defer func() { <-d.sem }() @@ -95,14 +155,8 @@ func (d *syncDispatcher) runOne(ctx context.Context, vol *config.Volume, destNam if ctx.Err() != nil { return } - d.execute(ctx, vol, destName) -} - -// execute invokes the sync runner and emits the finished/error logs, -// mirroring the scheduler's kicked/finished/error discipline. A sync that -// stalls is failed by the runner's transfer timeout (rclone is killed); -// its diagnosable error lands here and in the run row. -func (d *syncDispatcher) execute(ctx context.Context, vol *config.Volume, destName string) { + d.logger.Info("scheduler.kicked", + "kind", "sync", "volume", vol.Name, "destination", destName) start := d.now() rep := d.run(ctx, vol, destName) duration := d.now().Sub(start) @@ -122,27 +176,9 @@ func (d *syncDispatcher) execute(ctx context.Context, vol *config.Volume, destNa } } -// claim marks destName in flight, returning false when a sync to it is -// already running. release clears the mark. -func (d *syncDispatcher) claim(destName string) bool { - d.mu.Lock() - defer d.mu.Unlock() - if d.inflight[destName] { - return false - } - d.inflight[destName] = true - return true -} - -func (d *syncDispatcher) release(destName string) { - d.mu.Lock() - defer d.mu.Unlock() - delete(d.inflight, destName) -} - -// wait blocks until every dispatched sync has finished. The scheduler -// calls it during shutdown, after the run context is cancelled (which -// kills any in-flight rclone child), so it returns promptly. +// wait blocks until every destination worker has drained. The scheduler +// calls it during shutdown, after the run context is cancelled (which kills +// any in-flight rclone child), so it returns promptly. func (d *syncDispatcher) wait() { d.wg.Wait() } diff --git a/agent/dispatcher_test.go b/agent/dispatcher_test.go index c520775..c795341 100644 --- a/agent/dispatcher_test.go +++ b/agent/dispatcher_test.go @@ -49,37 +49,60 @@ func TestSyncDispatcherRunsDestinationsConcurrently(t *testing.T) { } } -// TestSyncDispatcherOneInFlightPerDestination proves the concurrency unit -// is the destination: a second volume targeting a destination that already -// has a sync in flight is skipped with the per-pair "in-flight sync run" -// log, and its runner never fires while the first is running. -func TestSyncDispatcherOneInFlightPerDestination(t *testing.T) { - var calls atomic.Int32 - entered := make(chan struct{}, 1) - release := make(chan struct{}) +// TestSyncDispatcherSerializesSameDestination proves the concurrency unit +// is the destination: two volumes due to the same destination on one tick +// both run — the second is queued behind the first, never starved — while +// at most one is ever in flight against that destination. +func TestSyncDispatcherSerializesSameDestination(t *testing.T) { + var active, maxActive atomic.Int32 + var photosRan, docsRan atomic.Bool + entered := make(chan string, 2) + release := make(chan struct{}) // one send per sync, in completion order run := func(ctx context.Context, vol *config.Volume, destName string) SyncRunReport { - calls.Add(1) - entered <- struct{}{} + n := active.Add(1) + for { + old := maxActive.Load() + if n <= old || maxActive.CompareAndSwap(old, n) { + break + } + } + switch vol.Name { + case "photos": + photosRan.Store(true) + case "docs": + docsRan.Store(true) + } + entered <- vol.Name <-release + active.Add(-1) return SyncRunReport{Status: store.RunStatusSuccess} } - buf := &bytes.Buffer{} - logger := slog.New(slog.NewTextHandler(buf, nil)) - d := newSyncDispatcher(run, logger, time.Now, 4) - + d := newSyncDispatcher(run, discardLogger(), time.Now, 4) d.dispatch(context.Background(), &config.Volume{Name: "photos"}, "cloudbox") - <-entered // the first sync has claimed cloudbox and is running d.dispatch(context.Background(), &config.Volume{Name: "docs"}, "cloudbox") - if got := calls.Load(); got != 1 { - t.Fatalf("runner called %d times; want 1 (second dispatch to a busy destination must skip)", got) + first := <-entered // FIFO: photos was dispatched first + if first != "photos" { + t.Fatalf("first sync = %q; want photos (FIFO order)", first) } - log := buf.String() - if !strings.Contains(log, `reason="in-flight sync run"`) || !strings.Contains(log, "volume=docs") { - t.Fatalf("expected a per-pair in-flight skip for docs→cloudbox, got:\n%s", log) + time.Sleep(20 * time.Millisecond) // give a wrongly-concurrent docs a chance to start + if got := active.Load(); got != 1 { + t.Fatalf("concurrent syncs to one destination = %d; want 1 (serialized)", got) } - close(release) + release <- struct{}{} // let photos finish; the worker then dequeues docs + if second := <-entered; second != "docs" { + t.Fatalf("second sync = %q; want docs (queued, not starved)", second) + } + release <- struct{}{} // let docs finish d.wait() + + if !photosRan.Load() || !docsRan.Load() { + t.Fatalf("both volumes must run to the shared destination; photos=%v docs=%v", + photosRan.Load(), docsRan.Load()) + } + if got := maxActive.Load(); got != 1 { + t.Fatalf("max concurrent syncs to one destination = %d; want 1", got) + } } // TestSyncDispatcherBoundsOverallParallelism proves the semaphore ceiling: diff --git a/agent/scheduler_test.go b/agent/scheduler_test.go index 2d03553..ff46705 100644 --- a/agent/scheduler_test.go +++ b/agent/scheduler_test.go @@ -124,10 +124,14 @@ func (f *fakeSyncRunner) Calls() []syncCall { // + fake sync runner. The scheduler's tick method drives behaviour // directly; the goroutine-driven run() loop is only used in the // integration test below. -func newSchedulerFixture(t *testing.T, volumeCfg *config.Volume) *schedulerFixture { +func newSchedulerFixture(t *testing.T, vols ...*config.Volume) *schedulerFixture { t.Helper() - if volumeCfg.Path == "" { - volumeCfg.Path = t.TempDir() + volumes := make(map[string]*config.Volume, len(vols)) + for _, v := range vols { + if v.Path == "" { + v.Path = t.TempDir() + } + volumes[v.Name] = v } dbPath := filepath.Join(t.TempDir(), "index.db") s, err := store.Open(dbPath) @@ -144,7 +148,7 @@ func newSchedulerFixture(t *testing.T, volumeCfg *config.Volume) *schedulerFixtu Listen: "127.0.0.1:0", Token: "tok", Version: "test", - Volumes: map[string]*config.Volume{volumeCfg.Name: volumeCfg}, + Volumes: volumes, Logger: logger, SyncRunner: syncRunner.Runner(), }, s) @@ -194,7 +198,6 @@ func (f *schedulerFixture) seedFile() { if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { f.t.Fatalf("write %s: %v", p, err) } - return } } @@ -468,6 +471,41 @@ func TestSchedulerSkipsWhenInFlightSyncRun(t *testing.T) { } } +// TestSchedulerTwoVolumesSameDestinationBothRun guards the starvation trap +// (#160): the tick iterates volumes by name, so if two volumes are due to +// the same destination on one tick the earlier one wins the destination. +// The dispatcher must queue the later one behind it and run it — not skip +// it every tick forever. "aaa" sorts before "zzz"; both must produce a sync +// run to "backup". +func TestSchedulerTwoVolumesSameDestinationBothRun(t *testing.T) { + f := newSchedulerFixture(t, + &config.Volume{Name: "aaa", SyncTo: []string{"backup"}, SyncEvery: time.Hour}, + &config.Volume{Name: "zzz", SyncTo: []string{"backup"}, SyncEvery: time.Hour}, + ) + f.seedFile() + sch := f.scheduler() + + sch.tick(context.Background()) + sch.dispatch.wait() + + var aaaRan, zzzRan bool + for _, c := range f.syncLog.Calls() { + if c.Destination != "backup" { + continue + } + switch c.Volume { + case "aaa": + aaaRan = true + case "zzz": + zzzRan = true + } + } + if !aaaRan || !zzzRan { + t.Fatalf("both volumes must sync to the shared destination; aaa=%v zzz=%v (calls=%+v)", + aaaRan, zzzRan, f.syncLog.Calls()) + } +} + // TestSchedulerSkipsWhenVolumeLockHeld pins the coordination with the // peer-sync surface: an incoming /v1/sync session holds the per-volume // router lock for the duration of the session; a scheduler tick during diff --git a/sync/rclone_test.go b/sync/rclone_test.go index 0b252ac..340eb2e 100644 --- a/sync/rclone_test.go +++ b/sync/rclone_test.go @@ -18,11 +18,14 @@ import ( // TestStallGuardFiresWithoutProgress proves the no-progress guard cancels // the run — and records that it fired — when no advance arrives within the // timeout. This is the F25 wedge: rclone alive but transferring nothing. +// The test waits for the guard rather than racing a deadline, so the exact +// timeout is not timing-sensitive; a generous value keeps it robust under +// load. func TestStallGuardFiresWithoutProgress(t *testing.T) { var cancelled atomic.Bool ctx, cancel := context.WithCancel(context.Background()) defer cancel() - g := newStallGuard(ctx, 30*time.Millisecond, func() { + g := newStallGuard(ctx, 100*time.Millisecond, func() { cancelled.Store(true) cancel() }) @@ -36,19 +39,30 @@ func TestStallGuardFiresWithoutProgress(t *testing.T) { } // TestStallGuardResetsOnProgress proves a transfer that keeps advancing is -// never killed: pokes arriving well inside the timeout keep resetting it, -// even across a span longer than a single timeout window. +// never killed: pokes driven off a ticker at a fraction of the stall window +// keep resetting it across a span twice as long as one window, so only a +// working reset explains the guard staying quiet. The wide ratio between +// the stall timeout and the poke interval keeps scheduler jitter from +// faking a stall. func TestStallGuardResetsOnProgress(t *testing.T) { var cancelled atomic.Bool ctx, cancel := context.WithCancel(context.Background()) defer cancel() - g := newStallGuard(ctx, 100*time.Millisecond, func() { + const stall = 400 * time.Millisecond + g := newStallGuard(ctx, stall, func() { cancelled.Store(true) cancel() }) - for i := 0; i < 8; i++ { // 160ms total, gaps of 20ms << the 100ms bound - time.Sleep(20 * time.Millisecond) - g.advance() + tick := time.NewTicker(stall / 20) // pokes at 20x the resolution of the bound + defer tick.Stop() + deadline := time.After(2 * stall) // outlast a single window, so a reset is required + for done := false; !done; { + select { + case <-tick.C: + g.advance() + case <-deadline: + done = true + } } if g.fired.Load() { t.Fatal("guard fired while progress was still arriving") From bb67dac2c030b3168b8df53e347d747b9c61ca9a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 17:31:06 +0000 Subject: [PATCH 12/14] =?UTF-8?q?store:=20regression-test=20the=20v24?= =?UTF-8?q?=E2=86=92v25=20runs=20rebuild=20preserves=20data=20+=20FKs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v24→v25 migration is the highest-risk change in #174: it rebuilds the FK-referenced runs table under PRAGMA foreign_keys=OFF to widen the status CHECK. The existing migration fixtures create an empty runs table, so no test drove real run rows (or a referencing FK row) through the rebuild. Add a focused test that builds a v24 DB with runs rows in every pre-v25 terminal state plus an in-flight 'running' row and a child row with a foreign key into runs, migrates to v25, and asserts: every row carried over with its id and status intact, the child FK still resolves (and PRAGMA foreign_key_check is clean), and the widened CHECK now admits 'refused'/'aborted' while still rejecting an unknown status. Test-only; no production change. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- store/migrate_v25_test.go | 118 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 store/migrate_v25_test.go diff --git a/store/migrate_v25_test.go b/store/migrate_v25_test.go new file mode 100644 index 0000000..fdf689e --- /dev/null +++ b/store/migrate_v25_test.go @@ -0,0 +1,118 @@ +package store + +import ( + "context" + "database/sql" + "path/filepath" + "testing" +) + +// TestMigrateV24ToV25PreservesRunsAndWidensStatus builds a minimal v24 +// database carrying runs rows in every pre-v25 terminal state plus an +// in-flight 'running' row, and a child row with a foreign key into runs, +// then migrates to v25. It pins the highest-risk part of the migration — +// the FK-off runs-table rebuild — proving it copies every row verbatim +// (id and status preserved), keeps a referencing FK resolving to the +// rebuilt table (the property the migration's own foreign_key_check +// guards on a full DB), and widens the status CHECK to admit 'refused' +// and 'aborted' without admitting a bogus status. +func TestMigrateV24ToV25PreservesRunsAndWidensStatus(t *testing.T) { + dsn := filepath.Join(t.TempDir(), "test.db") + rawDB, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatalf("raw sql.Open: %v", err) + } + v24DDL := []string{ + `CREATE TABLE schema_version (version INTEGER NOT NULL PRIMARY KEY)`, + `CREATE TABLE volumes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, path TEXT NOT NULL)`, + `CREATE TABLE nodes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, endpoint TEXT, public_key_fingerprint TEXT)`, + minimalRunsDDL, + // A child table with a real FK into runs proves the id-preserving + // rebuild keeps referencing rows resolving after runs is dropped + // and recreated under FK-off. + `CREATE TABLE run_child (id INTEGER PRIMARY KEY, run_id INTEGER NOT NULL REFERENCES runs(id))`, + `INSERT INTO schema_version (version) VALUES (24)`, + `INSERT INTO volumes (id, name, path) VALUES (1, 'v', '/v')`, + `INSERT INTO nodes (id, name) VALUES (1, 'self')`, + `INSERT INTO runs (id, kind, volume_id, destination, started_at_ns, ended_at_ns, status, error, file_count) + VALUES (1, 'index', 1, NULL, 100, 200, 'success', NULL, 5)`, + `INSERT INTO runs (id, kind, volume_id, destination, started_at_ns, ended_at_ns, status, error, file_count) + VALUES (2, 'sync', 1, 'bucket', 110, 210, 'failed', 'boom', 0)`, + `INSERT INTO runs (id, kind, volume_id, destination, started_at_ns, ended_at_ns, status, error, file_count) + VALUES (3, 'sync', 1, 'bucket', 120, 220, 'partial', NULL, 3)`, + `INSERT INTO runs (id, kind, volume_id, destination, started_at_ns, status, file_count) + VALUES (4, 'index', 1, NULL, 130, 'running', 0)`, + `INSERT INTO run_child (id, run_id) VALUES (1, 2)`, + } + for _, q := range v24DDL { + if _, err := rawDB.Exec(q); err != nil { + t.Fatalf("v24 DDL %q: %v", q, err) + } + } + rawDB.Close() + + s, err := Open(dsn) + if err != nil { + t.Fatalf("Open (migrates v24→v25): %v", err) + } + defer s.Close() + ctx := context.Background() + if v, _ := s.CurrentSchemaVersion(ctx); v != SchemaVersion { + t.Fatalf("schema_version = %d, want %d", v, SchemaVersion) + } + + // Every runs row carried over with its id and status intact. + wantStatus := map[int64]string{ + 1: RunStatusSuccess, 2: RunStatusFailed, 3: RunStatusPartial, 4: RunStatusRunning, + } + runs, err := s.ListRuns(ctx, ListRunsOpts{}) + if err != nil { + t.Fatalf("ListRuns: %v", err) + } + if len(runs) != len(wantStatus) { + t.Fatalf("runs after migration = %d, want %d", len(runs), len(wantStatus)) + } + for _, r := range runs { + if want := wantStatus[r.ID]; r.Status != want { + t.Fatalf("run %d status = %q, want %q", r.ID, r.Status, want) + } + } + + // The child FK still resolves to the rebuilt runs table, and no + // dangling reference slipped through the FK-off rebuild. + var refs int + if err := s.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM run_child c JOIN runs r ON r.id = c.run_id`).Scan(&refs); err != nil { + t.Fatalf("join child→runs: %v", err) + } + if refs != 1 { + t.Fatalf("child rows resolving to runs = %d, want 1", refs) + } + fkRows, err := s.db.QueryContext(ctx, `PRAGMA foreign_key_check`) + if err != nil { + t.Fatalf("foreign_key_check: %v", err) + } + violations := 0 + for fkRows.Next() { + violations++ + } + fkRows.Close() + if violations != 0 { + t.Fatalf("foreign_key_check reported %d violations after migration", violations) + } + + // The widened CHECK admits the two new terminal states... + for _, status := range []string{RunStatusRefused, RunStatusAborted} { + if _, err := s.db.ExecContext(ctx, + `INSERT INTO runs (kind, volume_id, destination, started_at_ns, status, file_count) + VALUES ('sync', 1, 'bucket', 300, ?, 0)`, status); err != nil { + t.Fatalf("insert %q run rejected by CHECK after migration: %v", status, err) + } + } + // ...but the CHECK still rejects an unknown status. + if _, err := s.db.ExecContext(ctx, + `INSERT INTO runs (kind, volume_id, destination, started_at_ns, status, file_count) + VALUES ('sync', 1, 'bucket', 320, 'bogus', 0)`); err == nil { + t.Fatalf("CHECK accepted a bogus status, want rejection") + } +} From 57b74543a337c529d6523c184352da20631d87a3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 17:59:29 +0000 Subject: [PATCH 13/14] status: surface genuine CLI errors that SilenceErrors was swallowing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The status command sets SilenceErrors so its scriptable green/amber/red exit-code signal (an exitCodeError) doesn't leak a cobra "Error:" line. But SilenceErrors is unconditional: it also silenced genuine failures — an unknown volume, a missing config, a store error — which then reached neither cobra's printer nor main.go's, leaving the user a bare exit 1 with no message. The existing tests missed it because they assert on the returned error value, not on user-visible output. Print non-exitCodeError errors from the command's RunE (the exit-code signal stays silent), and extend the unknown-volume and missing-config tests to assert the message actually reaches stderr. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/status.go | 13 ++++++++++++- cmd/squirrel/status_test.go | 16 +++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/cmd/squirrel/status.go b/cmd/squirrel/status.go index 8bae4fc..67f22b3 100644 --- a/cmd/squirrel/status.go +++ b/cmd/squirrel/status.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "io" "text/tabwriter" @@ -32,7 +33,17 @@ func newStatusCmd() *cobra.Command { if len(args) == 1 { volume = args[0] } - return runStatus(cmd, volume) + err := runStatus(cmd, volume) + // SilenceErrors keeps the scriptable green/amber/red exit-code + // signal (an exitCodeError) from leaking a cobra "Error:" line. But + // it also silences genuine failures — an unknown volume, a missing + // config, a store error — which must still reach the user, so print + // those here. The exit-code signal stays silent. + var ec exitCodeError + if err != nil && !errors.As(err, &ec) { + fmt.Fprintln(cmd.ErrOrStderr(), cmd.ErrPrefix(), err.Error()) + } + return err }, } return cmd diff --git a/cmd/squirrel/status_test.go b/cmd/squirrel/status_test.go index 16cea9c..3b96ba8 100644 --- a/cmd/squirrel/status_test.go +++ b/cmd/squirrel/status_test.go @@ -89,20 +89,30 @@ func TestCLIStatusAfterSyncIsGreen(t *testing.T) { } // TestCLIStatusUnknownVolume errors clearly on a volume not in config. +// SilenceErrors (needed to keep the amber/red exit-code signal quiet) must +// not swallow this genuine error: it has to reach the user's stderr, not +// just the returned error value. func TestCLIStatusUnknownVolume(t *testing.T) { cfg := writeStatusConfig(t) - _, err := runCLIExpectErr(t, "--config", cfg, "status", "nope") + out, err := runCLIExpectErr(t, "--config", cfg, "status", "nope") if err == nil || !strings.Contains(err.Error(), `unknown volume "nope"`) { t.Fatalf("expected unknown-volume error, got %v", err) } + if !strings.Contains(out, `unknown volume "nope"`) { + t.Fatalf("error must reach the user, not be silenced:\n%s", out) + } } // TestCLIStatusRequiresConfig: status is a question about configured -// coverage, so a missing config is a clear error, not an empty grid. +// coverage, so a missing config is a clear error, not an empty grid — and +// the message must be printed, not silently swallowed by SilenceErrors. func TestCLIStatusRequiresConfig(t *testing.T) { missing := filepath.Join(t.TempDir(), "no-config.toml") - _, err := runCLIExpectErr(t, "--config", missing, "status") + out, err := runCLIExpectErr(t, "--config", missing, "status") if err == nil || !strings.Contains(err.Error(), "no config at") { t.Fatalf("expected missing-config error, got %v", err) } + if !strings.Contains(out, "no config at") { + t.Fatalf("error must reach the user, not be silenced:\n%s", out) + } } From c0029c8531bc2708cfc630314e11777ac489428a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 18:58:32 +0000 Subject: [PATCH 14/14] agent: skip redundant pre-sync index for an in-flight sync (#160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Async dispatch (this PR) freed the tick loop to keep firing every 30s while a dispatched sync runs in the background. But maybeRunSync ran the pre-sync index before the per-pair in-flight check, and syncDue stays true for the whole duration of an in-flight sync — the finished-run watermark only advances when the sync completes. So a slow multi-hour push made the scheduler re-index the source on every tick, burning I/O and flooding the never-pruned runs audit trail with a kind='index' row per tick. The old serial scheduler never hit this: the tick blocked inside the sync, so it could not re-evaluate until the sync finished. Filter each volume's due destinations down to the pairs that are not already syncing before running the pre-sync index, and skip the index entirely when none remain (there is no new push for it to precede). The in-flight signal reads from two sources that compose to close every window: the dispatcher's in-memory queue (set the instant a pair is enqueued, before its run row is visible) and the runs table (authoritative once the row exists, and the signal that also catches a stale row or a concurrent CLI `squirrel sync`). The ordering invariant holds — a volume syncing to one in-flight dest and one free dest still indexes once and dispatches the free one — and the per-pair scheduler.skipped log is unchanged. runSync becomes syncInFlight (a bool predicate, no dispatch); maybeRunSync is decomposed into dueSyncDests/dispatchableSyncDests to stay small. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- agent/dispatcher.go | 16 ++++ agent/scheduler.go | 125 ++++++++++++++++++---------- agent/scheduler_test.go | 177 +++++++++++++++++++++++++++++++++++----- 3 files changed, 254 insertions(+), 64 deletions(-) diff --git a/agent/dispatcher.go b/agent/dispatcher.go index b8f0cf8..9f99a57 100644 --- a/agent/dispatcher.go +++ b/agent/dispatcher.go @@ -121,6 +121,22 @@ func (d *syncDispatcher) worker(ctx context.Context, destName string, vol *confi } } +// inFlight reports whether (destName, volName) is currently active or queued +// on destName's pipeline. The scheduler consults it alongside the runs table +// so a pair whose dispatch from an earlier tick is still working — but whose +// run row is not yet visible — is recognised as in flight and not dragged +// through a redundant pre-sync index every tick (#160). Safe for concurrent +// use; a nil-runner dispatcher (no dests) always reports false. +func (d *syncDispatcher) inFlight(destName, volName string) bool { + d.mu.Lock() + defer d.mu.Unlock() + q := d.dests[destName] + if q == nil { + return false + } + return q.queued[volName] +} + // next removes the just-finished volume from destName's pipeline and returns // the next queued volume, or nil (dropping the now-idle destination) when // none remain. diff --git a/agent/scheduler.go b/agent/scheduler.go index 0394688..8524404 100644 --- a/agent/scheduler.go +++ b/agent/scheduler.go @@ -234,42 +234,87 @@ func (s *scheduler) resolveVolume(ctx context.Context, name, absPath string) (st return created, nil } -// maybeRunSync evaluates the sync_every cadence and, if any destination -// is due, runs a pre-sync index and then the per-destination syncs. -// Returns true when the scheduler took any sync-related action (kicked, -// skipped, or errored) so the caller can suppress a redundant -// standalone-index check. +// maybeRunSync evaluates the sync_every cadence and, for the destinations +// that are due and not already syncing, runs a pre-sync index and then hands +// each pair to the per-destination dispatcher. Returns true when the +// scheduler took any sync-related action (kicked, skipped, or errored) so the +// caller can suppress a redundant standalone-index check. // // Invariant: a scheduled sync always runs an index immediately before -// pushing. If the pre-sync index fails or is skipped, no syncs run -// this tick — the next tick re-evaluates. +// pushing. If the pre-sync index fails or is skipped, no syncs run this tick +// — the next tick re-evaluates. +// +// A due destination whose (volume, destination) sync from an earlier tick is +// still in flight is dropped before the pre-sync index runs. Dispatch is +// asynchronous (#160), so without this the tick loop would re-index the source +// on every 30s tick for the entire duration of a slow multi-hour push — +// burning I/O and flooding the never-pruned runs audit trail with a +// kind='index' row per tick. When every due destination is already syncing, +// the pre-sync index is skipped entirely: there is no new push for it to +// precede, and the in-flight pushes were each already preceded by their own. func (s *scheduler) maybeRunSync(ctx context.Context, vol *config.Volume, volumeID int64) bool { if vol.SyncEvery == 0 { return false } - var dueDests []string + dueDests := s.dueSyncDests(ctx, vol, volumeID) + if len(dueDests) == 0 { + return false + } + dispatchable := s.dispatchableSyncDests(ctx, vol, volumeID, dueDests) + if len(dispatchable) == 0 { + // Every due destination is already syncing (each skip logged per + // pair). Report the sync action so the standalone-index branch stays + // suppressed — otherwise its own re-index would reintroduce the exact + // per-tick flood this guard exists to prevent. + return true + } + if !s.maybeRunIndex(ctx, vol, volumeID, "pre-sync", 0) { + return true + } + for _, destName := range dispatchable { + s.dispatch.dispatch(ctx, vol, destName) + } + return true +} + +// dueSyncDests returns the destinations on the volume's sync_to whose +// sync_every cadence has elapsed. A per-destination lookup error is logged and +// that destination is dropped from this tick's evaluation (the next tick +// re-evaluates), never aborting the others. +func (s *scheduler) dueSyncDests(ctx context.Context, vol *config.Volume, volumeID int64) []string { + var due []string for _, destName := range vol.SyncTo { - due, err := s.syncDue(ctx, volumeID, destName, vol.SyncEvery) + ok, err := s.syncDue(ctx, volumeID, destName, vol.SyncEvery) if err != nil { s.logger.Error("scheduler.error", "kind", "sync", "volume", vol.Name, "destination", destName, "err", err.Error()) continue } - if due { - dueDests = append(dueDests, destName) + if ok { + due = append(due, destName) } } - if len(dueDests) == 0 { - return false - } - if !s.maybeRunIndex(ctx, vol, volumeID, "pre-sync", 0) { - return true - } + return due +} + +// dispatchableSyncDests filters dueDests to the pairs the dispatcher can start +// now, dropping any whose sync is already in flight and logging the same +// per-pair scheduler.skipped the serial scheduler emitted — so an operator +// still sees why a due destination isn't moving, it just no longer drags a +// redundant pre-sync index along with it. +func (s *scheduler) dispatchableSyncDests(ctx context.Context, vol *config.Volume, volumeID int64, dueDests []string) []string { + var dispatchable []string for _, destName := range dueDests { - s.runSync(ctx, vol, volumeID, destName) + if s.syncInFlight(ctx, vol, volumeID, destName) { + s.logger.Info("scheduler.skipped", + "kind", "sync", "volume", vol.Name, "destination", destName, + "reason", "in-flight sync run") + continue + } + dispatchable = append(dispatchable, destName) } - return true + return dispatchable } // syncDue computes (now - last_finished) ≥ cadence for the (volume, @@ -463,34 +508,28 @@ func indexRunStatus(rep index.Report, err error) (string, bool) { return store.RunStatusSuccess, true } -// runSync evaluates the per-pair in-flight gate, then hands the sync to -// the per-destination dispatcher (#160). The DB check catches a sync of -// this exact pair already running — a concurrent CLI `squirrel sync`, or a -// stale row not yet reaped — producing the same clean per-pair skip the -// serial scheduler did; the dispatcher additionally serialises different -// volumes that target the same destination and confines a slow transfer to -// its own worker. The SyncRunner's downstream call (sync.RunPair) adds its -// own atomic BeginSyncRunIfClear gate, so a race that sneaks past the -// pre-check still produces a clean skip rather than a duplicate run. -// -// Dispatch is non-blocking: a slow or wedged destination is confined to its -// own worker and never stalls the tick loop or syncs to other destinations, -// which is the freeze F25 recorded. A nil SyncRunner surfaces as a clean -// skip inside the dispatcher so an agent running pure index-only schedules -// can omit the sync wiring without logging at error level each tick. -func (s *scheduler) runSync(ctx context.Context, vol *config.Volume, volumeID int64, destName string) { +// syncInFlight reports whether a sync of this exact (volume, destination) pair +// is already active — a stale row, a concurrent CLI `squirrel sync`, or (the +// common case) a dispatch from an earlier tick still working. It consults two +// sources so neither a not-yet-visible run row nor a just-drained queue slot +// can fool it: the dispatcher's in-memory queue (set the instant a pair is +// enqueued, before its run row exists) and the runs table (authoritative once +// the row is written, and the signal that also catches a stale row or a +// concurrent CLI sync). The downstream dispatch and sync.RunPair's atomic +// BeginSyncRunIfClear gate remain the final guard against a duplicate run, so +// this check only needs to be good enough to suppress a redundant pre-sync +// index. A store error is treated as in-flight so a transient DB hiccup skips +// this tick rather than racing a duplicate push. +func (s *scheduler) syncInFlight(ctx context.Context, vol *config.Volume, volumeID int64, destName string) bool { + if s.dispatch.inFlight(destName, vol.Name) { + return true + } running, err := s.store.HasRunningRun(ctx, store.RunKindSync, volumeID, destName) if err != nil { s.logger.Error("scheduler.error", "kind", "sync", "volume", vol.Name, "destination", destName, "err", err.Error()) - return - } - if running { - s.logger.Info("scheduler.skipped", - "kind", "sync", "volume", vol.Name, "destination", destName, - "reason", "in-flight sync run") - return + return true } - s.dispatch.dispatch(ctx, vol, destName) + return running } diff --git a/agent/scheduler_test.go b/agent/scheduler_test.go index ff46705..617a9a9 100644 --- a/agent/scheduler_test.go +++ b/agent/scheduler_test.go @@ -416,9 +416,12 @@ func TestSchedulerSkipsWhenInFlightIndexRun(t *testing.T) { } } -// TestSchedulerSkipsWhenInFlightSyncRun plants a stale running sync -// row and asserts the scheduler skips the sync (but, per the -// invariant, the pre-sync index still ran in this tick). We use a +// TestSchedulerSkipsWhenInFlightSyncRun plants a stale running sync row and +// asserts the scheduler skips the sync — and, since that pair is the volume's +// only due destination, skips the pre-sync index too (#160). Re-indexing the +// source on every tick for the whole duration of an in-flight push would burn +// I/O and flood the never-pruned runs audit trail with a kind='index' row per +// tick; the in-flight push was already preceded by its own index. We use a // fake sync runner so we can verify it never gets invoked. func TestSchedulerSkipsWhenInFlightSyncRun(t *testing.T) { f := newSchedulerFixture(t, &config.Volume{ @@ -449,25 +452,12 @@ func TestSchedulerSkipsWhenInFlightSyncRun(t *testing.T) { if !f.containsLogLine("scheduler.skipped", "kind=sync", "in-flight") { t.Fatalf("missing in-flight sync skip log:\n%s", f.logBuf.String()) } - // Pre-sync index still ran — the invariant says sync always - // indexes first, and the only thing skipped here is the sync - // itself. + // No new pre-sync index: with the sole due destination already in + // flight, there is no new push for an index to precede. Only the planted + // 'running' sync row should exist. runs, _ := f.store.ListRuns(context.Background(), store.ListRunsOpts{}) - var sawIndex, sawSync bool - for _, r := range runs { - switch r.Kind { - case store.RunKindIndex: - sawIndex = true - case store.RunKindSync: - sawSync = true - } - } - if !sawIndex { - t.Fatalf("expected a pre-sync index row even when sync is gated: %+v", runs) - } - if !sawSync { - // The planted 'running' sync row counts as a sync; just sanity. - t.Fatalf("expected the planted running sync row to remain: %+v", runs) + if len(runs) != 1 || runs[0].Kind != store.RunKindSync || runs[0].Status != store.RunStatusRunning { + t.Fatalf("runs=%+v; want only the planted running sync row (no redundant pre-sync index)", runs) } } @@ -579,6 +569,151 @@ func TestSchedulerCadenceUsesLastFinished(t *testing.T) { } } +// waitUntil polls cond until it returns true or a short deadline passes, +// failing the test on timeout. It synchronises with the dispatcher's async +// workers without a fixed sleep, so a test can wait for one destination's +// queue to drain while another destination's sync is still (deliberately) +// blocked. +func waitUntil(t *testing.T, cond func() bool, what string) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +// TestSchedulerInFlightSyncNotReindexedEachTick is the #160 regression guard: +// once a sync is dispatched and stays in flight across ticks, the scheduler +// must NOT re-run that volume's pre-sync index every tick — otherwise a slow +// multi-hour push would flood the never-pruned runs audit trail with a +// kind='index' row per 30s tick. The pre-sync index runs exactly once (on the +// tick that dispatches the sync), while a different, not-in-flight +// (volume, destination) keeps indexing and dispatching normally on the very +// ticks the wedged one is skipped — proving the guard is per-pair, not global. +func TestSchedulerInFlightSyncNotReindexedEachTick(t *testing.T) { + f := newSchedulerFixture(t, + &config.Volume{Name: "photos", SyncTo: []string{"cloudbox"}, SyncEvery: time.Hour}, + &config.Volume{Name: "docs", SyncTo: []string{"backup"}, SyncEvery: time.Hour}, + ) + f.seedFile() + + release := make(chan struct{}) + photosInFlight := make(chan struct{}, 1) + docsFinished := make(chan struct{}, 8) + f.syncLog.runFn = func(ctx context.Context, vol *config.Volume, destName string) SyncRunReport { + v, err := f.store.GetVolumeByName(ctx, vol.Name) + if err != nil { + return SyncRunReport{Err: err} + } + id, blocker, err := f.store.BeginSyncRunIfClear(ctx, store.SyncRunSpec{ + VolumeID: v.ID, Destination: destName, + }) + if err != nil { + return SyncRunReport{Err: err} + } + if blocker != nil { + return SyncRunReport{RunID: blocker.ID, Err: fmt.Errorf("already running")} + } + if vol.Name == "photos" { + // Announce the running row exists, then block — keeping the + // photos->cloudbox sync in flight across the following ticks + // without ever finishing it (the F25 wedge, minus the kill). + photosInFlight <- struct{}{} + <-release + } + if err := f.store.FinishRun(ctx, id, store.RunStatusSuccess, "", 0); err != nil { + return SyncRunReport{RunID: id, Err: err} + } + if vol.Name == "docs" { + docsFinished <- struct{}{} + } + return SyncRunReport{RunID: id, Status: store.RunStatusSuccess} + } + + sch := f.scheduler() + ctx := context.Background() + + countIndex := func(volName string) int { + t.Helper() + v, err := f.store.GetVolumeByName(ctx, volName) + if err != nil { + t.Fatalf("GetVolumeByName(%s): %v", volName, err) + } + runs, err := f.store.ListRuns(ctx, store.ListRunsOpts{}) + if err != nil { + t.Fatalf("ListRuns: %v", err) + } + n := 0 + for _, r := range runs { + if r.Kind == store.RunKindIndex && r.VolumeID.Valid && r.VolumeID.Int64 == v.ID { + n++ + } + } + return n + } + // drainDocs waits for docs' just-dispatched sync to finish and its queue + // slot to clear, so the next tick sees docs as not-in-flight while photos + // stays wedged. + drainDocs := func(tick int) { + <-docsFinished + waitUntil(t, func() bool { return !sch.dispatch.inFlight("backup", "docs") }, + fmt.Sprintf("docs queue to drain after tick %d", tick)) + } + + // Tick 1: both due, neither in flight -> each indexes once and dispatches. + // photos' sync blocks (stays running); docs' sync finishes. + sch.tick(ctx) + <-photosInFlight + drainDocs(1) + if got := countIndex("photos"); got != 1 { + t.Fatalf("after tick 1: photos index runs = %d; want 1", got) + } + if got := countIndex("docs"); got != 1 { + t.Fatalf("after tick 1: docs index runs = %d; want 1", got) + } + + // Ticks 2 and 3: photos is still in flight -> skipped with NO new + // pre-sync index. docs comes due again each hour and must keep indexing + + // dispatching normally on the very ticks photos is skipped. + for tick := 2; tick <= 3; tick++ { + f.clock.Add(time.Hour + time.Minute) + sch.tick(ctx) + drainDocs(tick) + if got := countIndex("photos"); got != 1 { + t.Fatalf("after tick %d: photos index runs = %d; want 1 (in-flight sync must not be re-indexed)", tick, got) + } + if got := countIndex("docs"); got != tick { + t.Fatalf("after tick %d: docs index runs = %d; want %d (not-in-flight volume indexes each due tick)", tick, got, tick) + } + } + + var photosCalls, docsCalls int + for _, c := range f.syncLog.Calls() { + switch c.Volume { + case "photos": + photosCalls++ + case "docs": + docsCalls++ + } + } + if photosCalls != 1 { + t.Fatalf("photos sync calls = %d; want 1 (dispatched once, then skipped while in flight)", photosCalls) + } + if docsCalls != 3 { + t.Fatalf("docs sync calls = %d; want 3 (once per due tick)", docsCalls) + } + if !f.containsLogLine("scheduler.skipped", "kind=sync", "volume=photos", "in-flight") { + t.Fatalf("missing in-flight skip log for photos:\n%s", f.logBuf.String()) + } + + close(release) + sch.dispatch.wait() +} + // TestSchedulerFailedSyncStillConsumesCadence guards the failure // policy ("Failed runs aren't specially retried — the next tick // re-evaluates"). After a failed sync, the next within-cadence tick