Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions agent/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -1556,7 +1556,8 @@ func (r *peerSyncRouter) handleClose(w http.ResponseWriter, req *http.Request) {
// already failing, and the run row's stuck-'running' state is the worse
// outcome the caller surfaces, so there's nothing to return through.
func (r *peerSyncRouter) finalizeFailedClose(ctx context.Context, runID int64, committed int, cause error) {
err := r.srv.store.FinishRun(ctx, runID, store.RunStatusFailed, cause.Error(), int64(committed))
err := r.srv.store.FinishRunChanged(ctx, runID, store.RunStatusFailed, cause.Error(),
int64(committed), int64(committed))
if err == nil {
return
}
Expand Down Expand Up @@ -1623,7 +1624,10 @@ func (r *peerSyncRouter) closeSession(ctx context.Context, sess *peerSession, st
if finishStatus != store.RunStatusSuccess && finishStatus != store.RunStatusPartial && finishStatus != store.RunStatusFailed {
finishStatus = store.RunStatusPartial
}
if err := r.srv.store.FinishRun(ctx, sess.receiverRunID, finishStatus, "", int64(committed)); err != nil {
// committed is both counts on the receiver side: every row written
// here is a path whose content this session changed locally, so a
// session that committed nothing reads as the no-op it is (#182).
if err := r.srv.store.FinishRunChanged(ctx, sess.receiverRunID, finishStatus, "", int64(committed), int64(committed)); err != nil {
return committed, fmt.Errorf("finish run: %w", err)
}
return committed, nil
Expand Down
4 changes: 3 additions & 1 deletion cmd/squirrel/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,9 @@ func finishDurabilityPullRun(ctx context.Context, s *store.Store, runID int64, v
note := fmt.Sprintf("volume=%s peer=%s fetched=%d applied=%d dropped=%d rewinds=%d",
volume, peer, rep.Fetched, rep.Applied, rep.Dropped, len(rep.Rewinds))
auditErr := s.AppendRunAudit(ctx, store.RunAuditEntry{RunID: runID, Transition: store.TransitionPullDurability, Note: note})
finErr := s.FinishRun(ctx, runID, status, errMsg, int64(rep.Applied))
// Applied is both counts: a pull that merged no durability rows changed
// nothing locally, so it folds out of the steady-state noise (#182).
finErr := s.FinishRunChanged(ctx, runID, status, errMsg, int64(rep.Applied), int64(rep.Applied))
return agent.DurabilityPullReport{
RunID: runID,
Status: status,
Expand Down
29 changes: 22 additions & 7 deletions cmd/squirrel/polish_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,16 @@ func TestRunIsInteresting(t *testing.T) {
{"failed", store.Run{ID: 3, Status: store.RunStatusFailed}, true},
{"refused", store.Run{ID: 4, Status: store.RunStatusRefused}, true},
{"success with conflict", store.Run{ID: 7, Status: store.RunStatusSuccess, FileCount: 0}, true},
// #182: a bucket push counts already-correct files in file_count,
// so only changed_count can tell the in-sync no-op from the push
// that actually moved content.
{"in-sync bucket push", store.Run{ID: 8, Status: store.RunStatusSuccess, FileCount: 42,
ChangedCount: sql.NullInt64{Int64: 0, Valid: true}}, false},
{"bucket push that transferred", store.Run{ID: 9, Status: store.RunStatusSuccess, FileCount: 42,
ChangedCount: sql.NullInt64{Int64: 3, Valid: true}}, true},
// Pre-v28 history has no changed count and keeps the conservative
// file_count rendering.
{"pre-v28 bucket push", store.Run{ID: 10, Status: store.RunStatusSuccess, FileCount: 42}, true},
}
for _, tc := range tests {
if got := runIsInteresting(tc.r, conflicts); got != tc.want {
Expand All @@ -54,20 +64,25 @@ func TestRunIsInteresting(t *testing.T) {
// TestFilterRuns exercises both filters end to end, including that the
// descending order is preserved.
func TestFilterRuns(t *testing.T) {
changed := func(n int64) sql.NullInt64 { return sql.NullInt64{Int64: n, Valid: true} }
runs := []store.Run{
{ID: 5, Status: store.RunStatusSuccess, FileCount: 0}, // no-op
{ID: 4, Status: store.RunStatusSuccess, FileCount: 3}, // change
{ID: 3, Status: store.RunStatusFailed}, // failed
{ID: 2, Status: store.RunStatusRefused}, // refused
{ID: 1, Status: store.RunStatusSuccess, FileCount: 0}, // no-op
// A steady-state bucket push and an all-unchanged re-index: both
// consider the whole volume, both changed nothing (#182).
{ID: 7, Status: store.RunStatusSuccess, FileCount: 42, ChangedCount: changed(0)},
{ID: 6, Status: store.RunStatusSuccess, FileCount: 42, ChangedCount: changed(2)}, // moved content
{ID: 5, Status: store.RunStatusSuccess, FileCount: 0}, // no-op
{ID: 4, Status: store.RunStatusSuccess, FileCount: 3}, // change
{ID: 3, Status: store.RunStatusFailed}, // failed
{ID: 2, Status: store.RunStatusRefused}, // refused
{ID: 1, Status: store.RunStatusSuccess, FileCount: 0}, // no-op
}
conflicts := map[int64]int{}

if ids := runIDs(filterRuns(runs, conflicts, true, false)); !sameIDs(ids, []int64{3, 2}) {
t.Fatalf("--failed ids = %v, want [3 2]", ids)
}
if ids := runIDs(filterRuns(runs, conflicts, false, true)); !sameIDs(ids, []int64{4, 3, 2}) {
t.Fatalf("--changes ids = %v, want [4 3 2]", ids)
if ids := runIDs(filterRuns(runs, conflicts, false, true)); !sameIDs(ids, []int64{6, 4, 3, 2}) {
t.Fatalf("--changes ids = %v, want [6 4 3 2]", ids)
}
}

Expand Down
28 changes: 11 additions & 17 deletions cmd/squirrel/runs.go
Original file line number Diff line number Diff line change
Expand Up @@ -194,16 +194,16 @@ func gatherRuns(cmd *cobra.Command, s *store.Store, volumeID *int64, limit int,

// conflictCountsForFilter loads only the conflict counts the active filter
// needs. --failed consults none — an attention-worthy run is unconditionally
// kept — so it returns an empty map. --changes needs them only for the
// ambiguous rows (a clean success that touched no files), so it counts just
// those rather than the whole unbounded fetch.
// kept — so it returns an empty map. --changes needs them only for the rows
// that would otherwise be hidden as no-ops, so it counts just those rather
// than the whole unbounded fetch.
func conflictCountsForFilter(cmd *cobra.Command, s *store.Store, runs []store.Run, onlyFailed bool) (map[int64]int, error) {
if onlyFailed {
return map[int64]int{}, nil
}
var candidates []store.Run
for _, r := range runs {
if r.Status == store.RunStatusSuccess && r.FileCount == 0 {
if r.NoOp() {
candidates = append(candidates, r)
}
}
Expand Down Expand Up @@ -238,22 +238,16 @@ func runNeedsAttention(status string) bool {
}

// runIsInteresting reports whether a run is worth showing under --changes:
// anything needing attention, still running, that touched files, or that
// resolved conflicts. A clean success that touched nothing is the routine
// anything needing attention, still running, that changed files, or that
// resolved conflicts. A clean success that changed nothing is the routine
// no-op the filter hides.
//
// runs.file_count conflates transferred with already-correct for bucket and
// index runs (it is the total files considered, not the delta), so an
// in-sync bucket sync or an all-unchanged re-index still shows here — the
// filter deliberately errs toward showing rather than risk hiding a run
// that moved content. It reliably folds peer-sync no-ops, whose file_count
// is the receiver-verified (i.e. transferred) count and so is zero when
// nothing moved.
// store.Run.NoOp owns the "changed nothing" rule so this filter and the
// TUI's `f` fold hide exactly the same rows: the run's recorded
// changed_count where it has one, and the conservative file_count
// heuristic for the pre-v28 history and the drivers that report no count.
func runIsInteresting(r store.Run, conflicts map[int64]int) bool {
if r.Status != store.RunStatusSuccess {
return true
}
if r.FileCount > 0 {
if !r.NoOp() {
return true
}
return conflicts[r.ID] > 0
Expand Down
10 changes: 9 additions & 1 deletion design/friction-log.md
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ On the nas, `sync photos laptop` (received from laptop) and
the TUI — the destination column holds a peer name in both directions.
The audit trail answers "what happened" but not "who initiated".

**F19 · S3 — steady-state run noise buries signal.** With compressed
**F19 · S3 — ~~steady-state run noise buries signal.~~ (filters + fold, #179; `runs.changed_count`, #182)** With compressed
cadences the runs list is dominated by 0-file no-op rows (one per
volume × destination per tick). Nothing distinguishes "checked,
nothing to do" from "transferred nothing unexpectedly"; a day of real
Expand All @@ -211,6 +211,14 @@ TUI-side folding of consecutive no-ops would restore the audit trail's
readability. (The `runs` help text also still says "List index runs";
it lists every kind.)

#179 landed the filters and the fold, but both keyed on
`file_count == 0`, which only ever collapsed peer-sync no-ops — a bucket
push or an index run counts what it *considered*, so the bulk of the
noise stayed on screen. #182 added `runs.changed_count`, populated by
each driver from what it actually moved, and both surfaces now key on
that (falling back to the old heuristic for pre-v28 history, which stays
truthfully "unknown" rather than silently folding).

**F20 · S2 — ~~recovering a wrecked destination has no supported path.~~ (`squirrel destination reset` + empty-root guard, #176)**
After the F12 bug era the packed-layout guard refused every further
s3archive sync ("its history is not packed … point the layout at a
Expand Down
9 changes: 8 additions & 1 deletion index/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -293,14 +293,21 @@ func (i *indexer) beginRun() error {
// and the per-file error count: a fatal failure yields 'failed', per-file
// errors with a clean walk yield 'partial', and an entirely clean run yields
// 'success'.
//
// file_count is everything the walk considered; changed_count narrows that
// to the observations this run actually recorded — a new path, a path whose
// content was superseded, or a path that went missing. An all-unchanged
// re-index therefore reads as the no-op it is, instead of reporting the
// whole volume (#182).
func (i *indexer) finishRun(report *Report, fatalErr error) {
report.RunID = i.runID
if i.opts.DryRun || i.runID == 0 {
return
}
status, errMsg := runStatus(report, fatalErr)
fileCount := int64(report.Added + report.Modified + report.Unchanged)
if err := i.store.FinishRun(i.ctx, i.runID, status, errMsg, fileCount); err != nil {
changed := int64(report.Added + report.Modified + report.Missing)
if err := i.store.FinishRunChanged(i.ctx, i.runID, status, errMsg, fileCount, changed); err != nil {
// Surface as a per-run error rather than swallowing silently. The
// outer caller has already accepted a report and a fatal error (if
// any); we can only append.
Expand Down
56 changes: 56 additions & 0 deletions index/index_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,62 @@ func TestIndexRecordsRuns(t *testing.T) {
}
}

// TestIndexRecordsChangedCount pins the v28 column for index runs (#182):
// the first walk added a file and the second changed nothing, so the two
// rows carry the same file_count but different changed counts — and only
// the second folds out of `runs --changes` and the TUI's fold. A third walk
// after an edit and a delete counts both as changes.
func TestIndexRecordsChangedCount(t *testing.T) {
root := t.TempDir()
writeFile(t, filepath.Join(root, "a.txt"), "hello")
writeFile(t, filepath.Join(root, "b.txt"), "world")

s := setupStore(t)
ctx := context.Background()
if _, err := Index(ctx, s, root, Options{}); err != nil {
t.Fatalf("first Index: %v", err)
}
if _, err := Index(ctx, s, root, Options{}); err != nil {
t.Fatalf("second Index: %v", err)
}
writeFile(t, filepath.Join(root, "a.txt"), "hello again")
if err := os.Remove(filepath.Join(root, "b.txt")); err != nil {
t.Fatalf("remove b.txt: %v", err)
}
if _, err := Index(ctx, s, root, Options{}); err != nil {
t.Fatalf("third Index: %v", err)
}

absRoot, _ := filepath.Abs(root)
vol := volumeFor(t, s, absRoot)
runs, err := s.ListRuns(ctx, store.ListRunsOpts{VolumeID: &vol.ID})
if err != nil {
t.Fatalf("ListRuns: %v", err)
}
if len(runs) != 3 {
t.Fatalf("got %d run rows, want 3: %+v", len(runs), runs)
}
// added a.txt + b.txt, nothing, then a.txt modified + b.txt missing.
wantChanged := []int64{2, 0, 2}
for i, r := range runs {
if !r.ChangedCount.Valid {
t.Fatalf("run %d changed_count is NULL, want %d", i, wantChanged[i])
}
if r.ChangedCount.Int64 != wantChanged[i] {
t.Errorf("run %d changed_count = %d, want %d", i, r.ChangedCount.Int64, wantChanged[i])
}
}
if runs[0].NoOp() || !runs[1].NoOp() || runs[2].NoOp() {
t.Errorf("NoOp() = %v, want [false true false]",
[]bool{runs[0].NoOp(), runs[1].NoOp(), runs[2].NoOp()})
}
// The all-unchanged walk still considered every file — the point of
// the new column is that file_count alone couldn't say it was a no-op.
if runs[1].FileCount != 2 {
t.Errorf("no-op run file_count = %d, want 2 (files considered)", runs[1].FileCount)
}
}

// TestIndexRecordsShallowFlag pins that the runs row carries the
// Options.Shallow value the call was made with: shallow=true for the
// fast (size, mtime) path and shallow=false for the rehash-everything
Expand Down
9 changes: 6 additions & 3 deletions offload/offload.go
Original file line number Diff line number Diff line change
Expand Up @@ -372,8 +372,10 @@ func beginRun(ctx context.Context, s *store.Store, volumeID int64, volumeName st
// the fatal error, 'partial' when any per-file operation errored, and
// 'success' otherwise (skipped files are decisions, so a run that only
// skipped is still a success). file_count records how many files were
// actually offloaded. A failed terminal write is surfaced on the report
// — the per-file flips already committed individually and stand.
// actually offloaded, and so does changed_count — an offloaded row is a
// files-row flip, and a run that only skipped changed nothing (#182). A
// failed terminal write is surfaced on the report — the per-file flips
// already committed individually and stand.
func finishRun(ctx context.Context, s *store.Store, runID int64, report *Report, fatalErr error) {
status, errMsg := store.RunStatusSuccess, ""
switch {
Expand All @@ -382,7 +384,8 @@ func finishRun(ctx context.Context, s *store.Store, runID int64, report *Report,
case report.Errors > 0:
status = store.RunStatusPartial
}
if err := s.FinishRun(ctx, runID, status, errMsg, int64(report.Offloaded)); err != nil {
offloaded := int64(report.Offloaded)
if err := s.FinishRunChanged(ctx, runID, status, errMsg, offloaded, offloaded); err != nil {
report.FinishErr = fmt.Errorf("finish offload run %d: %w", runID, err)
}
}
Expand Down
6 changes: 4 additions & 2 deletions store/migrate_v27_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,10 @@ func TestMigrateV26ToV27PreservesEveryRow(t *testing.T) {
}
want := before
if table == "schema_version" {
// The migration records v27 in the table it just rebuilt.
want++
// The v27 migration records its own version in the table it
// just rebuilt, and Open carries on to SchemaVersion — one
// row per step from the v26 fixture.
want += int64(SchemaVersion - 26)
}
if after := countsAfter[table]; after != want {
t.Errorf("%s rows = %d after migration, want %d", table, after, want)
Expand Down
105 changes: 105 additions & 0 deletions store/migrate_v28_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package store

import (
"context"
"database/sql"
"path/filepath"
"testing"
)

// TestMigrateToV28LeavesHistoryUnknown builds a pre-v28 database holding
// the two run shapes #182 is about — a bucket push whose file_count counts
// the whole volume, and a peer sync whose file_count is already the
// transferred count — and migrates it forward.
//
// The point of the nullable column is that history stays truthful: neither
// row gains a fabricated zero, both read back as "unknown", and Run.NoOp
// therefore keeps rendering them exactly as it did before the migration.
func TestMigrateToV28LeavesHistoryUnknown(t *testing.T) {
dsn := filepath.Join(t.TempDir(), "test.db")
rawDB, err := sql.Open("sqlite", dsn)
if err != nil {
t.Fatalf("raw sql.Open: %v", err)
}
ddl := []string{
`CREATE TABLE schema_version (version INTEGER NOT NULL PRIMARY KEY)`,
`CREATE TABLE volumes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, path TEXT NOT NULL)`,
`CREATE TABLE nodes (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, endpoint TEXT, public_key_fingerprint TEXT)`,
minimalRunsDDL,
// Present since v11, so a v24 database has it; the fixture needs
// it because FinishRun writes the run's transition here.
`CREATE TABLE runs_audit (
id INTEGER PRIMARY KEY,
run_id INTEGER NOT NULL REFERENCES runs(id),
transition TEXT NOT NULL,
operator TEXT,
at_ns INTEGER NOT NULL,
note TEXT
)`,
`INSERT INTO schema_version (version) VALUES (24)`,
`INSERT INTO volumes (id, name, path) VALUES (1, 'v', '/v')`,
`INSERT INTO nodes (id, name) VALUES (1, 'self')`,
// An in-sync bucket push: nothing moved, but file_count counted
// every already-correct file.
`INSERT INTO runs (id, kind, volume_id, destination, started_at_ns, ended_at_ns, status, file_count)
VALUES (1, 'sync', 1, 'bucket', 100, 200, 'success', 42)`,
// A peer-sync no-op: file_count is the receiver-verified count, so
// it was already zero.
`INSERT INTO runs (id, kind, volume_id, destination, started_at_ns, ended_at_ns, status, file_count)
VALUES (2, 'sync', 1, 'htpc', 110, 210, 'success', 0)`,
}
for _, q := range ddl {
if _, err := rawDB.Exec(q); err != nil {
t.Fatalf("pre-v28 DDL %q: %v", q, err)
}
}
rawDB.Close()

s, err := OpenWithOptions(dsn, OpenOptions{DisablePreMigrationBackup: true})
if err != nil {
t.Fatalf("Open (migrates to v%d): %v", SchemaVersion, err)
}
defer s.Close()
ctx := context.Background()
if v, _ := s.CurrentSchemaVersion(ctx); v != SchemaVersion {
t.Fatalf("schema_version = %d, want %d", v, SchemaVersion)
}

runs, err := s.ListRuns(ctx, ListRunsOpts{})
if err != nil {
t.Fatalf("ListRuns: %v", err)
}
if len(runs) != 2 {
t.Fatalf("runs after migration = %d, want 2", len(runs))
}
for _, r := range runs {
if r.ChangedCount.Valid {
t.Errorf("run %d changed_count = %+v, want NULL — history must not gain a fabricated count",
r.ID, r.ChangedCount)
}
}
// The fallback keeps each row's pre-migration rendering: the bucket
// push stays visible (conservative), the peer-sync no-op still folds.
if runs[0].NoOp() {
t.Error("pre-v28 bucket push should stay visible, not fold on a guess")
}
if !runs[1].NoOp() {
t.Error("pre-v28 peer-sync no-op should still fold")
}

// New runs on the migrated database record the real count.
runID, err := s.BeginRun(ctx, RunKindSync, 1, "bucket", false)
if err != nil {
t.Fatalf("BeginRun: %v", err)
}
if err := s.FinishRunChanged(ctx, runID, RunStatusSuccess, "", 42, 0); err != nil {
t.Fatalf("FinishRunChanged: %v", err)
}
fresh, err := s.GetRun(ctx, runID)
if err != nil {
t.Fatalf("GetRun: %v", err)
}
if !fresh.NoOp() {
t.Errorf("in-sync bucket push should fold after v28: %+v", fresh.ChangedCount)
}
}
Loading
Loading