From f29df8e6bb042d60ed2ef292f8f37605ffadec46 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:39:08 +0000 Subject: [PATCH 1/6] store: audit-preserving destination reset (F20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ResetDestination: an operator-driven verb that forgets a destination's recorded remote state (remote_objects, remote_packs, the durability vector, and push-freshness) so the next sync treats it as fresh and re-uploads. It is the supported recovery for a wrecked or repointed destination, replacing hand SQL or a cross-machine rename. The reset is audit-preserving: the runs table and the append-only destination_run_ids_history advance log are untouched, and the content/files rows are never modified. The reset itself is recorded as a kind='audit' run (reusing the destination-scoped audit shape, so no schema change) with a new free-text 'reset-destination' runs_audit transition carrying the cleared counts. Everything runs in one transaction — a reset is all-or-nothing. CountDestinationRecordedState backs the CLI's dry-run preview. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- store/destination_reset.go | 177 ++++++++++++++++++++++++++++ store/destination_reset_test.go | 201 ++++++++++++++++++++++++++++++++ store/runs_audit.go | 7 ++ 3 files changed, 385 insertions(+) create mode 100644 store/destination_reset.go create mode 100644 store/destination_reset_test.go diff --git a/store/destination_reset.go b/store/destination_reset.go new file mode 100644 index 0000000..198f731 --- /dev/null +++ b/store/destination_reset.go @@ -0,0 +1,177 @@ +package store + +import ( + "context" + "database/sql" + "fmt" +) + +// DestinationResetCounts reports how many recorded-state rows a destination +// reset cleared (or, on a dry run, would clear). The four categories are the +// full derived record squirrel keeps about a destination: the per-content +// and per-pack upload ledgers, the live durability vector, and the +// push-freshness maxima. They span every volume — a destination's recorded +// state is not volume-scoped — so the counts are totals across the index. +type DestinationResetCounts struct { + RemoteObjects int64 + RemotePacks int64 + VectorComponents int64 + FreshnessRows int64 +} + +// Total is the sum of all four categories, used as the reset run's +// file_count so the audit trail carries the magnitude at a glance. +func (c DestinationResetCounts) Total() int64 { + return c.RemoteObjects + c.RemotePacks + c.VectorComponents + c.FreshnessRows +} + +// Empty reports whether the destination has no recorded state at all — the +// "nothing to reset" case the CLI reports affirmatively rather than minting +// an empty run. +func (c DestinationResetCounts) Empty() bool { return c.Total() == 0 } + +// rowQuerier is the subset of *sql.DB and *sql.Tx that +// countDestinationRecordedState needs, so the same count runs read-only on +// the DB (the dry-run preview) and inside the reset transaction (the note +// the run records). +type rowQuerier interface { + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// CountDestinationRecordedState returns how much derived state the index +// holds for destination, without changing anything. It is the read-only +// question behind `squirrel destination reset --dry-run`. +func (s *Store) CountDestinationRecordedState(ctx context.Context, destination string) (DestinationResetCounts, error) { + if destination == "" { + return DestinationResetCounts{}, fmt.Errorf("CountDestinationRecordedState: destination must be non-empty") + } + return countDestinationRecordedState(ctx, s.db, destination) +} + +func countDestinationRecordedState(ctx context.Context, q rowQuerier, destination string) (DestinationResetCounts, error) { + var c DestinationResetCounts + counts := []struct { + dst *int64 + query string + }{ + {&c.RemoteObjects, `SELECT COUNT(*) FROM remote_objects WHERE destination = ?`}, + {&c.RemotePacks, `SELECT COUNT(*) FROM remote_packs WHERE destination = ?`}, + {&c.VectorComponents, `SELECT COUNT(*) FROM destination_run_ids WHERE destination = ?`}, + {&c.FreshnessRows, `SELECT COUNT(*) FROM destination_push_freshness WHERE destination = ?`}, + } + for _, item := range counts { + if err := q.QueryRowContext(ctx, item.query, destination).Scan(item.dst); err != nil { + return DestinationResetCounts{}, fmt.Errorf("count destination %q recorded state: %w", destination, err) + } + } + return c, nil +} + +// ResetDestination forgets everything the index records about a destination's +// remote state — its per-content and per-pack upload ledgers, its live +// durability vector, and its push-freshness maxima — so the next sync treats +// the destination as fresh and re-uploads. It is the supported recovery for a +// wrecked or repointed destination (friction log F20): before it, a fresh +// `root` still refused because the layout guard keyed on run history by +// destination name, leaving hand SQL or a config-wide rename as the only outs. +// +// The reset is audit-preserving. It never touches the runs table or the +// append-only destination_run_ids_history advance log, so the record of what +// was ever asserted durable survives; only the live derived state is cleared, +// exactly as RevokeDestinationRunIDsFromSource clears a peer's live +// assertions while leaving the history intact. The contents and files rows — +// the content squirrel must never lose track of — are untouched. +// +// The reset is itself recorded as a run: a kind='audit' row (the run kinds +// are fixed by a CHECK, so this reuses the destination-scoped audit shape +// BeginRemoteVerifyRun already uses — volume_id and destination NULL) whose +// runs_audit trail carries a 'reset-destination' note with the destination +// name and the cleared counts. Everything runs in one transaction: on any +// error nothing is cleared and no run is recorded, so a reset is all-or-nothing. +func (s *Store) ResetDestination(ctx context.Context, destination string) (int64, DestinationResetCounts, error) { + if destination == "" { + return 0, DestinationResetCounts{}, fmt.Errorf("ResetDestination: destination must be non-empty") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, DestinationResetCounts{}, fmt.Errorf("begin reset destination %q: %w", destination, err) + } + defer func() { _ = tx.Rollback() }() + + counts, err := countDestinationRecordedState(ctx, tx, destination) + if err != nil { + return 0, DestinationResetCounts{}, err + } + atNs := NowNs() + runID, err := insertResetRunTx(ctx, tx, atNs) + if err != nil { + return 0, DestinationResetCounts{}, err + } + if err := deleteDestinationRecordedStateTx(ctx, tx, destination); err != nil { + return 0, DestinationResetCounts{}, err + } + note := fmt.Sprintf("destination=%q objects=%d packs=%d vector=%d freshness=%d", + destination, counts.RemoteObjects, counts.RemotePacks, counts.VectorComponents, counts.FreshnessRows) + if err := appendRunAuditTx(ctx, tx, + RunAuditEntry{RunID: runID, Transition: TransitionResetDestination, Note: note}, atNs); err != nil { + return 0, DestinationResetCounts{}, err + } + if err := finishResetRunTx(ctx, tx, runID, atNs, counts.Total()); err != nil { + return 0, DestinationResetCounts{}, err + } + if err := tx.Commit(); err != nil { + return 0, DestinationResetCounts{}, fmt.Errorf("commit reset destination %q: %w", destination, err) + } + return runID, counts, nil +} + +// insertResetRunTx opens the kind='audit' run that records the reset. The +// runs CHECK keeps audit rows' volume_id and destination NULL (the destination +// travels in the 'reset-destination' runs_audit note instead), matching +// BeginRemoteVerifyRun's destination-scoped audit shape. +func insertResetRunTx(ctx context.Context, tx *sql.Tx, atNs int64) (int64, error) { + res, err := tx.ExecContext(ctx, ` + INSERT INTO runs (kind, volume_id, destination, started_at_ns, status, file_count) + VALUES ('audit', NULL, NULL, ?, 'running', 0) + `, atNs) + if err != nil { + return 0, fmt.Errorf("insert reset run: %w", err) + } + id, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("reset run last insert id: %w", err) + } + return id, nil +} + +// deleteDestinationRecordedStateTx clears the four derived-state tables for +// destination. The append-only destination_run_ids_history and the runs +// table are deliberately absent — the reset preserves the audit trail. +func deleteDestinationRecordedStateTx(ctx context.Context, tx *sql.Tx, destination string) error { + stmts := []string{ + `DELETE FROM remote_objects WHERE destination = ?`, + `DELETE FROM remote_packs WHERE destination = ?`, + `DELETE FROM destination_run_ids WHERE destination = ?`, + `DELETE FROM destination_push_freshness WHERE destination = ?`, + } + for _, stmt := range stmts { + if _, err := tx.ExecContext(ctx, stmt, destination); err != nil { + return fmt.Errorf("clear destination %q recorded state: %w", destination, err) + } + } + return nil +} + +// finishResetRunTx moves the reset run to success with the cleared total as +// its file_count and appends the 'finish' runs_audit line, mirroring +// FinishRun so the reset run reads like any other terminal run. +func finishResetRunTx(ctx context.Context, tx *sql.Tx, runID, atNs, total int64) error { + if _, err := tx.ExecContext(ctx, ` + UPDATE runs SET ended_at_ns = ?, status = 'success', file_count = ? + WHERE id = ? + `, atNs, total, runID); err != nil { + return fmt.Errorf("finish reset run %d: %w", runID, err) + } + return appendRunAuditTx(ctx, tx, + RunAuditEntry{RunID: runID, Transition: TransitionFinish, Note: RunStatusSuccess}, atNs) +} diff --git a/store/destination_reset_test.go b/store/destination_reset_test.go new file mode 100644 index 0000000..f3223a6 --- /dev/null +++ b/store/destination_reset_test.go @@ -0,0 +1,201 @@ +package store + +import ( + "context" + "testing" +) + +// seedDestinationState populates every category of derived state for +// destination on volume vID, plus one durability advance for a second +// destination, and returns the self node id. It leaves a runs row (the +// index run) and a destination_run_ids_history row behind so the +// audit-preservation assertions have something to check against. +func seedDestinationState(t *testing.T, s *Store, vID, runID int64, destination string) int64 { + t.Helper() + ctx := context.Background() + self, err := s.GetSelfNode(ctx) + if err != nil { + t.Fatalf("GetSelfNode: %v", err) + } + + objContent := packContentFixture(t, s, vID, runID, "obj.txt", 0xa1) + if err := s.InsertRemoteObject(ctx, RemoteObject{ + ContentID: objContent, Destination: destination, UploadedRunID: runID, + }); err != nil { + t.Fatalf("InsertRemoteObject: %v", err) + } + + packContent := packContentFixture(t, s, vID, runID, "packed.txt", 0xa2) + if err := s.InsertPacks(ctx, []PackWrite{{ + Pack: Pack{PackKey: packKey(0x11), SizeBytes: 10, MemberCount: 1, CreatedRunID: runID}, + Members: []PackMember{{ContentID: packContent, ByteOffset: 0, ByteLength: 1}}, + }}); err != nil { + t.Fatalf("InsertPacks: %v", err) + } + pack, err := s.GetPackByKey(ctx, packKey(0x11)) + if err != nil { + t.Fatalf("GetPackByKey: %v", err) + } + if err := s.InsertRemotePack(ctx, RemotePack{ + PackID: pack.ID, Destination: destination, UploadedRunID: runID, + }); err != nil { + t.Fatalf("InsertRemotePack: %v", err) + } + + if err := s.UpsertDestinationRunIDVerified(ctx, vID, destination, self.ID, 5, VerifyMethodBlake3, false); err != nil { + t.Fatalf("UpsertDestinationRunIDVerified: %v", err) + } + if err := s.UpsertDestinationPushFreshness(ctx, vID, destination, self.ID, 5); err != nil { + t.Fatalf("UpsertDestinationPushFreshness: %v", err) + } + // A second destination whose state must survive a reset of the first. + if err := s.UpsertDestinationRunIDVerified(ctx, vID, "other", self.ID, 7, VerifyMethodBlake3, false); err != nil { + t.Fatalf("UpsertDestinationRunIDVerified(other): %v", err) + } + return self.ID +} + +// TestResetDestinationClearsDerivedStateAuditPreservingly is the F20 core: +// reset forgets a destination's upload ledgers, vector, and freshness while +// leaving the runs table, the append-only advance log, other destinations, +// and the content rows intact — and records itself as an audit run. +func TestResetDestinationClearsDerivedStateAuditPreservingly(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + vID := makeVolume(t, s, "/v") + runID := makeRun(t, s, vID) + self := seedDestinationState(t, s, vID, runID, "arch") + + counts, err := s.CountDestinationRecordedState(ctx, "arch") + if err != nil { + t.Fatalf("CountDestinationRecordedState: %v", err) + } + if counts.RemoteObjects != 1 || counts.RemotePacks != 1 || counts.VectorComponents != 1 || counts.FreshnessRows != 1 { + t.Fatalf("pre-reset counts = %+v, want one of each", counts) + } + + _, cleared, err := s.ResetDestination(ctx, "arch") + if err != nil { + t.Fatalf("ResetDestination: %v", err) + } + if cleared != counts { + t.Fatalf("cleared = %+v, want %+v", cleared, counts) + } + + // Derived state is gone. + after, err := s.CountDestinationRecordedState(ctx, "arch") + if err != nil { + t.Fatalf("CountDestinationRecordedState after: %v", err) + } + if !after.Empty() { + t.Fatalf("post-reset counts = %+v, want empty", after) + } + vec, err := s.ListDestinationRunIDs(ctx, vID, "arch") + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + if len(vec) != 0 { + t.Fatalf("vector for arch = %+v, want empty", vec) + } + + // The append-only advance log survives — the reset is audit-preserving. + hist, err := s.ListDestinationRunIDHistory(ctx, vID, "arch") + if err != nil { + t.Fatalf("ListDestinationRunIDHistory: %v", err) + } + if len(hist) == 0 { + t.Fatalf("advance history for arch was cleared; reset must preserve it") + } + + // The other destination is untouched. + otherVec, err := s.ListDestinationRunIDs(ctx, vID, "other") + if err != nil { + t.Fatalf("ListDestinationRunIDs(other): %v", err) + } + if len(otherVec) != 1 || otherVec[0].OriginNodeID != self { + t.Fatalf("other vector = %+v, want one component for self", otherVec) + } + + // The content rows are never touched. + if _, err := s.GetByPath(ctx, vID, "obj.txt"); err != nil { + t.Fatalf("content row for obj.txt lost: %v", err) + } +} + +// TestResetDestinationRecordsAuditRun: the reset is a kind='audit' run that +// finishes success and carries a 'reset-destination' runs_audit note with +// the cleared counts, alongside the usual 'finish' transition. +func TestResetDestinationRecordsAuditRun(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + vID := makeVolume(t, s, "/v") + runID := makeRun(t, s, vID) + seedDestinationState(t, s, vID, runID, "arch") + + resetRun, cleared, err := s.ResetDestination(ctx, "arch") + if err != nil { + t.Fatalf("ResetDestination: %v", err) + } + run, err := s.GetRun(ctx, resetRun) + if err != nil { + t.Fatalf("GetRun: %v", err) + } + if run.Kind != RunKindAudit || run.Status != RunStatusSuccess || run.VolumeID.Valid || run.Destination.Valid { + t.Fatalf("reset run = %+v, want a success audit run with NULL volume+destination", run) + } + if run.FileCount != cleared.Total() { + t.Fatalf("run file_count = %d, want cleared total %d", run.FileCount, cleared.Total()) + } + + audits, err := s.ListRunAudit(ctx, resetRun) + if err != nil { + t.Fatalf("ListRunAudit: %v", err) + } + var sawReset, sawFinish bool + for _, a := range audits { + switch a.Transition { + case TransitionResetDestination: + sawReset = true + if !a.Note.Valid || a.Note.String == "" { + t.Fatalf("reset-destination note is empty: %+v", a) + } + case TransitionFinish: + sawFinish = true + } + } + if !sawReset || !sawFinish { + t.Fatalf("audit transitions = %+v, want both reset-destination and finish", audits) + } + + // The original index run is preserved — reset never prunes runs. + if _, err := s.GetRun(ctx, runID); err != nil { + t.Fatalf("original index run %d lost: %v", runID, err) + } +} + +// TestResetDestinationEmpty: a destination with no recorded state reports +// empty counts, so the CLI can report "nothing to reset" rather than mint a +// run. +func TestResetDestinationEmpty(t *testing.T) { + s := openTestStore(t) + counts, err := s.CountDestinationRecordedState(context.Background(), "never-used") + if err != nil { + t.Fatalf("CountDestinationRecordedState: %v", err) + } + if !counts.Empty() { + t.Fatalf("counts = %+v, want empty for an unused destination", counts) + } +} + +// TestResetDestinationRejectsEmptyName: both entry points refuse a blank +// destination rather than clearing everything with an empty-string match. +func TestResetDestinationRejectsEmptyName(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + if _, err := s.CountDestinationRecordedState(ctx, ""); err == nil { + t.Fatalf("CountDestinationRecordedState(\"\") succeeded, want error") + } + if _, _, err := s.ResetDestination(ctx, ""); err == nil { + t.Fatalf("ResetDestination(\"\") succeeded, want error") + } +} diff --git a/store/runs_audit.go b/store/runs_audit.go index e74fb48..14be1d4 100644 --- a/store/runs_audit.go +++ b/store/runs_audit.go @@ -33,6 +33,13 @@ const ( // CHECK keeps destination NULL on audit rows, so this entry is where // the audit trail names the verified destination. TransitionVerifyDestination = "verify-destination" + // TransitionResetDestination records a `squirrel destination reset`: + // the operator forgetting a destination's recorded upload and + // durability state (ResetDestination). It shares the destination-scoped + // kind='audit' run shape, so — like TransitionVerifyDestination — this + // note is where the audit trail names the reset destination and carries + // the cleared counts, the runs CHECK keeping destination NULL on the row. + TransitionResetDestination = "reset-destination" ) // RunAudit is one row of the insert-only runs_audit log: a single From 599dacf984b99058bd0eb204eec86bb16e4a3861 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:39:26 +0000 Subject: [PATCH 2/6] sync: recognise an empty remote root as a fresh start (F20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The packed and content-addressed layout guards keyed on run history by destination name: a recorded success whose manifest segment / pack map is absent was refused outright, so following the guard's own advice (a fresh root) still refused. Now, when the marker is missing, the guard probes the configured root; an empty root — a wiped or repointed destination, or one cleared by `squirrel destination reset` — is treated as a fresh start and re-uploaded. A non-empty root, or any error probing it, keeps the refusal (fail-closed). The refusal message now also points at `destination reset`. Adds Rclone.remoteRootEmpty (recursive lsf) and a shared freshStartOnEmptyRoot helper. The fake-rclone test shim learns `lsf`; the mirror-era guard tests now seed a remote file so they exercise a genuine layout conflict, and new tests cover the empty-root fresh start on both layouts. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- sync/content_addressed.go | 5 ++- sync/content_addressed_test.go | 67 ++++++++++++++++++++++++++++++++++ sync/packed.go | 5 ++- sync/packed_test.go | 33 +++++++++++++++++ sync/rclone.go | 21 +++++++++++ sync/sync.go | 16 ++++++++ 6 files changed, 145 insertions(+), 2 deletions(-) diff --git a/sync/content_addressed.go b/sync/content_addressed.go index bbe0ecc..0ffc32f 100644 --- a/sync/content_addressed.go +++ b/sync/content_addressed.go @@ -211,7 +211,10 @@ func (h *contentAddressedHandler) watermark(ctx context.Context, volID int64) (i } segURI := h.segmentURI(last.ID) if _, err := h.rcl.statRemote(ctx, segURI, checkersArgs(h.dest)...); err != nil { - return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no manifest segment at %s — its history does not look content-addressed; point the layout at a fresh destination or root instead of switching an existing one: %w", h.dest.Name, last.ID, segURI, err) + if freshStartOnEmptyRoot(ctx, h.rcl, h.dest) { + return 0, nil + } + return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no manifest segment at %s — its history does not look content-addressed; point the layout at a fresh destination or root, or (after wiping the remote root) run `squirrel destination reset %s`, instead of switching an existing one: %w", h.dest.Name, last.ID, segURI, h.dest.Name, err) } return last.ID, nil } diff --git a/sync/content_addressed_test.go b/sync/content_addressed_test.go index fbb3f6d..d5f29e4 100644 --- a/sync/content_addressed_test.go +++ b/sync/content_addressed_test.go @@ -64,6 +64,7 @@ while [ $# -gt 0 ]; do --hash-type) shift; hashtypes="$hashtypes $1" ;; --include) shift; includes="$includes $1" ;; --checkers) shift ;; + -R) ;; --*) ;; *) if [ -z "$a1" ]; then a1="$1"; else a2="$1"; fi ;; esac @@ -119,6 +120,22 @@ lsjson) printf ']\n' fi ;; +lsf) + # Directory listing for the layout guard's emptiness probe. Resolve the + # base directory ignoring any crypt filename suffix — we only need to + # know whether any file exists beneath the root — then list recursively. + case "$a1" in + *:*) + p="${a1#*:}" + case "$p" in + "${RCLONE_FAKE_STRIP:-//none//}"/*) p="${p#"${RCLONE_FAKE_STRIP}"/}" ;; + esac + dir="$RCLONE_FAKE_ROOT/$p" ;; + *) dir="$a1" ;; + esac + [ -d "$dir" ] && find "$dir" -type f + exit 0 + ;; *) echo "unexpected rclone subcommand: $cmd $*" >&2; exit 64 ;; esac ` @@ -622,6 +639,10 @@ func TestContentAddressedWatermarkGuard(t *testing.T) { if err := f.store.FinishRun(ctx, mirrorRun, store.RunStatusSuccess, "", 1); err != nil { t.Fatalf("finish mirror-era run: %v", err) } + // A real mirror era leaves the volume's files mirrored under the root, + // so the root is non-empty: a genuine layout conflict, distinct from a + // wiped destination (which the fresh-start recognition below handles). + seedRemoteFile(t, f, "pics", "a.txt") rep, err := f.sync(t) if err == nil || !strings.Contains(err.Error(), "does not look content-addressed") { @@ -632,6 +653,52 @@ func TestContentAddressedWatermarkGuard(t *testing.T) { } } +// seedRemoteFile writes one file under the fixture's remote root, standing +// in for the bytes a prior (mirror) era left there so the root is non-empty. +func seedRemoteFile(t *testing.T, f *caFixture, parts ...string) { + t.Helper() + p := f.remotePath(parts...) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("mkdir remote file dir: %v", err) + } + if err := os.WriteFile(p, []byte("mirror-era-bytes"), 0o644); err != nil { + t.Fatalf("seed remote file: %v", err) + } +} + +// TestContentAddressedFreshStartOnEmptyRoot is the F20 fix: when name-keyed +// run history records a prior success but the configured root is empty on the +// remote (a wiped or repointed destination, or one cleared by +// `squirrel destination reset`), the guard treats it as a fresh start and +// re-uploads instead of refusing. +func TestContentAddressedFreshStartOnEmptyRoot(t *testing.T) { + f := setupContentAddressedFixture(t) + f.write(t, "a.txt", "alpha") + f.index(t) + + ctx := context.Background() + staleRun, err := f.store.BeginRun(ctx, store.RunKindSync, f.volumeID(t), "offsite", false) + if err != nil { + t.Fatalf("seed stale run: %v", err) + } + if err := f.store.FinishRun(ctx, staleRun, store.RunStatusSuccess, "", 1); err != nil { + t.Fatalf("finish stale run: %v", err) + } + // Remote root left empty — no seedRemoteFile — so the guard reads it as + // a fresh start. + + rep, err := f.sync(t) + if err != nil { + t.Fatalf("fresh-start sync failed: %v (rep=%+v)", err, rep) + } + if rep.Status != store.RunStatusSuccess { + t.Fatalf("Status = %q, want success", rep.Status) + } + if _, err := os.Stat(f.remoteBlob(ObjectsDirName, blake3Hex("alpha"))); err != nil { + t.Fatalf("fresh start did not re-upload the object: %v", err) + } +} + func TestContentAddressedDryRunRefused(t *testing.T) { f := setupContentAddressedFixture(t) f.write(t, "a.txt", "alpha") diff --git a/sync/packed.go b/sync/packed.go index 88bca90..2f5d5a3 100644 --- a/sync/packed.go +++ b/sync/packed.go @@ -264,7 +264,10 @@ func (h *packedHandler) watermark(ctx context.Context, volID int64) (int64, erro } mapURI := h.mapURI(last.ID) if _, err := h.rcl.statRemote(ctx, mapURI, checkersArgs(h.dest)...); err != nil { - return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no pack placement map at %s — its history is not packed (a mirror or content-addressed root); point the layout at a fresh destination or root instead of switching an existing one: %w", h.dest.Name, last.ID, mapURI, err) + if freshStartOnEmptyRoot(ctx, h.rcl, h.dest) { + return 0, nil + } + return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no pack placement map at %s — its history is not packed (a mirror or content-addressed root); point the layout at a fresh destination or root, or (after wiping the remote root) run `squirrel destination reset %s`, instead of switching an existing one: %w", h.dest.Name, last.ID, mapURI, h.dest.Name, err) } return last.ID, nil } diff --git a/sync/packed_test.go b/sync/packed_test.go index ba285b3..10dcc6d 100644 --- a/sync/packed_test.go +++ b/sync/packed_test.go @@ -450,6 +450,9 @@ func TestPackedWatermarkGuardRefusesMirror(t *testing.T) { if err := f.store.FinishRun(ctx, mirrorRun, store.RunStatusSuccess, "", 1); err != nil { t.Fatalf("finish mirror run: %v", err) } + // A real mirror era leaves files under the root, so the root is + // non-empty — a genuine layout conflict, not a wiped destination. + seedRemoteFile(t, f, "pics", "a.txt") rep, err := f.sync(t) if err == nil || !strings.Contains(err.Error(), "not packed") { t.Fatalf("expected packed-history refusal, got %v", err) @@ -459,6 +462,36 @@ func TestPackedWatermarkGuardRefusesMirror(t *testing.T) { } } +// TestPackedFreshStartOnEmptyRoot is the packed analogue of the F20 fix: a +// prior success recorded under this destination name, but an empty remote +// root, is a fresh start rather than a refusal. +func TestPackedFreshStartOnEmptyRoot(t *testing.T) { + f := setupPackedFixture(t, "1MiB") + f.write(t, "a.txt", "tiny") + f.index(t) + + ctx := context.Background() + staleRun, err := f.store.BeginRun(ctx, store.RunKindSync, f.volumeID(t), "offsite", false) + if err != nil { + t.Fatalf("seed stale run: %v", err) + } + if err := f.store.FinishRun(ctx, staleRun, store.RunStatusSuccess, "", 1); err != nil { + t.Fatalf("finish stale run: %v", err) + } + // Remote root left empty — the guard reads it as a fresh start. + + rep, err := f.sync(t) + if err != nil { + t.Fatalf("fresh-start sync failed: %v (rep=%+v)", err, rep) + } + if rep.Status != store.RunStatusSuccess { + t.Fatalf("Status = %q, want success", rep.Status) + } + if _, err := os.Stat(f.remoteBlob(PacksDirName, fmt.Sprintf("map-%d", rep.RunID))); err != nil { + t.Fatalf("fresh start wrote no placement map: %v", err) + } +} + // TestPackedDryRunRefused: the packed push has no dry-run mode yet. func TestPackedDryRunRefused(t *testing.T) { f := setupPackedFixture(t, "1MiB") diff --git a/sync/rclone.go b/sync/rclone.go index 19316e7..f0d2c25 100644 --- a/sync/rclone.go +++ b/sync/rclone.go @@ -388,6 +388,27 @@ func (r *Rclone) deleteFile(ctx context.Context, fileURI string) error { return err } +// remoteRootEmpty reports whether rootURI holds no files, listing it +// recursively via `rclone lsf -R --files-only`. A missing root (rclone +// exits non-zero with empty output, as when the directory was never +// created) counts as empty, mirroring listSnapshots' treatment of an +// absent snapshot directory. The layout guards use it to recognise a +// wiped or repointed destination — including one whose recorded state was +// cleared by `squirrel destination reset` — as a fresh start rather than +// a layout conflict. A genuine listing error (non-empty stderr, empty +// stdout) surfaces so the caller can stay fail-closed and refuse. +func (r *Rclone) remoteRootEmpty(ctx context.Context, rootURI string, extraArgs ...string) (bool, error) { + args := append([]string{"lsf", "-R", "--files-only"}, extraArgs...) + out, err := r.runPlain(ctx, append(args, rootURI)...) + if err != nil { + if len(bytes.TrimSpace(out)) == 0 { + return true, nil + } + return false, err + } + return len(bytes.TrimSpace(out)) == 0, nil +} + // statRemote returns the size of the single object at uri via // `rclone lsjson --stat`. The content-addressed push uses it to confirm // presence and size of each uploaded object and manifest segment; diff --git a/sync/sync.go b/sync/sync.go index 24a1ca1..e8dbbc8 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -634,6 +634,22 @@ func remoteSubpathURI(dest *config.Destination, subpath string) string { } } +// freshStartOnEmptyRoot reports whether a missing layout marker (a manifest +// segment or pack placement map absent at the last recorded success) should +// be read as a fresh start rather than a layout conflict. It is true only +// when dest's configured root holds no files on the remote — a wiped or +// repointed destination, or one whose recorded state was cleared by +// `squirrel destination reset` — so the name-keyed run history no longer +// describes anything on disk and a fresh full push is safe (it skips +// nothing). A non-empty root, or any error probing it, returns false so the +// caller keeps its refusal: fail-closed, because refusing a real +// layout-switch is recoverable while a delta against a stale watermark +// silently skips content. +func freshStartOnEmptyRoot(ctx context.Context, rcl *Rclone, dest *config.Destination) bool { + empty, err := rcl.remoteRootEmpty(ctx, remoteSubpathURI(dest, ""), checkersArgs(dest)...) + return err == nil && empty +} + // destinationVolumeURI returns the rclone destination spec for the given // volume under dest. func destinationVolumeURI(dest *config.Destination, volumeName string) string { From 9ec30093fc9d43ca211b959258fec5e22f551d4f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:39:36 +0000 Subject: [PATCH 3/6] cmd: add `destination reset` and signpost machine replacement (F20, F28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `squirrel destination reset ` is a weighty change command: it prints what it will clear, refuses without --yes, and offers --dry-run — mirroring ux-principle 2. Parent `destination` command and the `reset` subcommand live in separate files. A configured-but-unrecorded destination reports "nothing to reset" rather than minting an empty run. restore's node dead-end ("restore from node destinations is not supported") now points at the machine-replacement runbook — rebuilding an edge machine from its hub is a reverse peer push driven from the hub, not a restore. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/destination.go | 19 ++++ cmd/squirrel/destination_reset.go | 117 +++++++++++++++++++++ cmd/squirrel/destination_reset_test.go | 134 +++++++++++++++++++++++++ cmd/squirrel/restore.go | 8 +- cmd/squirrel/root.go | 1 + 5 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 cmd/squirrel/destination.go create mode 100644 cmd/squirrel/destination_reset.go create mode 100644 cmd/squirrel/destination_reset_test.go diff --git a/cmd/squirrel/destination.go b/cmd/squirrel/destination.go new file mode 100644 index 0000000..a9081b7 --- /dev/null +++ b/cmd/squirrel/destination.go @@ -0,0 +1,19 @@ +package main + +import "github.com/spf13/cobra" + +// newDestinationCmd returns the `squirrel destination` parent command +// grouping the destination-level recovery verbs. It has no behaviour of its +// own and prints help when invoked bare. +func newDestinationCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "destination", + Short: "Manage a destination's recorded state", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, + } + cmd.AddCommand(newDestinationResetCmd()) + return cmd +} diff --git a/cmd/squirrel/destination_reset.go b/cmd/squirrel/destination_reset.go new file mode 100644 index 0000000..88e649b --- /dev/null +++ b/cmd/squirrel/destination_reset.go @@ -0,0 +1,117 @@ +package main + +import ( + "fmt" + "io" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/store" +) + +// newDestinationResetCmd returns `squirrel destination reset `: +// the explicit operator verb that forgets a destination's recorded upload and +// durability state so the next sync treats it as fresh (friction log F20). +// It is a *change* command in the ux-principles sense — weighty and +// irreversible in effect — so it defaults to refusing, offers a --dry-run +// preview, and requires an explicit --yes to proceed. The runs table and the +// append-only durability advance log are preserved; the reset itself is +// recorded as an audit run. +func newDestinationResetCmd() *cobra.Command { + var ( + yes bool + dryRun bool + ) + cmd := &cobra.Command{ + Use: "reset ", + Short: "Forget a destination's recorded upload and durability state (audit-preserving)", + Long: `Forget everything the index records about a destination's remote state: +its per-content and per-pack upload ledgers, its live durability vector, and +its push-freshness maxima. The next sync then treats the destination as fresh +and re-uploads. + +Use it to recover a wrecked or repointed destination — e.g. after wiping the +remote root, or pointing an existing destination name at a fresh root, where +the layout guard would otherwise refuse on name-keyed run history alone. + +This clears derived state only. The runs table and the append-only durability +advance log are untouched, so the audit trail survives; the reset is itself +recorded as an audit run. The remote bytes are not deleted — wipe or repoint +the remote root separately if you want the destination genuinely empty.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runDestinationReset(cmd, args[0], yes, dryRun) + }, + } + cmd.Flags().BoolVar(&yes, "yes", false, "confirm the reset; required to actually clear recorded state") + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "print what would be cleared without changing anything") + return cmd +} + +func runDestinationReset(cmd *cobra.Command, name string, yes, dryRun bool) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + if _, ok := cfg.Destinations[name]; !ok { + return fmt.Errorf("unknown destination %q (declare it in %s); destination reset clears a bucket destination's recorded state — a peer node's durability assertions are revoked separately", name, cfg.Path) + } + + s, err := openStore(cmd, cfg) + if err != nil { + return err + } + defer s.Close() + + counts, err := s.CountDestinationRecordedState(cmd.Context(), name) + if err != nil { + return err + } + out := cmd.OutOrStdout() + if counts.Empty() { + fmt.Fprintf(out, "destination %q has no recorded state — nothing to reset\n", name) + return nil + } + if dryRun { + printResetPreview(out, name, counts, true) + return nil + } + if !yes { + printResetPreview(out, name, counts, false) + return fmt.Errorf("refusing to clear recorded state without confirmation; re-run with --yes (or --dry-run to preview)") + } + + runID, cleared, err := s.ResetDestination(cmd.Context(), name) + if err != nil { + return err + } + printResetResult(out, name, cleared, runID) + return nil +} + +// printResetPreview renders the per-category counts that a reset would clear. +// dryRun toggles only the leading verb so the same table serves both the +// explicit --dry-run preview and the confirmation gate. +func printResetPreview(out io.Writer, name string, c store.DestinationResetCounts, dryRun bool) { + prefix := "reset would clear" + if !dryRun { + prefix = "this will clear" + } + fmt.Fprintf(out, "%s destination %q recorded state:\n", prefix, name) + printResetCounts(out, c) + fmt.Fprintln(out, " (runs history and the durability advance log are preserved; remote bytes are untouched)") +} + +// printResetResult renders the loud confirmation after a reset ran, naming +// the audit run that recorded it and what to do next. +func printResetResult(out io.Writer, name string, c store.DestinationResetCounts, runID int64) { + fmt.Fprintf(out, "reset destination %q (run %d):\n", name, runID) + printResetCounts(out, c) + fmt.Fprintf(out, "next: the next sync re-uploads to %q; for a content-addressed or packed layout, wipe or repoint the remote root so the layout guard sees a fresh start\n", name) +} + +func printResetCounts(out io.Writer, c store.DestinationResetCounts) { + fmt.Fprintf(out, " upload records: %d objects, %d packs\n", c.RemoteObjects, c.RemotePacks) + fmt.Fprintf(out, " durability vector: %d component(s)\n", c.VectorComponents) + fmt.Fprintf(out, " push freshness: %d row(s)\n", c.FreshnessRows) +} diff --git a/cmd/squirrel/destination_reset_test.go b/cmd/squirrel/destination_reset_test.go new file mode 100644 index 0000000..d8f9a5a --- /dev/null +++ b/cmd/squirrel/destination_reset_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/mbertschler/squirrel/store" +) + +// writeResetConfig writes a config with one local destination "arch" and a +// db path, returning the config path and the db path. +func writeResetConfig(t *testing.T) (configPath, dbPath string) { + t.Helper() + dir := t.TempDir() + dbPath = filepath.Join(dir, "index.db") + destDir := filepath.Join(dir, "arch") + body := fmt.Sprintf("db = %q\n\n[destinations.arch]\ntype = \"local\"\nroot = %q\n", dbPath, destDir) + configPath = writeCheckConfig(t, body) + return configPath, dbPath +} + +// seedResetVector opens the fixture DB directly and records one durability +// vector component for destination, so the reset has something to clear. +func seedResetVector(t *testing.T, dbPath, destination string) { + t.Helper() + ctx := context.Background() + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + defer s.Close() + v, err := s.GetOrCreateVolume(ctx, "/v") + if err != nil { + t.Fatalf("GetOrCreateVolume: %v", err) + } + self, err := s.GetSelfNode(ctx) + if err != nil { + t.Fatalf("GetSelfNode: %v", err) + } + if err := s.UpsertDestinationRunIDVerified(ctx, v.ID, destination, self.ID, 5, store.VerifyMethodBlake3, false); err != nil { + t.Fatalf("UpsertDestinationRunIDVerified: %v", err) + } +} + +// resetVectorCount reports how many vector components destination still has. +func resetVectorCount(t *testing.T, dbPath, destination string) int { + t.Helper() + ctx := context.Background() + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + defer s.Close() + v, err := s.GetOrCreateVolume(ctx, "/v") + if err != nil { + t.Fatalf("GetOrCreateVolume: %v", err) + } + vec, err := s.ListDestinationRunIDs(ctx, v.ID, destination) + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + return len(vec) +} + +// TestCLIDestinationResetUnknown: a destination not in config is refused +// before the store is touched. +func TestCLIDestinationResetUnknown(t *testing.T) { + cfgPath, _ := writeResetConfig(t) + out, err := runCLIExpectErr(t, "--config", cfgPath, "destination", "reset", "ghost", "--yes") + if !strings.Contains(out, "unknown destination") && !strings.Contains(err.Error(), "unknown destination") { + t.Fatalf("expected unknown-destination error, got out=%q err=%v", out, err) + } +} + +// TestCLIDestinationResetNothing: a configured destination with no recorded +// state reports "nothing to reset" affirmatively and mints no run. +func TestCLIDestinationResetNothing(t *testing.T) { + cfgPath, _ := writeResetConfig(t) + out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--yes") + if !strings.Contains(out, "nothing to reset") { + t.Fatalf("expected nothing-to-reset message:\n%s", out) + } +} + +// TestCLIDestinationResetDryRun: --dry-run previews the counts and clears +// nothing. +func TestCLIDestinationResetDryRun(t *testing.T) { + cfgPath, dbPath := writeResetConfig(t) + seedResetVector(t, dbPath, "arch") + + out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--dry-run") + if !strings.Contains(out, "would clear") { + t.Fatalf("expected dry-run preview:\n%s", out) + } + if n := resetVectorCount(t, dbPath, "arch"); n != 1 { + t.Fatalf("dry-run cleared state: vector count = %d, want 1", n) + } +} + +// TestCLIDestinationResetNeedsConfirmation: without --yes the command +// previews and refuses, changing nothing (principle 2: weighty change). +func TestCLIDestinationResetNeedsConfirmation(t *testing.T) { + cfgPath, dbPath := writeResetConfig(t) + seedResetVector(t, dbPath, "arch") + + out, err := runCLIExpectErr(t, "--config", cfgPath, "destination", "reset", "arch") + if !strings.Contains(err.Error(), "--yes") { + t.Fatalf("expected confirmation-required error, got %v", err) + } + if !strings.Contains(out, "this will clear") { + t.Fatalf("expected preview before refusal:\n%s", out) + } + if n := resetVectorCount(t, dbPath, "arch"); n != 1 { + t.Fatalf("refused reset still cleared state: vector count = %d, want 1", n) + } +} + +// TestCLIDestinationResetConfirmed: --yes clears the state and names the +// audit run that recorded it. +func TestCLIDestinationResetConfirmed(t *testing.T) { + cfgPath, dbPath := writeResetConfig(t) + seedResetVector(t, dbPath, "arch") + + out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--yes") + if !strings.Contains(out, "reset destination") || !strings.Contains(out, "run ") { + t.Fatalf("expected loud reset confirmation with run id:\n%s", out) + } + if n := resetVectorCount(t, dbPath, "arch"); n != 0 { + t.Fatalf("confirmed reset left state: vector count = %d, want 0", n) + } +} diff --git a/cmd/squirrel/restore.go b/cmd/squirrel/restore.go index deab82e..edb8339 100644 --- a/cmd/squirrel/restore.go +++ b/cmd/squirrel/restore.go @@ -195,9 +195,11 @@ func pickSingleRestoreDestination(cfg *config.Config, vol *config.Volume) (*conf } dest, ok := cfg.Destinations[vol.SyncTo[0]] if !ok { - // sync_to may name a node, not a bucket; restore doesn't yet - // pull from peer nodes, so surface that mismatch explicitly. - return nil, fmt.Errorf("volume %q syncs only to %q which is not a bucket destination — restore from node destinations is not supported", vol.Name, vol.SyncTo[0]) + // sync_to may name a node, not a bucket; restore doesn't pull from + // peer nodes. Rebuilding an edge machine from its hub is a reverse + // peer push driven from the hub — the same mechanism nas→htpc uses + // daily — not a restore, so point there rather than dead-ending. + return nil, fmt.Errorf("volume %q syncs only to peer node %q; restore pulls from bucket destinations, not nodes — to rebuild this machine from its hub, drive a reverse peer push from the hub (see the machine-replacement runbook in docs/guides/recovery)", vol.Name, vol.SyncTo[0]) } return dest, nil } diff --git a/cmd/squirrel/root.go b/cmd/squirrel/root.go index af77560..da59108 100644 --- a/cmd/squirrel/root.go +++ b/cmd/squirrel/root.go @@ -57,6 +57,7 @@ func newRootCmd() *cobra.Command { root.AddCommand(newDBCmd()) root.AddCommand(newConfigCmd()) root.AddCommand(newNodeCmd()) + root.AddCommand(newDestinationCmd()) return root } From bf1b7f43290c5325934aca6db2da1b990da543af Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:39:45 +0000 Subject: [PATCH 4/6] docs: recovery & disaster runbooks (F20, F28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New "Recovery & disaster runbooks" guide: resetting a wrecked destination, rebuilding a dead machine from its hub via a reverse peer push (re-pairing with `node pair`), and the manual DR that already works and stays — mirror restore, ride-along index-snapshot recovery, and packed/content-addressed recovery. Adds `squirrel destination reset` to the CLI reference and the guide to the sidebar. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- docs/astro.config.mjs | 1 + docs/src/content/docs/guides/recovery.md | 132 +++++++++++++++++++++++ docs/src/content/docs/reference/cli.md | 31 ++++++ 3 files changed, 164 insertions(+) create mode 100644 docs/src/content/docs/guides/recovery.md diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index 59f3d82..85a9acb 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -62,6 +62,7 @@ export default defineConfig({ { label: "Offsite verification", link: "/guides/verification/" }, { label: "Offloading", link: "/guides/offloading/" }, { label: "Restoring", link: "/guides/restore/" }, + { label: "Recovery & disaster runbooks", link: "/guides/recovery/" }, { label: "Hooks", link: "/guides/hooks/" }, { label: "Auditing for drift", link: "/guides/auditing/" }, { label: "Peer sync", link: "/guides/peer-sync/" }, diff --git a/docs/src/content/docs/guides/recovery.md b/docs/src/content/docs/guides/recovery.md new file mode 100644 index 0000000..09f70f9 --- /dev/null +++ b/docs/src/content/docs/guides/recovery.md @@ -0,0 +1,132 @@ +--- +title: Recovery & disaster runbooks +description: Reset a wrecked destination, rebuild a dead machine from its hub, and the manual disaster-recovery paths that already work — mirror restore, index-snapshot recovery, and packed/content-addressed recovery. +--- + +Recovery in squirrel is deliberately a set of **explicit, human-driven acts** +— never something the agent does on a cadence. Each one preserves the audit +trail: the runs table is never pruned, and no history is overwritten without +an opt-in operator command. This page collects the recovery paths. + +## Resetting a wrecked destination + +`squirrel destination reset ` forgets everything the index records about +a destination's remote state — its per-content and per-pack upload ledgers, its +live durability vector, and its push-freshness maxima — so the next sync treats +the destination as fresh and re-uploads. + +Use it when a destination's recorded state no longer matches reality and a sync +refuses to proceed. The classic case: after wiping a remote, or pointing an +existing destination name at a fresh `root`, the +[content-addressed](/squirrel/layouts/content-addressed/) or +[packed](/squirrel/layouts/packed/) layout guard refuses — its recorded history +under that destination name expects a manifest segment or placement map that is +no longer there. Before this verb, the only escape was hand-editing SQLite or +renaming the destination across every machine's config. + +```sh +squirrel destination reset s3archive --dry-run # preview what would clear +squirrel destination reset s3archive --yes # actually clear it +``` + +It is a **change** command, so it is weighty by design: it prints what it will +clear, and refuses without `--yes` (or a `--dry-run` preview). + +### What is and isn't cleared + +Cleared (derived state only): + +- `remote_objects` and `remote_packs` — the upload ledgers. +- the destination's durability vector and push-freshness rows. + +Preserved (the audit trail): + +- the **runs** table — every past sync, verify, and audit stays; the reset is + itself recorded as a new audit run. +- the append-only **durability advance log** — the record of what was ever + asserted durable survives, exactly as [revoking a peer](/squirrel/guides/peer-sync/) + leaves its history intact. +- the **content and files** rows — squirrel never loses track of content. + +The **remote bytes are not touched** — reset only clears squirrel's *record*. +If you want the destination genuinely empty, wipe or repoint the remote root +separately. Once the configured root is empty on the remote, the layout guard +recognises the fresh start on its own and the next sync re-uploads from scratch. + +:::note[Stop the agent first] +A running agent may kick a sync on its cadence mid-reset. Nothing is lost — a +content-addressed re-upload is idempotent — but pausing the agent keeps the +recovery legible. +::: + +## Rebuilding a machine from its hub + +When an edge machine dies (the most likely household disaster), rebuild it from +the hub with a **reverse peer push** — the same peer-sync mechanism the hub uses +to feed a receive-only node every day. There is no separate restore verb for +this: `squirrel restore` pulls from bucket destinations, not from peer nodes, +and the machinery to push a volume to a node already exists. + +The story, using the [reference setup](/squirrel/reference/configuration/) +(rebuilding `laptop` from `nas`): + +1. **Install and re-pair the replacement.** Install squirrel, generate its + agent cert, and re-establish the peer relationship with the hub. Use the + pairing helper so the token matrix is correct by construction: + + ```sh + squirrel agent cert # on the replacement: new cert + pin + squirrel node pair nas # emits matching config halves for both + ``` + + Paste each half into the named machine's config and fill any placeholders + (see [Peer sync](/squirrel/guides/peer-sync/)). The replacement runs an agent + that **listens** so the hub can dial it — the reverse of its normal + initiate-only role. + +2. **Point the hub at the replacement, temporarily.** On the hub, add the + replacement as a `[nodes.laptop]` entry and list it in the `sync_to` of the + volumes to restore. Then drive the push: + + ```sh + squirrel sync photos --to laptop + squirrel sync docs --to laptop + ``` + + The hub enumerates its master copy and pushes; the replacement receives and + re-hashes every path (`peer-blake3`), so the restore is content-verified end + to end. Both sides record correlated runs — the recovery is in the audit + trail like any other sync. + +3. **Return to steady state.** Once the replacement holds the volumes, remove + the temporary hub-side `sync_to`/`[nodes.laptop]` push config, and resume the + normal edge → hub direction (`laptop` initiating to `nas`). + +## Manual disaster recovery that already works + +These paths need no new tooling and are the right answer for whole-repository or +hub loss. They stay first-class. + +### Mirror restore + +For a [mirror](/squirrel/layouts/mirror/) destination (including an encrypted +one), [`squirrel restore`](/squirrel/guides/restore/) pulls the volume back +byte-for-byte, decrypting on the way down and verifying BLAKE3 where the +destination exposes hashes. + +### Recovering the index too + +squirrel rides an [index snapshot](/squirrel/configuration/index-snapshots/) +along to destination buckets under `.squirrel-index/`. When the hub itself dies, +a restore-from-cloud yields the data *and* the index that explains it — fetch +the ride-along snapshot and swap it in as the live index with +[`squirrel db restore`](/squirrel/reference/cli/#squirrel-db). `runs` and `query` +answer immediately against the recovered catalog. + +### Packed & content-addressed recovery + +The [content-addressed](/squirrel/layouts/content-addressed/) and +[packed](/squirrel/layouts/packed/) formats are deliberately simple enough to +[recover without squirrel](/squirrel/reference/formats/#disaster-recovery-without-squirrel) +— the manifest and pack formats are documented, so the bytes are recoverable +with standard tools even if squirrel is unavailable. diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index 07a992d..9d8d2ae 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -26,6 +26,7 @@ squirrel runs fail squirrel hooks [--volume NAME] [--limit N] squirrel volumes squirrel restore [--from NAME] [--to PATH] [--shallow] [--dry-run] [--in-place] +squirrel destination reset [--yes] [--dry-run] squirrel audit [] [--deep | --folders] squirrel peer-sync history squirrel peer-sync pull-durability [--allow-rewind] @@ -229,6 +230,36 @@ because rclone crypt remotes don't expose content hashes. --- +## squirrel destination + +**Manage a destination's recorded state.** + +On its own it prints help. See [Recovery & disaster runbooks](/squirrel/guides/recovery/). + +### squirrel destination reset + +**Forget a destination's recorded upload and durability state (audit-preserving).** + +``` +squirrel destination reset +``` + +One positional — the destination name. Clears the `remote_objects`/`remote_packs` +upload ledgers, the durability vector, and the push-freshness rows for the +destination, so the next sync treats it as fresh and re-uploads. The runs table +and the append-only durability advance log are preserved, and the reset is +recorded as an audit run; the remote bytes are untouched. + +| Flag | Default | Meaning | +|---|---|---| +| `--yes` | `false` | Confirm the reset; required to actually clear state. | +| `--dry-run` | `false` | Print what would be cleared without changing anything. | + +Use it to recover a wrecked or repointed destination — see +[Resetting a wrecked destination](/squirrel/guides/recovery/#resetting-a-wrecked-destination). + +--- + ## squirrel audit **Walk a volume looking for out-of-band drift since the last index.** From 38650462d5f6d3d845a392421e58395353ebe5dc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:40:55 +0000 Subject: [PATCH 5/6] design: mark F20 and F28 addressed by #176 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- design/friction-log.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/design/friction-log.md b/design/friction-log.md index 496c68f..0ded92f 100644 --- a/design/friction-log.md +++ b/design/friction-log.md @@ -205,7 +205,7 @@ TUI-side folding of consecutive no-ops would restore the audit trail's readability. (The `runs` help text also still says "List index runs"; it lists every kind.) -**F20 · S2 — recovering a wrecked destination has no supported path.** +**F20 · S2 — ~~recovering a wrecked destination has no supported path.~~ (`squirrel destination reset` + empty-root guard, #176)** After the F12 bug era the packed-layout guard refused every further s3archive sync ("its history is not packed … point the layout at a fresh destination or root"). But the guard keys on the pair's *run @@ -337,7 +337,7 @@ content-verified method and relay that — the schema already carries ## Checkpoint 8 — restore day -**F28 · S2 — a dead edge machine has no supported restore path.** The +**F28 · S2 — ~~a dead edge machine has no supported restore path.~~ (reverse-peer-push runbook + restore signpost, #176)** The laptop syncs only to the nas (a node), and `restore` refuses nodes outright ("restore from node destinations is not supported" — clear, at least). The machinery for the recovery *exists* — a reverse peer From fbea42423b7ab48d9196bf329b0e429452da1045 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:48:57 +0000 Subject: [PATCH 6/6] sync,cmd: address Copilot review on #176 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FIX 1 (fail-closed): remoteRootEmpty no longer reads any lsf error with empty stdout as an empty root — runPlain carries stderr only in the error, so an auth/network failure was being mistaken for a wiped destination and letting the layout guard proceed. Now only rclone's canonical "directory not found" (or "object not found") counts as a fresh root; every other error surfaces so the guard fails closed. New test injects an auth-style lsf error (guard refuses) vs a not-found (fresh root). FIX 2: the restore node-restore signpost pointed at a repo source path (docs/guides/recovery) that doesn't exist for an installed binary; it now names the published docs route (guides/recovery). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/restore.go | 2 +- sync/content_addressed_test.go | 38 +++++++++++++++++++++++++++++++--- sync/rclone.go | 35 +++++++++++++++++++++++-------- 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/cmd/squirrel/restore.go b/cmd/squirrel/restore.go index edb8339..f92c1fa 100644 --- a/cmd/squirrel/restore.go +++ b/cmd/squirrel/restore.go @@ -199,7 +199,7 @@ func pickSingleRestoreDestination(cfg *config.Config, vol *config.Volume) (*conf // peer nodes. Rebuilding an edge machine from its hub is a reverse // peer push driven from the hub — the same mechanism nas→htpc uses // daily — not a restore, so point there rather than dead-ending. - return nil, fmt.Errorf("volume %q syncs only to peer node %q; restore pulls from bucket destinations, not nodes — to rebuild this machine from its hub, drive a reverse peer push from the hub (see the machine-replacement runbook in docs/guides/recovery)", vol.Name, vol.SyncTo[0]) + return nil, fmt.Errorf("volume %q syncs only to peer node %q; restore pulls from bucket destinations, not nodes — to rebuild this machine from its hub, drive a reverse peer push from the hub (see the machine-replacement runbook: the Recovery guide, guides/recovery)", vol.Name, vol.SyncTo[0]) } return dest, nil } diff --git a/sync/content_addressed_test.go b/sync/content_addressed_test.go index d5f29e4..93d968e 100644 --- a/sync/content_addressed_test.go +++ b/sync/content_addressed_test.go @@ -121,9 +121,13 @@ lsjson) fi ;; lsf) - # Directory listing for the layout guard's emptiness probe. Resolve the - # base directory ignoring any crypt filename suffix — we only need to - # know whether any file exists beneath the root — then list recursively. + # Directory listing for the layout guard's emptiness probe. An injected + # error simulates a not-found (fresh root) or a real failure (auth etc.) + # so the guard's fail-closed classification can be tested. + if [ -n "$RCLONE_FAKE_LSF_ERROR" ]; then echo "$RCLONE_FAKE_LSF_ERROR" >&2; exit 1; fi + # Resolve the base directory ignoring any crypt filename suffix — we only + # need to know whether any file exists beneath the root — then list + # recursively. case "$a1" in *:*) p="${a1#*:}" @@ -198,6 +202,7 @@ func setupCAFixture(t *testing.T, destBlock, strip string) *caFixture { t.Setenv("RCLONE_FAKE_HASH_VALUE", "") t.Setenv("RCLONE_FAKE_HASH_PREFIX", "") t.Setenv("RCLONE_FAKE_CRYPT_SUFFIX", "") + t.Setenv("RCLONE_FAKE_LSF_ERROR", "") root := t.TempDir() volPath := filepath.Join(root, "src") @@ -653,6 +658,33 @@ func TestContentAddressedWatermarkGuard(t *testing.T) { } } +// TestRemoteRootEmptyClassifiesErrors guards the layout guard's fail-closed +// contract: only rclone's canonical not-found reads as an empty (fresh) +// root; any other lsf error must surface so the guard refuses rather than +// mistaking a broken probe (auth, network) for a wiped destination. +func TestRemoteRootEmptyClassifiesErrors(t *testing.T) { + f := setupContentAddressedFixture(t) + ctx := context.Background() + + // A canonical directory-not-found is a fresh root. + t.Setenv("RCLONE_FAKE_LSF_ERROR", "2020/01/01 ERROR : directory not found") + empty, err := f.rcl.remoteRootEmpty(ctx, "offsite:/data") + if err != nil || !empty { + t.Fatalf("not-found lsf = (%t, %v), want (true, nil)", empty, err) + } + + // An auth/network error must surface so the guard fails closed — never + // reported as an empty root just because stdout was empty. + t.Setenv("RCLONE_FAKE_LSF_ERROR", "Failed to create file system: 401 Unauthorized") + empty, err = f.rcl.remoteRootEmpty(ctx, "offsite:/data") + if err == nil { + t.Fatalf("auth lsf error swallowed (empty=%t); guard would proceed instead of refusing", empty) + } + if empty { + t.Fatalf("auth lsf error reported empty root; must be false") + } +} + // seedRemoteFile writes one file under the fixture's remote root, standing // in for the bytes a prior (mirror) era left there so the root is non-empty. func seedRemoteFile(t *testing.T, f *caFixture, parts ...string) { diff --git a/sync/rclone.go b/sync/rclone.go index f0d2c25..5bc34fd 100644 --- a/sync/rclone.go +++ b/sync/rclone.go @@ -389,19 +389,22 @@ func (r *Rclone) deleteFile(ctx context.Context, fileURI string) error { } // remoteRootEmpty reports whether rootURI holds no files, listing it -// recursively via `rclone lsf -R --files-only`. A missing root (rclone -// exits non-zero with empty output, as when the directory was never -// created) counts as empty, mirroring listSnapshots' treatment of an -// absent snapshot directory. The layout guards use it to recognise a -// wiped or repointed destination — including one whose recorded state was -// cleared by `squirrel destination reset` — as a fresh start rather than -// a layout conflict. A genuine listing error (non-empty stderr, empty -// stdout) surfaces so the caller can stay fail-closed and refuse. +// recursively via `rclone lsf -R --files-only`. Only rclone's canonical +// "directory not found" (a root that was never created) — accepting the +// "object not found" phrasing too — counts as an empty/fresh root. Every +// other error (auth, network, permission) surfaces so the layout guards +// stay fail-closed and refuse rather than mistaking a broken probe for a +// wiped destination: runPlain carries stderr only in the returned error, +// not in stdout, so an empty stdout alone must never be read as "empty". +// The guards use the empty result to recognise a wiped or repointed +// destination — including one whose recorded state was cleared by +// `squirrel destination reset` — as a fresh start rather than a layout +// conflict. func (r *Rclone) remoteRootEmpty(ctx context.Context, rootURI string, extraArgs ...string) (bool, error) { args := append([]string{"lsf", "-R", "--files-only"}, extraArgs...) out, err := r.runPlain(ctx, append(args, rootURI)...) if err != nil { - if len(bytes.TrimSpace(out)) == 0 { + if isRemoteNotFound(err) { return true, nil } return false, err @@ -409,6 +412,20 @@ func (r *Rclone) remoteRootEmpty(ctx context.Context, rootURI string, extraArgs return len(bytes.TrimSpace(out)) == 0, nil } +// isRemoteNotFound reports whether err is rclone's canonical +// "directory not found" (or "object not found") — the only listing failure +// the layout guard reads as an empty, fresh root. Any other error must +// surface so the guard fails closed. #150's isNotFoundErr is not on this +// branch, so the canonical-phrase check lives here; runPlain formats +// rclone's stderr into the error string, so matching err.Error() catches it. +func isRemoteNotFound(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "directory not found") || strings.Contains(msg, "object not found") +} + // statRemote returns the size of the single object at uri via // `rclone lsjson --stat`. The content-addressed push uses it to confirm // presence and size of each uploaded object and manifest segment;