recovery: destination reset + machine-replacement runbook (F20, F28)

cmd/squirrel/destination.go

@@ -0,0 +1,19 @@
+package main
+
+import "github.com/spf13/cobra"
+
+// newDestinationCmd returns the `squirrel destination` parent command
+// grouping the destination-level recovery verbs. It has no behaviour of its
+// own and prints help when invoked bare.
+func newDestinationCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "destination",
+		Short: "Manage a destination's recorded state",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			return cmd.Help()
+		},
+	}
+	cmd.AddCommand(newDestinationResetCmd())
+	return cmd
+}

cmd/squirrel/destination_reset.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/spf13/cobra"
+
+	"github.com/mbertschler/squirrel/store"
+)
+
+// newDestinationResetCmd returns `squirrel destination reset <destination>`:
+// the explicit operator verb that forgets a destination's recorded upload and
+// durability state so the next sync treats it as fresh (friction log F20).
+// It is a *change* command in the ux-principles sense — weighty and
+// irreversible in effect — so it defaults to refusing, offers a --dry-run
+// preview, and requires an explicit --yes to proceed. The runs table and the
+// append-only durability advance log are preserved; the reset itself is
+// recorded as an audit run.
+func newDestinationResetCmd() *cobra.Command {
+	var (
+		yes    bool
+		dryRun bool
+	)
+	cmd := &cobra.Command{
+		Use:   "reset <destination>",
+		Short: "Forget a destination's recorded upload and durability state (audit-preserving)",
+		Long: `Forget everything the index records about a destination's remote state:
+its per-content and per-pack upload ledgers, its live durability vector, and
+its push-freshness maxima. The next sync then treats the destination as fresh
+and re-uploads.
+
+Use it to recover a wrecked or repointed destination — e.g. after wiping the
+remote root, or pointing an existing destination name at a fresh root, where
+the layout guard would otherwise refuse on name-keyed run history alone.
+
+This clears derived state only. The runs table and the append-only durability
+advance log are untouched, so the audit trail survives; the reset is itself
+recorded as an audit run. The remote bytes are not deleted — wipe or repoint
+the remote root separately if you want the destination genuinely empty.`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runDestinationReset(cmd, args[0], yes, dryRun)
+		},
+	}
+	cmd.Flags().BoolVar(&yes, "yes", false, "confirm the reset; required to actually clear recorded state")
+	cmd.Flags().BoolVar(&dryRun, "dry-run", false, "print what would be cleared without changing anything")
+	return cmd
+}
+
+func runDestinationReset(cmd *cobra.Command, name string, yes, dryRun bool) error {
+	cfg, err := requireConfig(cmd)
+	if err != nil {
+		return err
+	}
+	if _, ok := cfg.Destinations[name]; !ok {
+		return fmt.Errorf("unknown destination %q (declare it in %s); destination reset clears a bucket destination's recorded state — a peer node's durability assertions are revoked separately", name, cfg.Path)
+	}
+
+	s, err := openStore(cmd, cfg)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	counts, err := s.CountDestinationRecordedState(cmd.Context(), name)
+	if err != nil {
+		return err
+	}
+	out := cmd.OutOrStdout()
+	if counts.Empty() {
+		fmt.Fprintf(out, "destination %q has no recorded state — nothing to reset\n", name)
+		return nil
+	}
+	if dryRun {
+		printResetPreview(out, name, counts, true)
+		return nil
+	}
+	if !yes {
+		printResetPreview(out, name, counts, false)
+		return fmt.Errorf("refusing to clear recorded state without confirmation; re-run with --yes (or --dry-run to preview)")
+	}
+
+	runID, cleared, err := s.ResetDestination(cmd.Context(), name)
+	if err != nil {
+		return err
+	}
+	printResetResult(out, name, cleared, runID)
+	return nil
+}
+
+// printResetPreview renders the per-category counts that a reset would clear.
+// dryRun toggles only the leading verb so the same table serves both the
+// explicit --dry-run preview and the confirmation gate.
+func printResetPreview(out io.Writer, name string, c store.DestinationResetCounts, dryRun bool) {
+	prefix := "reset would clear"
+	if !dryRun {
+		prefix = "this will clear"
+	}
+	fmt.Fprintf(out, "%s destination %q recorded state:\n", prefix, name)
+	printResetCounts(out, c)
+	fmt.Fprintln(out, "  (runs history and the durability advance log are preserved; remote bytes are untouched)")
+}
+
+// printResetResult renders the loud confirmation after a reset ran, naming
+// the audit run that recorded it and what to do next.
+func printResetResult(out io.Writer, name string, c store.DestinationResetCounts, runID int64) {
+	fmt.Fprintf(out, "reset destination %q (run %d):\n", name, runID)
+	printResetCounts(out, c)
+	fmt.Fprintf(out, "next: the next sync re-uploads to %q; for a content-addressed or packed layout, wipe or repoint the remote root so the layout guard sees a fresh start\n", name)
+}
+
+func printResetCounts(out io.Writer, c store.DestinationResetCounts) {
+	fmt.Fprintf(out, "  upload records: %d objects, %d packs\n", c.RemoteObjects, c.RemotePacks)
+	fmt.Fprintf(out, "  durability vector: %d component(s)\n", c.VectorComponents)
+	fmt.Fprintf(out, "  push freshness: %d row(s)\n", c.FreshnessRows)
+}

cmd/squirrel/destination_reset_test.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mbertschler/squirrel/store"
+)
+
+// writeResetConfig writes a config with one local destination "arch" and a
+// db path, returning the config path and the db path.
+func writeResetConfig(t *testing.T) (configPath, dbPath string) {
+	t.Helper()
+	dir := t.TempDir()
+	dbPath = filepath.Join(dir, "index.db")
+	destDir := filepath.Join(dir, "arch")
+	body := fmt.Sprintf("db = %q\n\n[destinations.arch]\ntype = \"local\"\nroot = %q\n", dbPath, destDir)
+	configPath = writeCheckConfig(t, body)
+	return configPath, dbPath
+}
+
+// seedResetVector opens the fixture DB directly and records one durability
+// vector component for destination, so the reset has something to clear.
+func seedResetVector(t *testing.T, dbPath, destination string) {
+	t.Helper()
+	ctx := context.Background()
+	s, err := store.Open(dbPath)
+	if err != nil {
+		t.Fatalf("store.Open: %v", err)
+	}
+	defer s.Close()
+	v, err := s.GetOrCreateVolume(ctx, "/v")
+	if err != nil {
+		t.Fatalf("GetOrCreateVolume: %v", err)
+	}
+	self, err := s.GetSelfNode(ctx)
+	if err != nil {
+		t.Fatalf("GetSelfNode: %v", err)
+	}
+	if err := s.UpsertDestinationRunIDVerified(ctx, v.ID, destination, self.ID, 5, store.VerifyMethodBlake3, false); err != nil {
+		t.Fatalf("UpsertDestinationRunIDVerified: %v", err)
+	}
+}
+
+// resetVectorCount reports how many vector components destination still has.
+func resetVectorCount(t *testing.T, dbPath, destination string) int {
+	t.Helper()
+	ctx := context.Background()
+	s, err := store.Open(dbPath)
+	if err != nil {
+		t.Fatalf("store.Open: %v", err)
+	}
+	defer s.Close()
+	v, err := s.GetOrCreateVolume(ctx, "/v")
+	if err != nil {
+		t.Fatalf("GetOrCreateVolume: %v", err)
+	}
+	vec, err := s.ListDestinationRunIDs(ctx, v.ID, destination)
+	if err != nil {
+		t.Fatalf("ListDestinationRunIDs: %v", err)
+	}
+	return len(vec)
+}
+
+// TestCLIDestinationResetUnknown: a destination not in config is refused
+// before the store is touched.
+func TestCLIDestinationResetUnknown(t *testing.T) {
+	cfgPath, _ := writeResetConfig(t)
+	out, err := runCLIExpectErr(t, "--config", cfgPath, "destination", "reset", "ghost", "--yes")
+	if !strings.Contains(out, "unknown destination") && !strings.Contains(err.Error(), "unknown destination") {
+		t.Fatalf("expected unknown-destination error, got out=%q err=%v", out, err)
+	}
+}
+
+// TestCLIDestinationResetNothing: a configured destination with no recorded
+// state reports "nothing to reset" affirmatively and mints no run.
+func TestCLIDestinationResetNothing(t *testing.T) {
+	cfgPath, _ := writeResetConfig(t)
+	out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--yes")
+	if !strings.Contains(out, "nothing to reset") {
+		t.Fatalf("expected nothing-to-reset message:\n%s", out)
+	}
+}
+
+// TestCLIDestinationResetDryRun: --dry-run previews the counts and clears
+// nothing.
+func TestCLIDestinationResetDryRun(t *testing.T) {
+	cfgPath, dbPath := writeResetConfig(t)
+	seedResetVector(t, dbPath, "arch")
+
+	out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--dry-run")
+	if !strings.Contains(out, "would clear") {
+		t.Fatalf("expected dry-run preview:\n%s", out)
+	}
+	if n := resetVectorCount(t, dbPath, "arch"); n != 1 {
+		t.Fatalf("dry-run cleared state: vector count = %d, want 1", n)
+	}
+}
+
+// TestCLIDestinationResetNeedsConfirmation: without --yes the command
+// previews and refuses, changing nothing (principle 2: weighty change).
+func TestCLIDestinationResetNeedsConfirmation(t *testing.T) {
+	cfgPath, dbPath := writeResetConfig(t)
+	seedResetVector(t, dbPath, "arch")
+
+	out, err := runCLIExpectErr(t, "--config", cfgPath, "destination", "reset", "arch")
+	if !strings.Contains(err.Error(), "--yes") {
+		t.Fatalf("expected confirmation-required error, got %v", err)
+	}
+	if !strings.Contains(out, "this will clear") {
+		t.Fatalf("expected preview before refusal:\n%s", out)
+	}
+	if n := resetVectorCount(t, dbPath, "arch"); n != 1 {
+		t.Fatalf("refused reset still cleared state: vector count = %d, want 1", n)
+	}
+}
+
+// TestCLIDestinationResetConfirmed: --yes clears the state and names the
+// audit run that recorded it.
+func TestCLIDestinationResetConfirmed(t *testing.T) {
+	cfgPath, dbPath := writeResetConfig(t)
+	seedResetVector(t, dbPath, "arch")
+
+	out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--yes")
+	if !strings.Contains(out, "reset destination") || !strings.Contains(out, "run ") {
+		t.Fatalf("expected loud reset confirmation with run id:\n%s", out)
+	}
+	if n := resetVectorCount(t, dbPath, "arch"); n != 0 {
+		t.Fatalf("confirmed reset left state: vector count = %d, want 0", n)
+	}
+}

cmd/squirrel/restore.go

@@ -195,9 +195,11 @@ func pickSingleRestoreDestination(cfg *config.Config, vol *config.Volume) (*conf
 	}
 	dest, ok := cfg.Destinations[vol.SyncTo[0]]
 	if !ok {
-		// sync_to may name a node, not a bucket; restore doesn't yet
-		// pull from peer nodes, so surface that mismatch explicitly.
-		return nil, fmt.Errorf("volume %q syncs only to %q which is not a bucket destination — restore from node destinations is not supported", vol.Name, vol.SyncTo[0])
+		// sync_to may name a node, not a bucket; restore doesn't pull from
+		// peer nodes. Rebuilding an edge machine from its hub is a reverse
+		// peer push driven from the hub — the same mechanism nas→htpc uses
+		// daily — not a restore, so point there rather than dead-ending.
+		return nil, fmt.Errorf("volume %q syncs only to peer node %q; restore pulls from bucket destinations, not nodes — to rebuild this machine from its hub, drive a reverse peer push from the hub (see the machine-replacement runbook: the Recovery guide, guides/recovery)", vol.Name, vol.SyncTo[0])
 	}
 	return dest, nil
 }

cmd/squirrel/root.go

@@ -57,6 +57,7 @@ func newRootCmd() *cobra.Command {
 	root.AddCommand(newDBCmd())
 	root.AddCommand(newConfigCmd())
 	root.AddCommand(newNodeCmd())
+	root.AddCommand(newDestinationCmd())
 	return root
 }
 

design/friction-log.md

@@ -205,7 +205,7 @@ TUI-side folding of consecutive no-ops would restore the audit trail's
 readability. (The `runs` help text also still says "List index runs";
 it lists every kind.)
 
-**F20 · S2 — recovering a wrecked destination has no supported path.**
+**F20 · S2 — ~~recovering a wrecked destination has no supported path.~~ (`squirrel destination reset` + empty-root guard, #176)**
 After the F12 bug era the packed-layout guard refused every further
 s3archive sync ("its history is not packed … point the layout at a
 fresh destination or root"). But the guard keys on the pair's *run
@@ -337,7 +337,7 @@ content-verified method and relay that — the schema already carries
 
 ## Checkpoint 8 — restore day
 
-**F28 · S2 — a dead edge machine has no supported restore path.** The
+**F28 · S2 — ~~a dead edge machine has no supported restore path.~~ (reverse-peer-push runbook + restore signpost, #176)** The
 laptop syncs only to the nas (a node), and `restore` refuses nodes
 outright ("restore from node destinations is not supported" — clear,
 at least). The machinery for the recovery *exists* — a reverse peer

docs/astro.config.mjs

@@ -62,6 +62,7 @@ export default defineConfig({
             { label: "Offsite verification", link: "/guides/verification/" },
             { label: "Offloading", link: "/guides/offloading/" },
             { label: "Restoring", link: "/guides/restore/" },
+            { label: "Recovery & disaster runbooks", link: "/guides/recovery/" },
             { label: "Hooks", link: "/guides/hooks/" },
             { label: "Auditing for drift", link: "/guides/auditing/" },
             { label: "Peer sync", link: "/guides/peer-sync/" },