From 0c2a859ba37948998d9d2456a44054dbbe94cc61 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 24 Jul 2026 11:14:19 +0000 Subject: [PATCH 01/15] config: accept plaintext crypt passwords, obscure at render (F2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crypt `password`/`password2` are now plaintext by default (a literal or { env = "VAR" }); squirrel obscures them into rclone's on-disk form at load time, retiring the manual `rclone obscure` step. The obscure helper matches rclone's format (AES-CTR under rclone's public key, base64url) but derives the IV deterministically from the plaintext so the rendered rclone.conf stays byte-stable — the "unexpected rewrite is a signal" invariant WriteRcloneConfig relies on. Determinism costs nothing here: obscure is obfuscation, not encryption (the key is public). Breaking change: a config whose values are already obscured must add `obscured = true` to the crypt block to keep them verbatim, or reveal them back to plaintext. Docs and the existing crypt tests carry the new marker. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- README.md | 2 +- config/config.go | 13 +-- config/config_test.go | 3 + config/crypt_plaintext_test.go | 79 +++++++++++++++++ config/destinations.go | 70 ++++++++++++--- config/obscure.go | 61 +++++++++++++ config/obscure_test.go | 85 +++++++++++++++++++ docs/src/content/docs/layouts/encrypted.md | 12 ++- .../content/docs/reference/configuration.md | 5 +- sync/rclone_test.go | 1 + 10 files changed, 307 insertions(+), 24 deletions(-) create mode 100644 config/crypt_plaintext_test.go create mode 100644 config/obscure.go create mode 100644 config/obscure_test.go diff --git a/README.md b/README.md index 50ce89f..2e8a758 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ password = { env = "OFFSITE_CRYPT_PASSWORD" } password2 = { env = "OFFSITE_CRYPT_SALT" } # salt — optional but recommended ``` -`password` and `password2` are **rclone-obscured** values, the same representation `rclone config` stores for its own crypt remotes — generate one with `rclone obscure `. Both accept a literal or `{ env = "VAR" }`. Squirrel renders two sections into its `rclone.conf` — the underlying remote plus a crypt remote wrapping it — and addresses all sync and restore transfers through the crypt remote. Keep the passwords safe: restoring from an encrypted destination requires them. +`password` and `password2` are **plaintext** (a literal or `{ env = "VAR" }`); squirrel obscures them into rclone's on-disk form when it renders `rclone.conf`, so you never run `rclone obscure` yourself. A config written before this behaviour, whose values are already obscured, keeps them verbatim by adding `obscured = true` to the crypt block. Squirrel renders two sections into its `rclone.conf` — the underlying remote plus a crypt remote wrapping it — and addresses all sync and restore transfers through the crypt remote. Keep the passwords safe: restoring from an encrypted destination requires them. Two properties to be aware of: diff --git a/config/config.go b/config/config.go index eac542a..772cbd5 100644 --- a/config/config.go +++ b/config/config.go @@ -231,12 +231,15 @@ const ( // filename_encryption is fixed off, keeping the destination tree layout // identical to an unencrypted destination. type Crypt struct { - // Password is the content-encryption password in rclone-obscured form, - // the same representation rclone's own crypt config stores (generate - // one with `rclone obscure`). Accepts a literal or { env = "VAR" }. + // Password is the content-encryption password already in rclone-obscured + // form — the representation rclone's own crypt config stores and rclone + // renders into rclone.conf. The config accepts plaintext (a literal or + // { env = "VAR" }) and squirrel obscures it at load time; a config that + // still supplies a pre-obscured value sets `obscured = true` so it is + // stored here verbatim. Password string - // Password2 is the salt, also rclone-obscured. Optional but - // recommended, matching rclone's crypt config. + // Password2 is the salt, in the same obscured form as Password. + // Optional but recommended, matching rclone's crypt config. Password2 string } diff --git a/config/config_test.go b/config/config_test.go index 85c9d62..44d247f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -562,6 +562,7 @@ root = "/data" password = "transport-pw" [destinations.offsite.crypt] +obscured = true password = { env = "CRYPT_PASSWORD" } password2 = "obscured-salt" `) @@ -595,6 +596,7 @@ root = "/data" password = "transport-pw" [destinations.offsite.crypt] +obscured = true password = "obscured-pw" password2 = "obscured-salt" `) @@ -663,6 +665,7 @@ user = "u" root = "/data" [destinations.offsite.crypt] +obscured = true password = "obscured-pw" `) cfg, err := Load(p) diff --git a/config/crypt_plaintext_test.go b/config/crypt_plaintext_test.go new file mode 100644 index 0000000..7d99cc9 --- /dev/null +++ b/config/crypt_plaintext_test.go @@ -0,0 +1,79 @@ +package config + +import ( + "strings" + "testing" +) + +// TestLoadCryptObscuresPlaintext is the default path (F2): a crypt password +// is supplied in plaintext and squirrel obscures it into rclone's form at +// load time, so the rendered value is not the plaintext and reveals back to +// it. +func TestLoadCryptObscuresPlaintext(t *testing.T) { + cfg, err := Load(writeConfig(t, ` +[destinations.offsite] +type = "sftp" +host = "h" +user = "u" +root = "/data" + +[destinations.offsite.crypt] +password = "hunter2" +password2 = "the-salt" +`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + c := cfg.Destinations["offsite"].Crypt + if c.Password == "hunter2" { + t.Fatalf("crypt password was stored plaintext, not obscured") + } + if got := revealRclone(t, c.Password); got != "hunter2" { + t.Fatalf("obscured password reveals to %q, want %q", got, "hunter2") + } + if got := revealRclone(t, c.Password2); got != "the-salt" { + t.Fatalf("obscured salt reveals to %q, want %q", got, "the-salt") + } +} + +// TestLoadCryptObscuredMarkerKeepsVerbatim covers the migration path: with +// `obscured = true` the pre-obscured values are stored (and rendered) +// verbatim. +func TestLoadCryptObscuredMarkerKeepsVerbatim(t *testing.T) { + cfg, err := Load(writeConfig(t, ` +[destinations.offsite] +type = "sftp" +host = "h" +user = "u" +root = "/data" + +[destinations.offsite.crypt] +obscured = true +password = "already-obscured" +password2 = "already-salt" +`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + c := cfg.Destinations["offsite"].Crypt + if c.Password != "already-obscured" || c.Password2 != "already-salt" { + t.Fatalf("obscured marker did not keep values verbatim: %+v", c) + } +} + +func TestLoadCryptObscuredMustBeBool(t *testing.T) { + _, err := Load(writeConfig(t, ` +[destinations.offsite] +type = "sftp" +host = "h" +user = "u" +root = "/data" + +[destinations.offsite.crypt] +obscured = "yes" +password = "hunter2" +`)) + if err == nil || !strings.Contains(err.Error(), "crypt.obscured must be a boolean") { + t.Fatalf("expected obscured-must-be-bool error, got %v", err) + } +} diff --git a/config/destinations.go b/config/destinations.go index b4e1ea7..1d73bbe 100644 --- a/config/destinations.go +++ b/config/destinations.go @@ -369,9 +369,13 @@ func optionalZstdLevel(raw map[string]any, key string, def int) (int, error) { } // resolveCrypt validates the optional `crypt` sub-table of a destination. -// A missing key yields nil (no encryption overlay). The two password -// fields go through the same secret resolution as destination credentials; -// password is required, password2 (the salt) is optional. +// A missing key yields nil (no encryption overlay). The two password fields +// go through the same secret resolution as destination credentials; +// password is required, password2 (the salt) is optional. Values are +// plaintext by default and squirrel obscures them into rclone's wire form +// at resolution time (retiring the manual `rclone obscure` step); a +// `obscured = true` marker keeps a pre-obscured value verbatim for configs +// written before this behaviour. func resolveCrypt(raw map[string]any, typ string) (*Crypt, error) { v, ok := raw["crypt"] if !ok { @@ -387,25 +391,65 @@ func resolveCrypt(raw map[string]any, typ string) (*Crypt, error) { if !ok { return nil, errors.New("crypt must be a table, e.g. [destinations.<name>.crypt]") } - password, err := resolveSecret(table, "password") + alreadyObscured, err := cryptObscuredFlag(table) if err != nil { - return nil, fmt.Errorf("crypt: %w", err) - } - if password == "" { - return nil, errors.New("crypt.password is required (rclone-obscured; generate with `rclone obscure`)") - } - password2, err := resolveSecret(table, "password2") - if err != nil { - return nil, fmt.Errorf("crypt: %w", err) + return nil, err } for k := range table { - if k != "password" && k != "password2" { + if k != "password" && k != "password2" && k != "obscured" { return nil, fmt.Errorf("crypt: unknown field %q", k) } } + password, err := resolveCryptSecret(table, "password", alreadyObscured) + if err != nil { + return nil, err + } + if password == "" { + return nil, errors.New("crypt.password is required (plaintext by default; set `obscured = true` to supply a pre-obscured value)") + } + password2, err := resolveCryptSecret(table, "password2", alreadyObscured) + if err != nil { + return nil, err + } return &Crypt{Password: password, Password2: password2}, nil } +// cryptObscuredFlag reads the optional `obscured` marker. Its default — +// false — means the password fields carry plaintext squirrel obscures +// itself; true means they already hold rclone-obscured values and must be +// passed through verbatim (the migration path for configs written before +// squirrel obscured on their behalf). +func cryptObscuredFlag(table map[string]any) (bool, error) { + v, ok := table["obscured"] + if !ok { + return false, nil + } + b, ok := v.(bool) + if !ok { + return false, errors.New("crypt.obscured must be a boolean") + } + return b, nil +} + +// resolveCryptSecret resolves one crypt password field (literal or +// { env = "VAR" }) and, unless it is flagged already-obscured, obscures the +// plaintext into rclone's wire form. An empty/absent field stays empty — an +// absent salt is legal, and an absent password is caught by the caller. +func resolveCryptSecret(table map[string]any, key string, alreadyObscured bool) (string, error) { + val, err := resolveSecret(table, key) + if err != nil { + return "", fmt.Errorf("crypt: %w", err) + } + if val == "" || alreadyObscured { + return val, nil + } + obscured, err := obscureRclone(val) + if err != nil { + return "", fmt.Errorf("crypt.%s: %w", key, err) + } + return obscured, nil +} + // validateCryptRemoteNames rejects a config where one destination's crypt // remote name is itself a declared destination — both would render an // rclone.conf section under the same name, and rclone would resolve the diff --git a/config/obscure.go b/config/obscure.go new file mode 100644 index 0000000..a030b5f --- /dev/null +++ b/config/obscure.go @@ -0,0 +1,61 @@ +package config + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/sha256" + "encoding/base64" + "fmt" +) + +// rcloneObscureCipher is the fixed AES-256 key rclone uses to obscure +// credentials in its config file. It is copied verbatim from rclone's +// fs/config/obscure package so squirrel can produce the exact same +// representation rclone's own `rclone obscure` emits and `rclone reveal` +// (and every rclone backend) decodes. +// +// This is obfuscation, not encryption: the key is public (baked into +// rclone's open source and reproduced here), so an obscured value protects +// only against casual snooping of the rendered rclone.conf, never against +// an attacker. squirrel keeps rclone.conf at mode 0600 for the real +// protection; obscuring exists solely to speak rclone's config dialect. +var rcloneObscureCipher = []byte{ + 0x9c, 0x93, 0x5b, 0x48, 0x73, 0x0a, 0x55, 0x4d, + 0x6b, 0xfd, 0x7c, 0x63, 0xc8, 0x86, 0xa9, 0x2b, + 0xd3, 0x90, 0x19, 0x8e, 0xb8, 0x12, 0x8a, 0xfb, + 0xf4, 0xde, 0x16, 0x2b, 0x8b, 0x95, 0xf6, 0x38, +} + +// obscureIVSalt namespaces the deterministic IV derivation below so the +// derived bytes can never coincide with another squirrel hash that also +// feeds plaintext into SHA-256. +const obscureIVSalt = "squirrel-rclone-obscure\x00" + +// obscureRclone returns the rclone-obscured form of a plaintext secret — +// AES-CTR under the public rcloneObscureCipher with the 16-byte IV +// prepended, base64-url encoded, exactly rclone's wire format. +// +// Unlike rclone's own obscure (which draws a random IV), squirrel derives +// the IV deterministically from the plaintext, so a given secret always +// obscures to the same bytes. That determinism is load-bearing: the +// rendered rclone.conf is compared against the file on disk and only +// rewritten when it changes (sync.WriteRcloneConfig), and an unexpected +// rewrite is a signal that a resolver regressed a credential — a random IV +// would rewrite the file on every run and destroy that signal. A +// deterministic IV loses nothing security-wise because obscure is not +// security in the first place (the key is public); deriving it per-plaintext +// (rather than a constant) still avoids reusing an AES-CTR keystream across +// two different secrets. +func obscureRclone(plaintext string) (string, error) { + block, err := aes.NewCipher(rcloneObscureCipher) + if err != nil { + return "", fmt.Errorf("obscure: init cipher: %w", err) + } + buf := make([]byte, aes.BlockSize+len(plaintext)) + iv := buf[:aes.BlockSize] + digest := sha256.Sum256([]byte(obscureIVSalt + plaintext)) + copy(iv, digest[:aes.BlockSize]) + stream := cipher.NewCTR(block, iv) + stream.XORKeyStream(buf[aes.BlockSize:], []byte(plaintext)) + return base64.RawURLEncoding.EncodeToString(buf), nil +} diff --git a/config/obscure_test.go b/config/obscure_test.go new file mode 100644 index 0000000..2754c94 --- /dev/null +++ b/config/obscure_test.go @@ -0,0 +1,85 @@ +package config + +import ( + "crypto/aes" + "crypto/cipher" + "encoding/base64" + "os/exec" + "strings" + "testing" +) + +// revealRclone is the inverse of obscureRclone, implemented in the test to +// prove the obscure output round-trips (and matches rclone's format: IV in +// the first block, AES-CTR under the shared key). +func revealRclone(t *testing.T, obscured string) string { + t.Helper() + raw, err := base64.RawURLEncoding.DecodeString(obscured) + if err != nil { + t.Fatalf("base64 decode: %v", err) + } + if len(raw) < aes.BlockSize { + t.Fatalf("obscured value too short: %d bytes", len(raw)) + } + block, err := aes.NewCipher(rcloneObscureCipher) + if err != nil { + t.Fatalf("cipher: %v", err) + } + iv := raw[:aes.BlockSize] + out := make([]byte, len(raw)-aes.BlockSize) + cipher.NewCTR(block, iv).XORKeyStream(out, raw[aes.BlockSize:]) + return string(out) +} + +func TestObscureRcloneRoundTrips(t *testing.T) { + for _, plaintext := range []string{"", "hunter2", "a much longer passphrase with spaces"} { + got, err := obscureRclone(plaintext) + if err != nil { + t.Fatalf("obscureRclone(%q): %v", plaintext, err) + } + if got == plaintext && plaintext != "" { + t.Fatalf("obscureRclone(%q) returned the plaintext unchanged", plaintext) + } + if revealed := revealRclone(t, got); revealed != plaintext { + t.Fatalf("round trip: reveal(%q) = %q, want %q", got, revealed, plaintext) + } + } +} + +// TestObscureRcloneDeterministic pins the property WriteRcloneConfig relies +// on: the same plaintext always obscures to the same bytes, so a re-render +// is byte-identical and the "unexpected rewrite is a signal" invariant +// holds. Two different plaintexts must not share an IV (keystream reuse). +func TestObscureRcloneDeterministic(t *testing.T) { + a1, _ := obscureRclone("secret") + a2, _ := obscureRclone("secret") + if a1 != a2 { + t.Fatalf("obscureRclone is not deterministic: %q vs %q", a1, a2) + } + b, _ := obscureRclone("other") + if a1[:22] == b[:22] { + t.Fatalf("distinct plaintexts share an IV prefix (keystream reuse): %q / %q", a1, b) + } +} + +// TestObscureRcloneMatchesRclone cross-checks the output against the real +// `rclone reveal`, proving squirrel speaks rclone's dialect. Skipped when +// rclone is not installed. +func TestObscureRcloneMatchesRclone(t *testing.T) { + bin, err := exec.LookPath("rclone") + if err != nil { + t.Skip("rclone not on PATH") + } + const plaintext = "correct horse battery staple" + obscured, err := obscureRclone(plaintext) + if err != nil { + t.Fatalf("obscureRclone: %v", err) + } + out, err := exec.Command(bin, "reveal", obscured).Output() + if err != nil { + t.Fatalf("rclone reveal: %v", err) + } + if got := strings.TrimRight(string(out), "\n"); got != plaintext { + t.Fatalf("rclone reveal = %q, want %q", got, plaintext) + } +} diff --git a/docs/src/content/docs/layouts/encrypted.md b/docs/src/content/docs/layouts/encrypted.md index 88aa71c..e42ee53 100644 --- a/docs/src/content/docs/layouts/encrypted.md +++ b/docs/src/content/docs/layouts/encrypted.md @@ -13,9 +13,15 @@ password = { env = "OFFSITE_CRYPT_PASSWORD" } password2 = { env = "OFFSITE_CRYPT_SALT" } # salt — optional but recommended ``` -`password` and `password2` are **rclone-obscured** values — the same -representation `rclone config` stores for its own crypt remotes. Generate one -with `rclone obscure <plaintext>`. Both accept a literal or `{ env = "VAR" }`. +`password` and `password2` are **plaintext** — a literal or `{ env = "VAR" }`. +Squirrel obscures them into rclone's on-disk representation when it renders +`rclone.conf`, so you no longer run `rclone obscure` yourself. + +:::note[Migrating an already-obscured config] +Configs written before this behaviour hold pre-obscured values. Add +`obscured = true` to the `crypt` block to keep them verbatim, or replace them +with the plaintext and drop the marker. +::: Squirrel renders two sections into its `rclone.conf` — the underlying remote plus a crypt remote wrapping it — and addresses all sync and restore transfers diff --git a/docs/src/content/docs/reference/configuration.md b/docs/src/content/docs/reference/configuration.md index 4e595e9..5ddf661 100644 --- a/docs/src/content/docs/reference/configuration.md +++ b/docs/src/content/docs/reference/configuration.md @@ -96,8 +96,9 @@ destination). See [Encrypted (crypt)](/squirrel/layouts/encrypted/). | Key | Meaning | |---|---| -| `password` | rclone-obscured value (`rclone obscure <plaintext>`). Secret. | -| `password2` | rclone-obscured salt; optional but recommended. Secret. | +| `password` | Plaintext encryption password; squirrel obscures it at render time. Secret. | +| `password2` | Plaintext salt; optional but recommended. Secret. | +| `obscured` | `true` if `password`/`password2` are already rclone-obscured (renders them verbatim; default `false`). | Filenames are **not** encrypted (`filename_encryption = off`, fixed by design). diff --git a/sync/rclone_test.go b/sync/rclone_test.go index 8615ff0..c5a991b 100644 --- a/sync/rclone_test.go +++ b/sync/rclone_test.go @@ -339,6 +339,7 @@ root = "/data" password = "transport-pw" [destinations.offsite.crypt] +obscured = true password = "obscured-pw" password2 = "obscured-salt" `) From b6bf9fdeb2348efa372d9d08827a7f901caf0bb6 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:14:30 +0000 Subject: [PATCH 02/15] agent: self-signed cert generation + startup fingerprint pin (F1) `squirrel agent cert` generates an ECDSA self-signed cert+key at the configured [agent.tls] paths and prints the sha256: pin ready to paste into peers' [nodes.X.tls] cert_fingerprint. It refuses to clobber an existing cert without --force, since regenerating changes the fingerprint and breaks every peer that pinned it. The agent also logs its own cert fingerprint on the startup line, so an operator can read the pin straight from the log. The pin is computed the same way the peer-sync client verifies it (SHA-256 over the leaf DER), via a shared agent.FingerprintCertFile helper. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- agent/agent.go | 11 +++ agent/cert.go | 127 ++++++++++++++++++++++++++++++++ agent/cert_test.go | 105 ++++++++++++++++++++++++++ cmd/squirrel/agent.go | 23 +++++- cmd/squirrel/agent_cert.go | 115 +++++++++++++++++++++++++++++ cmd/squirrel/agent_cert_test.go | 91 +++++++++++++++++++++++ 6 files changed, 468 insertions(+), 4 deletions(-) create mode 100644 agent/cert.go create mode 100644 agent/cert_test.go create mode 100644 cmd/squirrel/agent_cert.go create mode 100644 cmd/squirrel/agent_cert_test.go diff --git a/agent/agent.go b/agent/agent.go index 5084e35..afc6d8c 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -174,6 +174,17 @@ func (s *Server) Handler() http.Handler { return s.handler } // dial. func (s *Server) HasTLS() bool { return s.cfg.TLSCert != "" } +// CertFingerprint returns the sha256: pin of the agent's configured TLS +// certificate — the value the startup banner prints so an operator can see +// what peers must pin. It errors when the agent serves plain HTTP (no cert) +// or the certificate file cannot be read. +func (s *Server) CertFingerprint() (string, error) { + if s.cfg.TLSCert == "" { + return "", errors.New("agent: no TLS certificate configured") + } + return FingerprintCertFile(s.cfg.TLSCert) +} + // Addr returns the configured listen address verbatim. For `:0`-style // binds the kernel-assigned port is only knowable from the net.Listener // the caller hands to Serve; this accessor is for the startup banner. diff --git a/agent/cert.go b/agent/cert.go new file mode 100644 index 0000000..cdcc27f --- /dev/null +++ b/agent/cert.go @@ -0,0 +1,127 @@ +package agent + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "time" +) + +// SelfSignedCert bundles a freshly generated self-signed certificate, its +// private key (both PEM-encoded), and the sha256: fingerprint pin peers +// paste into their [nodes.X.tls] cert_fingerprint. +type SelfSignedCert struct { + CertPEM []byte + KeyPEM []byte + Fingerprint string +} + +// certValidity is how long a generated agent certificate stays valid. +// Generous because these certs are pinned by fingerprint, not chained to a +// CA: a peer that pins the fingerprint trusts it regardless of expiry, and +// regenerating would change the fingerprint and force every peer to re-pin. +const certValidity = 10 * 365 * 24 * time.Hour + +// GenerateSelfSignedCert mints an ECDSA P-256 self-signed certificate for a +// squirrel agent. nodeName becomes the subject common name and a DNS SAN; +// localhost and the loopback addresses are added so a locally-dialed agent +// still validates for clients that do check names. The peer-sync client +// pins the fingerprint and skips name checks, but nothing should depend on +// that. The returned Fingerprint is the sha256: pin over the certificate's +// DER bytes — the exact value the peer-sync verifier compares. +func GenerateSelfSignedCert(nodeName string) (SelfSignedCert, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return SelfSignedCert{}, fmt.Errorf("generate key: %w", err) + } + tmpl, err := certTemplate(nodeName) + if err != nil { + return SelfSignedCert{}, err + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return SelfSignedCert{}, fmt.Errorf("create certificate: %w", err) + } + keyPEM, err := encodeKeyPEM(key) + if err != nil { + return SelfSignedCert{}, err + } + return SelfSignedCert{ + CertPEM: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + KeyPEM: keyPEM, + Fingerprint: fingerprintDER(der), + }, nil +} + +// certTemplate builds the x509 template for a self-signed agent cert. +func certTemplate(nodeName string) (*x509.Certificate, error) { + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, fmt.Errorf("generate serial: %w", err) + } + cn := nodeName + if cn == "" { + cn = "squirrel-agent" + } + dns := []string{"localhost"} + if nodeName != "" { + dns = append([]string{nodeName}, dns...) + } + now := time.Now() + return &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: cn}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(certValidity), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: dns, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + }, nil +} + +func encodeKeyPEM(key *ecdsa.PrivateKey) ([]byte, error) { + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, fmt.Errorf("marshal key: %w", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), nil +} + +// fingerprintDER renders the sha256: pin over a certificate's DER bytes, +// matching the peer-sync client's VerifyConnection comparison. +func fingerprintDER(der []byte) string { + sum := sha256.Sum256(der) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// FingerprintCertFile reads a PEM certificate file and returns its sha256: +// pin — the value the agent logs at startup and peers put in +// [nodes.X.tls] cert_fingerprint. The first CERTIFICATE block (the leaf) +// wins; trailing chain blocks are ignored. +func FingerprintCertFile(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read certificate %s: %w", path, err) + } + for { + block, rest := pem.Decode(data) + if block == nil { + return "", fmt.Errorf("no CERTIFICATE block in %s", path) + } + if block.Type == "CERTIFICATE" { + return fingerprintDER(block.Bytes), nil + } + data = rest + } +} diff --git a/agent/cert_test.go b/agent/cert_test.go new file mode 100644 index 0000000..1ca5b76 --- /dev/null +++ b/agent/cert_test.go @@ -0,0 +1,105 @@ +package agent + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "os" + "path/filepath" + "regexp" + "testing" +) + +// fingerprintShape mirrors config.fingerprintRE so the generated pin is +// guaranteed pasteable into [nodes.X.tls] cert_fingerprint. +var fingerprintShape = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`) + +func TestGenerateSelfSignedCert(t *testing.T) { + c, err := GenerateSelfSignedCert("nas") + if err != nil { + t.Fatalf("GenerateSelfSignedCert: %v", err) + } + if !fingerprintShape.MatchString(c.Fingerprint) { + t.Fatalf("fingerprint %q does not match sha256:<64-hex>", c.Fingerprint) + } + // cert + key must form a usable TLS pair. + if _, err := tls.X509KeyPair(c.CertPEM, c.KeyPEM); err != nil { + t.Fatalf("cert/key do not form a valid pair: %v", err) + } + // The fingerprint must equal the SHA-256 over the leaf's DER — exactly + // what the peer-sync client compares in VerifyConnection. + block, _ := pem.Decode(c.CertPEM) + if block == nil { + t.Fatal("no PEM block in CertPEM") + } + leaf, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("parse certificate: %v", err) + } + sum := sha256.Sum256(leaf.Raw) + if want := "sha256:" + hex.EncodeToString(sum[:]); want != c.Fingerprint { + t.Fatalf("fingerprint = %q, want %q", c.Fingerprint, want) + } + if leaf.Subject.CommonName != "nas" { + t.Fatalf("CommonName = %q, want nas", leaf.Subject.CommonName) + } +} + +func TestGenerateSelfSignedCertUnique(t *testing.T) { + a, _ := GenerateSelfSignedCert("nas") + b, _ := GenerateSelfSignedCert("nas") + if a.Fingerprint == b.Fingerprint { + t.Fatal("two generations produced the same fingerprint") + } +} + +func TestFingerprintCertFile(t *testing.T) { + c, err := GenerateSelfSignedCert("laptop") + if err != nil { + t.Fatalf("GenerateSelfSignedCert: %v", err) + } + path := filepath.Join(t.TempDir(), "agent.crt") + if err := os.WriteFile(path, c.CertPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + got, err := FingerprintCertFile(path) + if err != nil { + t.Fatalf("FingerprintCertFile: %v", err) + } + if got != c.Fingerprint { + t.Fatalf("FingerprintCertFile = %q, want %q", got, c.Fingerprint) + } +} + +func TestFingerprintCertFileMissing(t *testing.T) { + if _, err := FingerprintCertFile(filepath.Join(t.TempDir(), "nope.crt")); err == nil { + t.Fatal("expected error for missing cert file") + } +} + +// TestServerCertFingerprint covers the accessor logAgentStartup uses to log +// the pin at startup: it returns the configured cert's fingerprint and +// errors for a plain-HTTP agent. +func TestServerCertFingerprint(t *testing.T) { + c, err := GenerateSelfSignedCert("nas") + if err != nil { + t.Fatalf("GenerateSelfSignedCert: %v", err) + } + path := filepath.Join(t.TempDir(), "agent.crt") + if err := os.WriteFile(path, c.CertPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + tlsSrv := &Server{cfg: Config{TLSCert: path}} + got, err := tlsSrv.CertFingerprint() + if err != nil { + t.Fatalf("CertFingerprint: %v", err) + } + if got != c.Fingerprint { + t.Fatalf("CertFingerprint = %q, want %q", got, c.Fingerprint) + } + if _, err := (&Server{cfg: Config{}}).CertFingerprint(); err == nil { + t.Fatal("expected error for a plain-HTTP agent with no cert") + } +} diff --git a/cmd/squirrel/agent.go b/cmd/squirrel/agent.go index d554c77..f8b224f 100644 --- a/cmd/squirrel/agent.go +++ b/cmd/squirrel/agent.go @@ -24,9 +24,10 @@ const agentVersion = "0.0.0-dev" // newAgentCmd returns the `squirrel agent` cobra command. It starts the // HTTP server declared by the `[agent]` config block and blocks until -// the cobra context (wired to SIGINT/SIGTERM in main) is cancelled. +// the cobra context (wired to SIGINT/SIGTERM in main) is cancelled. The +// `cert` child is a one-shot bootstrap helper and does not start a server. func newAgentCmd() *cobra.Command { - return &cobra.Command{ + cmd := &cobra.Command{ Use: "agent", Short: "Run the squirrel agent (HTTP server + scheduled audits + cadence-driven index/sync)", Args: cobra.NoArgs, @@ -34,6 +35,8 @@ func newAgentCmd() *cobra.Command { return runAgent(cmd) }, } + cmd.AddCommand(newAgentCertCmd()) + return cmd } func runAgent(cmd *cobra.Command) error { @@ -226,9 +229,21 @@ func logAgentStartup(logger *slog.Logger, srv *agent.Server, addr string) { if srv.HasTLS() { scheme = "https" } - logger.Info("agent listening", + attrs := []any{ "addr", addr, "scheme", scheme, "version", agentVersion, - ) + } + // Surfacing the fingerprint at startup (F1) lets an operator read the + // pin peers must put in [nodes.X.tls] straight from the agent's log, + // without a separate command. A read failure is non-fatal — the agent + // still serves — so it is logged as a warning attribute, not an error. + if srv.HasTLS() { + if fp, err := srv.CertFingerprint(); err == nil { + attrs = append(attrs, "cert_fingerprint", fp) + } else { + attrs = append(attrs, "cert_fingerprint_error", err.Error()) + } + } + logger.Info("agent listening", attrs...) } diff --git a/cmd/squirrel/agent_cert.go b/cmd/squirrel/agent_cert.go new file mode 100644 index 0000000..164224e --- /dev/null +++ b/cmd/squirrel/agent_cert.go @@ -0,0 +1,115 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/agent" + "github.com/mbertschler/squirrel/config" +) + +// newAgentCertCmd returns `squirrel agent cert`: a bootstrap helper that +// generates the agent's self-signed TLS certificate + key at the paths the +// [agent.tls] block configures and prints the sha256: pin peers put in +// their [nodes.X.tls] cert_fingerprint (F1). It is a deliberate change (it +// writes key material), not introspection, and refuses to clobber an +// existing certificate without --force: regenerating changes the +// fingerprint and breaks every peer that already pinned it. +func newAgentCertCmd() *cobra.Command { + var force bool + cmd := &cobra.Command{ + Use: "cert", + Short: "Generate the agent's self-signed TLS cert+key and print its sha256: pin", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return runAgentCert(cmd, force) + }, + } + cmd.Flags().BoolVar(&force, "force", false, "overwrite an existing cert/key (changes the fingerprint — every peer must re-pin)") + return cmd +} + +func runAgentCert(cmd *cobra.Command, force bool) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + certPath, keyPath, err := agentCertPaths(cfg) + if err != nil { + return err + } + if err := guardExistingCert(certPath, keyPath, force); err != nil { + return err + } + generated, err := agent.GenerateSelfSignedCert(cfg.NodeName) + if err != nil { + return err + } + if err := writeCertFiles(certPath, keyPath, generated); err != nil { + return err + } + printCertResult(cmd, cfg, certPath, keyPath, generated.Fingerprint) + return nil +} + +// agentCertPaths pulls the configured cert/key paths, erroring with a +// pointer at the config when the [agent] block or its tls paths are absent — +// there is nowhere to write otherwise. +func agentCertPaths(cfg *config.Config) (certPath, keyPath string, err error) { + if cfg.Agent == nil { + return "", "", fmt.Errorf("no [agent] block in %s", cfg.Path) + } + if cfg.Agent.TLSCert == "" || cfg.Agent.TLSKey == "" { + return "", "", fmt.Errorf("no [agent.tls] cert/key configured in %s — set both paths before generating", cfg.Path) + } + return cfg.Agent.TLSCert, cfg.Agent.TLSKey, nil +} + +// guardExistingCert refuses to overwrite either file unless force is set. +func guardExistingCert(certPath, keyPath string, force bool) error { + if force { + return nil + } + for _, p := range []string{certPath, keyPath} { + _, err := os.Stat(p) + switch { + case err == nil: + return fmt.Errorf("%s already exists — refusing to overwrite (pass --force to regenerate; every peer that pinned the old fingerprint must re-pin)", p) + case !errors.Is(err, os.ErrNotExist): + return fmt.Errorf("stat %s: %w", p, err) + } + } + return nil +} + +// writeCertFiles writes both files owner-only (0600) under 0700 parent +// directories. The certificate is not secret, but keeping it 0600 alongside +// the key is harmless — peers obtain the fingerprint from the pairing helper +// and the startup log, never by reading this file. +func writeCertFiles(certPath, keyPath string, c agent.SelfSignedCert) error { + for path, data := range map[string][]byte{certPath: c.CertPEM, keyPath: c.KeyPEM} { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("create directory for %s: %w", path, err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + } + return nil +} + +func printCertResult(cmd *cobra.Command, cfg *config.Config, certPath, keyPath, fingerprint string) { + out := cmd.OutOrStdout() + fmt.Fprintf(out, "wrote certificate %s\n", certPath) + fmt.Fprintf(out, "wrote private key %s\n\n", keyPath) + name := cfg.NodeName + if name == "" { + name = "<this-node>" + } + fmt.Fprintf(out, "Peers pin this node (%s) by adding to their config:\n\n", name) + fmt.Fprintf(out, "[nodes.%s.tls]\ncert_fingerprint = %q\n", name, fingerprint) +} diff --git a/cmd/squirrel/agent_cert_test.go b/cmd/squirrel/agent_cert_test.go new file mode 100644 index 0000000..d13bbf2 --- /dev/null +++ b/cmd/squirrel/agent_cert_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mbertschler/squirrel/agent" +) + +// writeAgentCertConfig writes a config with an [agent] block whose tls +// cert/key point at paths inside a fresh temp dir, and returns the config +// path plus the two target paths. +func writeAgentCertConfig(t *testing.T) (cfgPath, certPath, keyPath string) { + t.Helper() + dir := t.TempDir() + certPath = filepath.Join(dir, "agent.crt") + keyPath = filepath.Join(dir, "agent.key") + cfgPath = filepath.Join(dir, "config.toml") + body := fmt.Sprintf(""+ + "node_name = \"nas\"\n\n"+ + "[agent]\nlisten = \"127.0.0.1:8443\"\nauth = { token = \"tok\" }\n\n"+ + "[agent.tls]\ncert = %q\nkey = %q\n", certPath, keyPath) + if err := os.WriteFile(cfgPath, []byte(body), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return cfgPath, certPath, keyPath +} + +func TestCLIAgentCertGenerates(t *testing.T) { + cfgPath, certPath, keyPath := writeAgentCertConfig(t) + + out := runCLI(t, "--config", cfgPath, "agent", "cert") + for _, p := range []string{certPath, keyPath} { + if _, err := os.Stat(p); err != nil { + t.Fatalf("expected %s to be written: %v", p, err) + } + } + want, err := agent.FingerprintCertFile(certPath) + if err != nil { + t.Fatalf("FingerprintCertFile: %v", err) + } + if !strings.Contains(out, want) { + t.Fatalf("output missing generated fingerprint %q:\n%s", want, out) + } + if !strings.Contains(out, "[nodes.nas.tls]") || !strings.Contains(out, "cert_fingerprint") { + t.Fatalf("output missing pasteable pin snippet:\n%s", out) + } +} + +func TestCLIAgentCertRefusesOverwrite(t *testing.T) { + cfgPath, _, _ := writeAgentCertConfig(t) + runCLI(t, "--config", cfgPath, "agent", "cert") + + out, err := runCLIExpectErr(t, "--config", cfgPath, "agent", "cert") + if !strings.Contains(err.Error(), "already exists") { + t.Fatalf("expected refuse-overwrite error, got %v\n%s", err, out) + } +} + +func TestCLIAgentCertForceRegenerates(t *testing.T) { + cfgPath, certPath, _ := writeAgentCertConfig(t) + runCLI(t, "--config", cfgPath, "agent", "cert") + first, err := agent.FingerprintCertFile(certPath) + if err != nil { + t.Fatalf("fingerprint: %v", err) + } + runCLI(t, "--config", cfgPath, "agent", "cert", "--force") + second, err := agent.FingerprintCertFile(certPath) + if err != nil { + t.Fatalf("fingerprint: %v", err) + } + if first == second { + t.Fatal("--force did not regenerate the certificate (fingerprint unchanged)") + } +} + +func TestCLIAgentCertNoTLSConfigured(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.toml") + body := "[agent]\nlisten = \"127.0.0.1:8443\"\nauth = { token = \"tok\" }\n" + if err := os.WriteFile(cfgPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + _, err := runCLIExpectErr(t, "--config", cfgPath, "agent", "cert") + if !strings.Contains(err.Error(), "no [agent.tls] cert/key configured") { + t.Fatalf("expected no-tls error, got %v", err) + } +} From 503ccfb4073fdb287031ff5735b84ae43e05a4c2 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:14:38 +0000 Subject: [PATCH 03/15] cmd: add `config check` and `node pair` bootstrap helpers (F3, F4, F34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `squirrel config check` parses and resolves the whole config (env vars included), stats every volume path and node byte-path, validates offload_requires against target capabilities, and prints the affirmative "N volumes, M destinations, K nodes — all resolvable" summary. It is strictly read-only (a question, per ux-principle 2): it never opens the store or creates a path. Missing/empty volume paths (F4/F8), missing node byte-paths (F34), and structurally-unsatisfiable offload targets — a mirror-layout crypt destination (F21) — are surfaced; problems exit non-zero, advisory warnings do not. `squirrel node pair <peer>` generates the two bearer tokens a bidirectional peer relationship needs and emits the matching config halves for both machines, with each token already placed in the two slots that must agree — killing the F3 four-binding token-matrix error class. It fills this node's endpoint/fingerprint where known and leaves clearly-marked placeholders for the peer-side facts it cannot know. It prints only; it never edits either config. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/config.go | 15 ++ cmd/squirrel/config_check.go | 285 ++++++++++++++++++++++++++++++ cmd/squirrel/config_check_test.go | 123 +++++++++++++ cmd/squirrel/node.go | 14 ++ cmd/squirrel/node_pair.go | 167 +++++++++++++++++ cmd/squirrel/node_pair_test.go | 115 ++++++++++++ cmd/squirrel/root.go | 2 + 7 files changed, 721 insertions(+) create mode 100644 cmd/squirrel/config.go create mode 100644 cmd/squirrel/config_check.go create mode 100644 cmd/squirrel/config_check_test.go create mode 100644 cmd/squirrel/node.go create mode 100644 cmd/squirrel/node_pair.go create mode 100644 cmd/squirrel/node_pair_test.go diff --git a/cmd/squirrel/config.go b/cmd/squirrel/config.go new file mode 100644 index 0000000..233b797 --- /dev/null +++ b/cmd/squirrel/config.go @@ -0,0 +1,15 @@ +package main + +import "github.com/spf13/cobra" + +// newConfigCmd returns the `squirrel config` parent command. Its children +// are introspection over the config file itself (currently `check`), kept +// distinct from the DB-facing question commands. +func newConfigCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "config", + Short: "Inspect the squirrel configuration", + } + cmd.AddCommand(newConfigCheckCmd()) + return cmd +} diff --git a/cmd/squirrel/config_check.go b/cmd/squirrel/config_check.go new file mode 100644 index 0000000..029a6be --- /dev/null +++ b/cmd/squirrel/config_check.go @@ -0,0 +1,285 @@ +package main + +import ( + "errors" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/config" +) + +// Per-check status markers. statusFail is fatal (the command exits +// non-zero); statusWarn is advisory (surfaced, but the config is usable); +// statusOK is a clean line. +const ( + statusOK = "ok" + statusWarn = "warn" + statusFail = "FAIL" +) + +// newConfigCheckCmd returns `squirrel config check`: parse + resolve the +// whole config (env vars included), stat volume paths and node byte-paths, +// validate offload policies against target capabilities, and print an +// affirmative summary (F4). It is strictly read-only — it never creates a +// path, cert, or database — per the "CLI is for change and for questions" +// principle: this is a question. +func newConfigCheckCmd() *cobra.Command { + return &cobra.Command{ + Use: "check", + Short: "Parse and resolve the config, stat its paths, and report what it declares", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return runConfigCheck(cmd) + }, + } +} + +// checkTally accumulates fatal/advisory counts across the report sections. +type checkTally struct { + fails int + warns int +} + +func (t *checkTally) add(status string) { + switch status { + case statusFail: + t.fails++ + case statusWarn: + t.warns++ + } +} + +func runConfigCheck(cmd *cobra.Command) error { + // requireConfig both loads and resolves: a parse error, an unknown + // field, or an unset { env = "VAR" } secret surfaces here as the + // check's failure, which is exactly the "did everything resolve?" + // question this command answers. + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "config: %s\n\n", cfg.Path) + + var tally checkTally + checkVolumes(cfg, out, &tally) + checkDestinations(cfg, out) + checkNodes(cfg, out, &tally) + checkOffloadPolicies(cfg, out, &tally) + + return printSummary(out, cfg, tally) +} + +// printCheckLine renders one aligned status line: " ok name detail". +func printCheckLine(out io.Writer, status, name, detail string) { + if detail == "" { + fmt.Fprintf(out, " %-4s %s\n", status, name) + return + } + fmt.Fprintf(out, " %-4s %s %s\n", status, name, detail) +} + +func checkVolumes(cfg *config.Config, out io.Writer, tally *checkTally) { + fmt.Fprintf(out, "volumes (%d)\n", len(cfg.Volumes)) + for _, name := range sortedKeys(cfg.Volumes) { + vol := cfg.Volumes[name] + status, detail := statVolumePath(vol.Path) + tally.add(status) + printCheckLine(out, status, name, joinDetail(vol.Path, detail)) + } +} + +func checkDestinations(cfg *config.Config, out io.Writer) { + fmt.Fprintf(out, "destinations (%d)\n", len(cfg.Destinations)) + for _, name := range sortedKeys(cfg.Destinations) { + d := cfg.Destinations[name] + parts := []string{d.Type, d.Layout} + if d.Crypt != nil { + parts = append(parts, "crypt") + } + // Destinations resolved (Load succeeded), so they are all ok here; + // their remote reachability is a sync-time concern, not something a + // read-only check probes. + printCheckLine(out, statusOK, name, strings.Join(parts, " ")) + } +} + +func checkNodes(cfg *config.Config, out io.Writer, tally *checkTally) { + fmt.Fprintf(out, "nodes (%d)\n", len(cfg.Nodes)) + for _, name := range sortedKeys(cfg.Nodes) { + n := cfg.Nodes[name] + status, detail := statNodeBytePath(n.Path) + tally.add(status) + endpoint := n.Endpoint.String() + printCheckLine(out, status, name, joinDetail(endpoint, "byte-path "+n.Path+detail)) + } +} + +func checkOffloadPolicies(cfg *config.Config, out io.Writer, tally *checkTally) { + names := offloadVolumeNames(cfg) + if len(names) == 0 { + return + } + fmt.Fprintf(out, "offload policies (%d)\n", len(names)) + for _, name := range names { + for _, target := range cfg.Volumes[name].OffloadRequires { + status, detail := offloadTargetStatus(cfg, target) + tally.add(status) + printCheckLine(out, status, name+" → "+target, detail) + } + } +} + +// offloadTargetStatus classifies one offload_requires target. A destination +// that can never yield offload-grade evidence (a mirror-layout crypt +// destination, F21) is a fatal misconfiguration — the gate would wait +// forever. A peer node yields peer-blake3 evidence directly. A name that is +// neither is an external target whose evidence must arrive relayed via a +// peer; that is legitimate, so it is reported without alarm. +func offloadTargetStatus(cfg *config.Config, target string) (status, detail string) { + if d, ok := cfg.Destinations[target]; ok { + if capable, reason := d.CanEverGateOffload(); !capable { + return statusFail, "never yields offload evidence: " + reason + } + return statusOK, "destination" + } + if _, ok := cfg.Nodes[target]; ok { + return statusOK, "peer node (peer-blake3 evidence)" + } + return statusOK, "external target — evidence must arrive relayed via a peer" +} + +func printSummary(out io.Writer, cfg *config.Config, tally checkTally) error { + fmt.Fprintln(out) + counts := fmt.Sprintf("%d volumes, %d destinations, %d nodes", + len(cfg.Volumes), len(cfg.Destinations), len(cfg.Nodes)) + switch { + case tally.fails > 0: + fmt.Fprintf(out, "%s — %s, %s\n", counts, + plural(tally.fails, "problem"), plural(tally.warns, "warning")) + return fmt.Errorf("config check found %d problem(s)", tally.fails) + case tally.warns > 0: + fmt.Fprintf(out, "%s — resolvable, %s\n", counts, plural(tally.warns, "warning")) + return nil + default: + fmt.Fprintf(out, "%s — all resolvable\n", counts) + return nil + } +} + +// statVolumePath stats a local volume root. A missing path or non-directory +// is fatal; an empty directory is advisory (F8: a typo'd or unmounted path +// looks identical to a genuinely new volume, so it must be flagged, not +// hard-failed). +func statVolumePath(path string) (status, detail string) { + info, err := os.Stat(path) + switch { + case errors.Is(err, os.ErrNotExist): + return statusFail, "does not exist" + case err != nil: + return statusFail, err.Error() + case !info.IsDir(): + return statusFail, "not a directory" + } + empty, err := dirIsEmpty(path) + if err != nil { + return statusWarn, "unreadable: " + err.Error() + } + if empty { + return statusWarn, "empty directory — new volume or wrong mount?" + } + return statusOK, "" +} + +// statNodeBytePath stats a node's rclone byte-path (F34). An rclone remote +// spec ("remote:path") is not a local path, so it is reported as unchecked +// rather than statted. A missing local mount is advisory: the share may +// legitimately be down when the check runs, but a silent absence is exactly +// what turns into undiagnosable transfer errors, so it is surfaced. The +// returned detail is a suffix appended after the path. +func statNodeBytePath(path string) (status, detail string) { + if isRcloneRemoteSpec(path) { + return statusOK, " (rclone remote spec — not checked)" + } + info, err := os.Stat(path) + switch { + case errors.Is(err, os.ErrNotExist): + return statusWarn, " (does not exist — mount not up?)" + case err != nil: + return statusWarn, " (" + err.Error() + ")" + case !info.IsDir(): + return statusWarn, " (not a directory)" + } + return statusOK, "" +} + +// isRcloneRemoteSpec reports whether p is an rclone "remote:path" reference +// rather than a filesystem path. An absolute path (leading /) is always a +// filesystem path; otherwise a leading "name:" segment marks a remote. +func isRcloneRemoteSpec(p string) bool { + if strings.HasPrefix(p, "/") { + return false + } + i := strings.IndexByte(p, ':') + return i > 0 +} + +// dirIsEmpty reports whether dir contains no entries, reading just one name +// rather than the whole listing. +func dirIsEmpty(dir string) (bool, error) { + f, err := os.Open(dir) + if err != nil { + return false, err + } + defer func() { _ = f.Close() }() + _, err = f.Readdirnames(1) + if errors.Is(err, io.EOF) { + return true, nil + } + return false, err +} + +// offloadVolumeNames returns the sorted names of volumes that declare an +// offload policy. +func offloadVolumeNames(cfg *config.Config) []string { + var names []string + for name, vol := range cfg.Volumes { + if len(vol.OffloadRequires) > 0 { + names = append(names, name) + } + } + sort.Strings(names) + return names +} + +// joinDetail appends a parenthetical/extra detail to a base string when the +// detail is non-empty. +func joinDetail(base, detail string) string { + if detail == "" { + return base + } + return base + " " + detail +} + +// plural renders "N thing" / "N things" for a count. +func plural(n int, thing string) string { + if n == 1 { + return fmt.Sprintf("%d %s", n, thing) + } + return fmt.Sprintf("%d %ss", n, thing) +} + +// sortedKeys returns the map keys sorted, for stable report ordering. +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/cmd/squirrel/config_check_test.go b/cmd/squirrel/config_check_test.go new file mode 100644 index 0000000..5d5514e --- /dev/null +++ b/cmd/squirrel/config_check_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// writeCheckConfig writes body to a config.toml in a fresh temp dir and +// returns its path. +func writeCheckConfig(t *testing.T, body string) string { + t.Helper() + configPath := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return configPath +} + +func TestCLIConfigCheckAffirmative(t *testing.T) { + dir := t.TempDir() + photos := filepath.Join(dir, "photos") + dest := filepath.Join(dir, "dest") + nodePath := filepath.Join(dir, "peer-mount") + for _, d := range []string{photos, dest, nodePath} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + writeTestFile(t, filepath.Join(photos, "a.jpg"), "x") + + body := "" + + "node_name = \"thisnode\"\n\n" + + "[volumes.photos]\npath = \"" + photos + "\"\nsync_to = [\"scratch\"]\n\n" + + "[destinations.scratch]\ntype = \"local\"\nroot = \"" + dest + "\"\n\n" + + "[nodes.peer]\nendpoint = \"https://peer.home:8443\"\npath = \"" + nodePath + "\"\n" + + "[nodes.peer.auth]\nbearer = \"tok\"\n" + cfgPath := writeCheckConfig(t, body) + + out := runCLI(t, "--config", cfgPath, "config", "check") + if !strings.Contains(out, "1 volumes, 1 destinations, 1 nodes — all resolvable") { + t.Fatalf("missing affirmative summary:\n%s", out) + } +} + +func TestCLIConfigCheckEmptyVolumeWarns(t *testing.T) { + dir := t.TempDir() + photos := filepath.Join(dir, "photos") + if err := os.MkdirAll(photos, 0o755); err != nil { + t.Fatal(err) + } + body := "[volumes.photos]\npath = \"" + photos + "\"\n" + cfgPath := writeCheckConfig(t, body) + + out := runCLI(t, "--config", cfgPath, "config", "check") + if !strings.Contains(out, "warn") || !strings.Contains(out, "empty") { + t.Fatalf("expected empty-volume warning:\n%s", out) + } + if !strings.Contains(out, "resolvable, 1 warning") { + t.Fatalf("expected warning summary:\n%s", out) + } +} + +func TestCLIConfigCheckMissingVolumeFails(t *testing.T) { + dir := t.TempDir() + missing := filepath.Join(dir, "not-there") + body := "[volumes.photos]\npath = \"" + missing + "\"\n" + cfgPath := writeCheckConfig(t, body) + + out, err := runCLIExpectErr(t, "--config", cfgPath, "config", "check") + if !strings.Contains(out, "does not exist") { + t.Fatalf("expected does-not-exist line:\n%s", out) + } + if !strings.Contains(err.Error(), "problem") { + t.Fatalf("expected problem error, got %v", err) + } +} + +// TestCLIConfigCheckOffloadMirrorCrypt exercises the offload_requires +// satisfiability check (F21): a mirror-layout crypt destination can never +// yield offload evidence, so naming it is a fatal misconfiguration. +func TestCLIConfigCheckOffloadMirrorCrypt(t *testing.T) { + dir := t.TempDir() + docs := filepath.Join(dir, "docs") + if err := os.MkdirAll(docs, 0o755); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(docs, "d.txt"), "x") + body := "" + + "[volumes.docs]\npath = \"" + docs + "\"\noffload_requires = [\"cloudbox\"]\n\n" + + "[destinations.cloudbox]\ntype = \"sftp\"\nhost = \"h\"\nuser = \"u\"\nroot = \"/data\"\n" + + "[destinations.cloudbox.crypt]\npassword = \"pw\"\n" + cfgPath := writeCheckConfig(t, body) + + out, err := runCLIExpectErr(t, "--config", cfgPath, "config", "check") + if !strings.Contains(out, "never yields offload evidence") { + t.Fatalf("expected offload-evidence problem:\n%s", out) + } + if !strings.Contains(err.Error(), "problem") { + t.Fatalf("expected problem error, got %v", err) + } +} + +// TestCLIConfigCheckReadOnly guards principle 2: `config check` must not +// mutate anything. The config declares a db path that must never be +// created by a check. +func TestCLIConfigCheckReadOnly(t *testing.T) { + dir := t.TempDir() + photos := filepath.Join(dir, "photos") + if err := os.MkdirAll(photos, 0o755); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(photos, "a.jpg"), "x") + dbPath := filepath.Join(dir, "index.db") + body := "db = \"" + dbPath + "\"\n\n[volumes.photos]\npath = \"" + photos + "\"\n" + cfgPath := writeCheckConfig(t, body) + + runCLI(t, "--config", cfgPath, "config", "check") + if _, err := os.Stat(dbPath); !os.IsNotExist(err) { + t.Fatalf("config check created the database at %s (must be read-only)", dbPath) + } +} diff --git a/cmd/squirrel/node.go b/cmd/squirrel/node.go new file mode 100644 index 0000000..27091ae --- /dev/null +++ b/cmd/squirrel/node.go @@ -0,0 +1,14 @@ +package main + +import "github.com/spf13/cobra" + +// newNodeCmd returns the `squirrel node` parent command grouping the +// peer-relationship bootstrap helpers. +func newNodeCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "node", + Short: "Manage peer node relationships", + } + cmd.AddCommand(newNodePairCmd()) + return cmd +} diff --git a/cmd/squirrel/node_pair.go b/cmd/squirrel/node_pair.go new file mode 100644 index 0000000..886c880 --- /dev/null +++ b/cmd/squirrel/node_pair.go @@ -0,0 +1,167 @@ +package main + +import ( + "crypto/rand" + "encoding/base64" + "fmt" + "io" + "regexp" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/agent" + "github.com/mbertschler/squirrel/config" +) + +// nodeNameRE mirrors config's volume/destination/node name rule so a bad +// peer name is rejected before it is templated into a TOML key. Kept local +// (config does not export its regexp) and deliberately identical. +var nodeNameRE = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]*$`) + +// nodePairOpts carries the peer-side facts this node cannot know on its own, +// supplied via flags; each empty value renders as a clearly-marked +// placeholder the operator fills in. +type nodePairOpts struct { + localEndpoint string + peerEndpoint string + peerPath string + peerFingerprint string +} + +// newNodePairCmd returns `squirrel node pair <peer>`: it generates the two +// bearer tokens a bidirectional peer relationship needs and emits the +// matching config halves for *both* machines, with each token already placed +// in the two slots that must agree. This kills the F3 token-matrix error +// class (four cross-referenced bindings written by hand). It prints only — +// it never edits either config — so the operator reviews and pastes. +func newNodePairCmd() *cobra.Command { + var opts nodePairOpts + cmd := &cobra.Command{ + Use: "pair <peer>", + Short: "Emit matching config halves (tokens, endpoints, fingerprints) for a peer relationship", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runNodePair(cmd, args[0], opts) + }, + } + cmd.Flags().StringVar(&opts.localEndpoint, "local-endpoint", "", "this node's agent endpoint as the peer dials it (e.g. https://nas.home:8443)") + cmd.Flags().StringVar(&opts.peerEndpoint, "peer-endpoint", "", "the peer's agent endpoint") + cmd.Flags().StringVar(&opts.peerPath, "peer-path", "", "the byte-path where this node mounts the peer's data") + cmd.Flags().StringVar(&opts.peerFingerprint, "peer-fingerprint", "", "the peer's TLS cert fingerprint (sha256:...)") + return cmd +} + +func runNodePair(cmd *cobra.Command, peer string, opts nodePairOpts) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + local := cfg.NodeName + if local == "" { + return fmt.Errorf("node_name is not set in %s — set it before pairing (it names this node to peers)", cfg.Path) + } + if !nodeNameRE.MatchString(peer) { + return fmt.Errorf("peer name %q is invalid (must match %s)", peer, nodeNameRE) + } + if peer == local { + return fmt.Errorf("peer name %q equals this node's name — a node cannot pair with itself", peer) + } + // tokenLtoP is presented when the local node calls the peer; tokenPtoL + // the reverse. Each appears in exactly two slots, cross-matched below. + tokenLtoP, err := newPairToken() + if err != nil { + return err + } + tokenPtoL, err := newPairToken() + if err != nil { + return err + } + printNodePair(cmd.OutOrStdout(), cfg, peer, tokenLtoP, tokenPtoL, opts) + return nil +} + +func printNodePair(out io.Writer, cfg *config.Config, peer, tokenLtoP, tokenPtoL string, opts nodePairOpts) { + local := cfg.NodeName + fmt.Fprintf(out, "# Node pairing: %s <-> %s\n", local, peer) + fmt.Fprintln(out, "# Two bearer tokens were generated and placed so the four bindings already") + fmt.Fprintln(out, "# cross-match. Paste each half into the named machine's config, fill any") + fmt.Fprintln(out, "# <...> placeholders, then restart both agents.") + fmt.Fprintln(out) + + fmt.Fprintf(out, "# ===== on %s (this machine) — add to %s =====\n\n", local, cfg.Path) + writeNodeBlock(out, peer, placeholder(opts.peerEndpoint, "https://<"+peer+"-host>:8443"), + placeholder(opts.peerPath, "<path where "+local+" mounts "+peer+"'s data>"), + tokenLtoP, placeholder(opts.peerFingerprint, "sha256:<"+peer+"'s fingerprint — run `squirrel agent cert` on "+peer+">")) + fmt.Fprintf(out, "\n[agent.auth.peers.%s]\nbearer = %q\n\n", peer, tokenPtoL) + + fmt.Fprintf(out, "# ===== on %s (the peer) — add to its config =====\n\n", peer) + writeNodeBlock(out, local, placeholder(opts.localEndpoint, localEndpointGuess(cfg)), + "<path where "+peer+" mounts "+local+"'s data>", + tokenPtoL, localFingerprint(cfg)) + fmt.Fprintf(out, "\n[agent.auth.peers.%s]\nbearer = %q\n", local, tokenLtoP) +} + +// writeNodeBlock renders one [nodes.<name>] block with its auth + tls +// sub-tables. +func writeNodeBlock(out io.Writer, name, endpoint, path, bearer, fingerprint string) { + fmt.Fprintf(out, "[nodes.%s]\n", name) + fmt.Fprintf(out, "endpoint = %q\n", endpoint) + fmt.Fprintf(out, "path = %q\n", path) + fmt.Fprintf(out, "[nodes.%s.auth]\n", name) + fmt.Fprintf(out, "bearer = %q\n", bearer) + fmt.Fprintf(out, "[nodes.%s.tls]\n", name) + fmt.Fprintf(out, "cert_fingerprint = %q\n", fingerprint) +} + +// localFingerprint returns this node's cert fingerprint when a cert already +// exists at the configured path, else a placeholder pointing at `agent cert`. +func localFingerprint(cfg *config.Config) string { + if cfg.Agent == nil || cfg.Agent.TLSCert == "" { + return "sha256:<this node has no [agent.tls] cert configured>" + } + fp, err := agent.FingerprintCertFile(cfg.Agent.TLSCert) + if err != nil { + return "sha256:<run `squirrel agent cert` on " + cfg.NodeName + " first>" + } + return fp +} + +// localEndpointGuess suggests this node's dialable endpoint from the agent's +// listen port; the host is unknowable from a bind address, so it stays a +// placeholder the operator completes. +func localEndpointGuess(cfg *config.Config) string { + port := "8443" + if cfg.Agent != nil && cfg.Agent.Listen != "" { + if _, p, ok := splitHostPort(cfg.Agent.Listen); ok && p != "" { + port = p + } + } + return "https://<" + cfg.NodeName + "-host>:" + port +} + +// splitHostPort splits "host:port" without erroring on a bare ":port" or an +// unbracketed IPv6, which net.SplitHostPort rejects; we only need the port. +func splitHostPort(addr string) (host, port string, ok bool) { + for i := len(addr) - 1; i >= 0; i-- { + if addr[i] == ':' { + return addr[:i], addr[i+1:], true + } + } + return addr, "", false +} + +func placeholder(value, fallback string) string { + if value != "" { + return value + } + return fallback +} + +// newPairToken mints a 256-bit URL-safe bearer token. +func newPairToken() (string, error) { + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return "", fmt.Errorf("generate pairing token: %w", err) + } + return base64.RawURLEncoding.EncodeToString(buf), nil +} diff --git a/cmd/squirrel/node_pair_test.go b/cmd/squirrel/node_pair_test.go new file mode 100644 index 0000000..949d6b6 --- /dev/null +++ b/cmd/squirrel/node_pair_test.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mbertschler/squirrel/agent" +) + +// bearerValues pulls the quoted value from every `bearer = "..."` line in +// the pairing output. +func bearerValues(out string) []string { + var vals []string + for _, line := range strings.Split(out, "\n") { + trimmed := strings.TrimSpace(line) + if rest, ok := strings.CutPrefix(trimmed, "bearer = "); ok { + vals = append(vals, strings.Trim(rest, `"`)) + } + } + return vals +} + +func TestCLINodePairEmitsBothHalves(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.toml") + if err := os.WriteFile(cfgPath, []byte("node_name = \"nas\"\n"), 0o600); err != nil { + t.Fatal(err) + } + out := runCLI(t, "--config", cfgPath, "node", "pair", "laptop") + + for _, want := range []string{ + "# ===== on nas", "# ===== on laptop", + "[nodes.laptop]", "[nodes.nas]", + "[agent.auth.peers.laptop]", "[agent.auth.peers.nas]", + } { + if !strings.Contains(out, want) { + t.Fatalf("pairing output missing %q:\n%s", want, out) + } + } + + // The four bearer bindings must reduce to exactly two distinct tokens, + // each used twice — the cross-match that kills the F3 error class. + vals := bearerValues(out) + if len(vals) != 4 { + t.Fatalf("expected 4 bearer lines, got %d:\n%s", len(vals), out) + } + counts := map[string]int{} + for _, v := range vals { + if len(v) < 20 { + t.Fatalf("bearer token looks too short: %q", v) + } + counts[v]++ + } + if len(counts) != 2 { + t.Fatalf("expected 2 distinct tokens, got %d: %v", len(counts), counts) + } + for tok, n := range counts { + if n != 2 { + t.Fatalf("token %q appears %d times, want 2 (cross-match broken)", tok, n) + } + } +} + +func TestCLINodePairRequiresNodeName(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.toml") + if err := os.WriteFile(cfgPath, []byte("[volumes.x]\npath = \"/tmp\"\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err := runCLIExpectErr(t, "--config", cfgPath, "node", "pair", "laptop") + if !strings.Contains(err.Error(), "node_name is not set") { + t.Fatalf("expected node_name error, got %v", err) + } +} + +func TestCLINodePairRejectsSelf(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.toml") + if err := os.WriteFile(cfgPath, []byte("node_name = \"nas\"\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err := runCLIExpectErr(t, "--config", cfgPath, "node", "pair", "nas") + if !strings.Contains(err.Error(), "cannot pair with itself") { + t.Fatalf("expected self-pair error, got %v", err) + } +} + +// TestCLINodePairEmbedsLocalFingerprint checks that when this node already +// has a cert, its real fingerprint is baked into the peer's half rather than +// a placeholder. +func TestCLINodePairEmbedsLocalFingerprint(t *testing.T) { + dir := t.TempDir() + certPath := filepath.Join(dir, "agent.crt") + keyPath := filepath.Join(dir, "agent.key") + cfgPath := filepath.Join(dir, "config.toml") + body := fmt.Sprintf(""+ + "node_name = \"nas\"\n\n"+ + "[agent]\nlisten = \"127.0.0.1:8443\"\nauth = { token = \"tok\" }\n\n"+ + "[agent.tls]\ncert = %q\nkey = %q\n", certPath, keyPath) + if err := os.WriteFile(cfgPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + runCLI(t, "--config", cfgPath, "agent", "cert") + fp, err := agent.FingerprintCertFile(certPath) + if err != nil { + t.Fatalf("fingerprint: %v", err) + } + out := runCLI(t, "--config", cfgPath, "node", "pair", "laptop") + if !strings.Contains(out, fp) { + t.Fatalf("pairing output missing this node's real fingerprint %q:\n%s", fp, out) + } +} diff --git a/cmd/squirrel/root.go b/cmd/squirrel/root.go index 4bb9def..af77560 100644 --- a/cmd/squirrel/root.go +++ b/cmd/squirrel/root.go @@ -55,6 +55,8 @@ func newRootCmd() *cobra.Command { root.AddCommand(newPeerSyncCmd()) root.AddCommand(newTUICmd()) root.AddCommand(newDBCmd()) + root.AddCommand(newConfigCmd()) + root.AddCommand(newNodeCmd()) return root } From 973b81073547e6b317fcd17339943930e87189d8 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:17:23 +0000 Subject: [PATCH 04/15] design: mark F1-F4 fixed and F34 partially addressed by #171 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow the friction log's own convention of striking closed entries with a PR reference: F1 (agent cert + startup fingerprint), F2 (plaintext crypt), F3 (node pair), F4 (config check). F34 is only partially closed — config check surfaces a missing node byte-path, but load-time validation is still absent — so it is annotated, not struck. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- design/friction-log.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/design/friction-log.md b/design/friction-log.md index 5b563db..496c68f 100644 --- a/design/friction-log.md +++ b/design/friction-log.md @@ -21,7 +21,7 @@ rclone 1.74.1, SeaweedFS 3.80, kopia 0.21.1, testbed of 2026-07-23. ## Checkpoint 1 — bootstrap day -**F1 · S2 — TLS cert + fingerprint setup is entirely DIY.** Pinning is +**F1 · S2 — ~~TLS cert + fingerprint setup is entirely DIY.~~ (fixed in #171)** Pinning is the documented trust anchor for LAN agents, but there is no help producing its ingredients: the operator must know the right openssl incantations for cert+key *and* for the `sha256:<hex of DER>` pin @@ -29,13 +29,13 @@ incantations for cert+key *and* for the `sha256:<hex of DER>` pin nor any command covers this. A `squirrel agent cert` (generate) + printed fingerprint at agent startup would remove a whole error class. -**F2 · S3 — crypt passwords must be pre-obscured with rclone by hand.** +**F2 · S3 — ~~crypt passwords must be pre-obscured with rclone by hand.~~ (fixed in #171)** Squirrel owns rclone.conf and hides rclone everywhere else, but crypt config leaks the `rclone obscure` step (4 manual invocations for two crypt destinations). Accepting plaintext-via-env and obscuring internally would match the "squirrel owns the rclone surface" stance. -**F3 · S2 — the peer-token matrix is easy to wire wrong.** One +**F3 · S2 — ~~the peer-token matrix is easy to wire wrong.~~ (`squirrel node pair`, #171)** One nas↔htpc relationship needs four token bindings across two files (each side's `[agent.auth.peers.X]` entry must equal the *other* side's `[nodes.Y].auth.bearer`). Nothing validates the pairing until @@ -43,7 +43,7 @@ a sync fails at runtime with 401. Writing these four configs, the cross-referencing was the single most error-prone part — and there is no `squirrel node add` / pairing flow to generate matching halves. -**F4 · S2 — a freshly configured machine reports nothing.** The +**F4 · S2 — ~~a freshly configured machine reports nothing.~~ (`squirrel config check`, #171)** The natural first command after writing config (`squirrel volumes`) prints an empty list with exit 0: it reads the *database*, not the config, so declared-but-never-indexed volumes are invisible. Config-parse success @@ -413,6 +413,8 @@ mount or rclone-style prefix that squirrel neither validates at load time nor mentions when it's wrong (bytes just fail to land); the htpc even needs a `[nodes.nas]` entry with a mandatory `path` that no bytes ever traverse, purely to enable durability pulls. +*Partially addressed in #171: `squirrel config check` now stats and +flags a missing node byte-path; load-time validation is still absent.* **F35 · S3 — cadence-only machines must still run the full agent.** A machine that never receives (laptop) runs the HTTP listener and From b06afaad88aa973d3d7eb08ef9de9bdcd8265984 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:24:52 +0000 Subject: [PATCH 05/15] Address Copilot review on #171 - config/obscure_test.go: the IV-uniqueness check now base64-decodes the obscured values and compares the actual leading IV bytes (via a new obscuredIV helper) instead of a base64 string prefix, and checks the obscureRclone errors so a regression can't hide behind zero values. - agent/cert_test.go: TestGenerateSelfSignedCertUnique now checks the errors from GenerateSelfSignedCert rather than comparing zero-value fingerprints on a silent failure. - config/destinations.go: cryptObscuredFlag's error is wrapped with the "crypt:" prefix at the call site, matching the other resolveCrypt errors (unknown field, secret resolution); the test asserts the prefixed message. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- agent/cert_test.go | 10 ++++++++-- config/crypt_plaintext_test.go | 2 +- config/destinations.go | 6 ++++-- config/obscure_test.go | 36 +++++++++++++++++++++++++++++----- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/agent/cert_test.go b/agent/cert_test.go index 1ca5b76..f734e86 100644 --- a/agent/cert_test.go +++ b/agent/cert_test.go @@ -48,8 +48,14 @@ func TestGenerateSelfSignedCert(t *testing.T) { } func TestGenerateSelfSignedCertUnique(t *testing.T) { - a, _ := GenerateSelfSignedCert("nas") - b, _ := GenerateSelfSignedCert("nas") + a, err := GenerateSelfSignedCert("nas") + if err != nil { + t.Fatalf("GenerateSelfSignedCert: %v", err) + } + b, err := GenerateSelfSignedCert("nas") + if err != nil { + t.Fatalf("GenerateSelfSignedCert: %v", err) + } if a.Fingerprint == b.Fingerprint { t.Fatal("two generations produced the same fingerprint") } diff --git a/config/crypt_plaintext_test.go b/config/crypt_plaintext_test.go index 7d99cc9..49462cb 100644 --- a/config/crypt_plaintext_test.go +++ b/config/crypt_plaintext_test.go @@ -73,7 +73,7 @@ root = "/data" obscured = "yes" password = "hunter2" `)) - if err == nil || !strings.Contains(err.Error(), "crypt.obscured must be a boolean") { + if err == nil || !strings.Contains(err.Error(), "crypt: obscured must be a boolean") { t.Fatalf("expected obscured-must-be-bool error, got %v", err) } } diff --git a/config/destinations.go b/config/destinations.go index 1d73bbe..649574a 100644 --- a/config/destinations.go +++ b/config/destinations.go @@ -393,7 +393,7 @@ func resolveCrypt(raw map[string]any, typ string) (*Crypt, error) { } alreadyObscured, err := cryptObscuredFlag(table) if err != nil { - return nil, err + return nil, fmt.Errorf("crypt: %w", err) } for k := range table { if k != "password" && k != "password2" && k != "obscured" { @@ -426,7 +426,9 @@ func cryptObscuredFlag(table map[string]any) (bool, error) { } b, ok := v.(bool) if !ok { - return false, errors.New("crypt.obscured must be a boolean") + // The caller wraps this with the "crypt:" prefix every other + // resolveCrypt error carries, so the message stays bare here. + return false, errors.New("obscured must be a boolean") } return b, nil } diff --git a/config/obscure_test.go b/config/obscure_test.go index 2754c94..ec9a7db 100644 --- a/config/obscure_test.go +++ b/config/obscure_test.go @@ -1,6 +1,7 @@ package config import ( + "bytes" "crypto/aes" "crypto/cipher" "encoding/base64" @@ -51,15 +52,40 @@ func TestObscureRcloneRoundTrips(t *testing.T) { // is byte-identical and the "unexpected rewrite is a signal" invariant // holds. Two different plaintexts must not share an IV (keystream reuse). func TestObscureRcloneDeterministic(t *testing.T) { - a1, _ := obscureRclone("secret") - a2, _ := obscureRclone("secret") + a1, err := obscureRclone("secret") + if err != nil { + t.Fatalf("obscureRclone(secret): %v", err) + } + a2, err := obscureRclone("secret") + if err != nil { + t.Fatalf("obscureRclone(secret): %v", err) + } if a1 != a2 { t.Fatalf("obscureRclone is not deterministic: %q vs %q", a1, a2) } - b, _ := obscureRclone("other") - if a1[:22] == b[:22] { - t.Fatalf("distinct plaintexts share an IV prefix (keystream reuse): %q / %q", a1, b) + b, err := obscureRclone("other") + if err != nil { + t.Fatalf("obscureRclone(other): %v", err) + } + // Compare the decoded leading IV bytes, not the base64 text: a base64 + // prefix comparison could mask a real IV collision or trip on unrelated + // encoding differences. + if bytes.Equal(obscuredIV(t, a1), obscuredIV(t, b)) { + t.Fatalf("distinct plaintexts share an IV (keystream reuse): %q / %q", a1, b) + } +} + +// obscuredIV decodes an obscured value and returns its leading IV block. +func obscuredIV(t *testing.T, obscured string) []byte { + t.Helper() + raw, err := base64.RawURLEncoding.DecodeString(obscured) + if err != nil { + t.Fatalf("base64 decode: %v", err) + } + if len(raw) < aes.BlockSize { + t.Fatalf("obscured value too short for an IV: %d bytes", len(raw)) } + return raw[:aes.BlockSize] } // TestObscureRcloneMatchesRclone cross-checks the output against the real From 78d56ba22ee01f789fa3f89db0525115940c3aca Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:35:32 +0000 Subject: [PATCH 06/15] agent: listener-less (cadence-only) mode (F35) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the HTTP listener optional. When `[agent] listen` is absent/empty the agent runs its background schedulers (index/sync/audit cadences) WITHOUT binding an HTTP server — the mode for cadence-only machines (a roaming laptop that pushes to a hub and never receives peer syncs). When `listen` is set, behaviour is unchanged: scheduler + HTTP server, token required. - config: `listen` no longer required; a bearer token is required only when a listener is configured (nothing to authenticate otherwise). - agent: validateConfig relaxed accordingly; new Server.RunSchedulers runs the scan + cadence loops with the same shutdown discipline as Serve but no listener. - cmd: runAgent dispatches to RunSchedulers when listen is empty, and fails fast if a listener-less agent has no cadence and no scan (silent idling is degradation, not a valid config); the marker/init boundary is untouched — the agent still never escalates. - docs: reference-setup laptop drops the dummy listen + token; agent guide and configuration reference describe listener-less operation. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- agent/agent.go | 9 +- agent/agent_test.go | 38 ++++++++- agent/serve.go | 20 +++++ cmd/squirrel/agent.go | 45 ++++++++++ cmd/squirrel/agent_listenerless_test.go | 85 +++++++++++++++++++ config/agent.go | 12 ++- config/config_test.go | 23 +++-- design/reference-setup.md | 6 +- docs/src/content/docs/guides/agent.md | 28 +++++- .../content/docs/reference/configuration.md | 6 ++ 10 files changed, 251 insertions(+), 21 deletions(-) create mode 100644 cmd/squirrel/agent_listenerless_test.go diff --git a/agent/agent.go b/agent/agent.go index afc6d8c..a22d2b2 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -191,10 +191,11 @@ func (s *Server) CertFingerprint() (string, error) { func (s *Server) Addr() string { return s.cfg.Listen } func validateConfig(cfg Config) error { - if cfg.Listen == "" { - return errors.New("agent: Config.Listen is required") - } - if cfg.Token == "" { + // An empty Listen selects listener-less mode (F35): the agent runs only + // its background schedulers, so neither a bind address nor a bearer + // token is required. A token is required only when there is an HTTP + // surface to protect. + if cfg.Listen != "" && cfg.Token == "" { return errors.New("agent: Config.Token is required") } if (cfg.TLSCert == "") != (cfg.TLSKey == "") { diff --git a/agent/agent_test.go b/agent/agent_test.go index 9606f21..9f36fc0 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -64,7 +64,6 @@ func TestNewRejectsBadConfig(t *testing.T) { cfg Config want string }{ - {"no listen", Config{Token: "t", Version: "v"}, "Listen is required"}, {"no token", Config{Listen: ":0", Version: "v"}, "Token is required"}, {"half tls", Config{Listen: ":0", Token: "t", TLSCert: "c", Version: "v"}, "must be set together"}, {"no version", Config{Listen: ":0", Token: "t"}, "Version is required"}, @@ -79,6 +78,43 @@ func TestNewRejectsBadConfig(t *testing.T) { } } +// TestNewListenerLess covers F35: an empty Listen (and no Token) is a +// valid scheduler-only agent, and the server reports no listener/TLS. +func TestNewListenerLess(t *testing.T) { + srv, err := New(Config{Version: "v"}, openTestStore(t)) + if err != nil { + t.Fatalf("New (listener-less): %v", err) + } + if srv.Addr() != "" { + t.Fatalf("Addr = %q, want empty for a listener-less agent", srv.Addr()) + } + if srv.HasTLS() { + t.Fatal("listener-less agent unexpectedly reports TLS") + } +} + +// TestRunSchedulersReturnsOnCancel pins that the listener-less run path +// blocks until its context is cancelled and then returns cleanly, without +// ever binding a listener. +func TestRunSchedulersReturnsOnCancel(t *testing.T) { + srv, err := New(Config{Version: "v"}, openTestStore(t)) + if err != nil { + t.Fatalf("New: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- srv.RunSchedulers(ctx) }() + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("RunSchedulers: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("RunSchedulers did not return after context cancel") + } +} + // TestNewDefaultsLoggerToDiscard pins the contract that callers may // leave Config.Logger nil and the agent will still be safe to use: // New() substitutes a discard logger so the future scheduler can diff --git a/agent/serve.go b/agent/serve.go index 8d13d34..544336c 100644 --- a/agent/serve.go +++ b/agent/serve.go @@ -93,6 +93,26 @@ func (s *Server) Serve(ctx context.Context, ln net.Listener) error { } } +// RunSchedulers runs the background scan + cadence loops without binding +// an HTTP listener — the listener-less agent mode (F35) for cadence-only +// machines that never receive peer syncs. It blocks until ctx is +// cancelled, then waits for an in-flight loop tick to finish its volume, +// mirroring Serve's shutdown discipline minus the HTTP server. The two +// loops are gated exactly as under Serve (scan only when ScanInterval is +// set; scheduler only when a volume declares a cadence), so an agent with +// nothing scheduled runs no goroutines and simply waits for cancellation. +func (s *Server) RunSchedulers(ctx context.Context) error { + loopCtx, cancelLoops := context.WithCancel(ctx) + defer cancelLoops() + var loopWG sync.WaitGroup + s.startScanLoop(loopCtx, &loopWG, s.scanLogger()) + s.startSchedulerLoop(loopCtx, &loopWG) + <-ctx.Done() + cancelLoops() + loopWG.Wait() + return nil +} + // startScanLoop spins up the drift-detection scheduler in a sibling // goroutine, but only when ScanInterval is set. The WaitGroup lets // Serve block on the loop's clean exit during shutdown so a tick diff --git a/cmd/squirrel/agent.go b/cmd/squirrel/agent.go index f8b224f..8498156 100644 --- a/cmd/squirrel/agent.go +++ b/cmd/squirrel/agent.go @@ -75,6 +75,24 @@ func runAgent(cmd *cobra.Command) error { if err != nil { return err } + return serveAgent(cmd, cfg, srv, logger) +} + +// serveAgent dispatches the built server to its run path: the listener-less +// scheduler-only run (F35) when [agent] listen is empty, or the HTTP server +// otherwise. Split from runAgent so the setup phase stays compact. +func serveAgent(cmd *cobra.Command, cfg *config.Config, srv *agent.Server, logger *slog.Logger) error { + // Listener-less mode (F35): no `listen`, so run the schedulers without an + // HTTP server. Refuse an agent that would do nothing at all — a + // scheduler-only agent with no cadences and no scan is silent + // degradation, not a valid config. + if cfg.Agent.Listen == "" { + if !agentHasWork(cfg) { + return fmt.Errorf("listener-less agent has nothing to run: set [agent] listen to receive peer syncs, or a sync_every/index_every/scan_interval cadence, in %s", cfg.Path) + } + logSchedulerOnlyStartup(logger) + return srv.RunSchedulers(cmd.Context()) + } // Bind first so a port-in-use (or any other listen failure) surfaces // as a CLI error and never logs a misleading "agent listening" line. // We also log the listener's resolved Addr so `:0` (and other @@ -88,6 +106,33 @@ func runAgent(cmd *cobra.Command) error { return srv.Serve(cmd.Context(), ln) } +// agentHasWork reports whether a listener-less agent has any background +// work: a drift-scan interval, or at least one volume with a scheduler +// cadence (sync, standalone index, or interval hook). It mirrors the +// agent scheduler's own "is anything scheduled" gate so the CLI refuses a +// do-nothing agent rather than letting it idle silently. +func agentHasWork(cfg *config.Config) bool { + if cfg.Agent.ScanInterval > 0 { + return true + } + for _, v := range cfg.Volumes { + if v.SyncEvery > 0 || v.IndexEvery > 0 { + return true + } + if v.Hook != nil && v.Hook.Interval > 0 { + return true + } + } + return false +} + +// logSchedulerOnlyStartup emits the listener-less counterpart of the +// "agent listening" banner so a journal shows the agent came up in +// scheduler-only mode with no bound port. +func logSchedulerOnlyStartup(logger *slog.Logger) { + logger.Info("agent scheduler running", "listener", "disabled", "version", agentVersion) +} + // openAgentStore extends the standard resolveDBPath precedence with the // agent-specific override: --db > cfg.Agent.DB > cfg.DB > default. We // can't reuse openStore directly because it doesn't know about the diff --git a/cmd/squirrel/agent_listenerless_test.go b/cmd/squirrel/agent_listenerless_test.go new file mode 100644 index 0000000..8629c1e --- /dev/null +++ b/cmd/squirrel/agent_listenerless_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// TestCLIAgentListenerLessRunsScheduler covers F35 end-to-end: an [agent] +// block with no `listen` starts the scheduler without binding an HTTP +// listener, logs the scheduler-only banner, and shuts down cleanly on +// context cancel. The volume declares an index cadence (no sync_to), so no +// rclone binary is needed to bring the agent up. +func TestCLIAgentListenerLessRunsScheduler(t *testing.T) { + dir := t.TempDir() + vol := filepath.Join(dir, "photos") + if err := os.MkdirAll(vol, 0o755); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(vol, "a.jpg"), "x") + dbPath := filepath.Join(dir, "index.db") + cfgPath := filepath.Join(dir, "config.toml") + body := fmt.Sprintf("db = %q\n\n[agent]\n\n[volumes.photos]\npath = %q\nindex_every = \"1h\"\n", dbPath, vol) + if err := os.WriteFile(cfgPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + isolateConfig(t) + ctx, cancel := context.WithCancel(context.Background()) + buf := &syncBuf{} + root := newRootCmd() + root.SetOut(buf) + root.SetErr(buf) + root.SetArgs([]string{"--config", cfgPath, "agent"}) + done := make(chan error, 1) + go func() { done <- root.ExecuteContext(ctx) }() + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) && !strings.Contains(buf.String(), "agent scheduler running") { + time.Sleep(20 * time.Millisecond) + } + if !strings.Contains(buf.String(), "agent scheduler running") { + cancel() + t.Fatalf("scheduler-only banner never appeared:\n%s", buf.String()) + } + if strings.Contains(buf.String(), "agent listening") { + cancel() + t.Fatalf("listener-less agent unexpectedly bound a listener:\n%s", buf.String()) + } + + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("agent exited with error: %v\noutput:\n%s", err, buf.String()) + } + case <-time.After(5 * time.Second): + t.Fatalf("agent did not shut down within timeout\noutput:\n%s", buf.String()) + } +} + +// TestCLIAgentListenerLessNoWork guards the fail-fast: a listener-less +// agent with no cadence and no scan has nothing to run and must refuse to +// start rather than idle silently. +func TestCLIAgentListenerLessNoWork(t *testing.T) { + dir := t.TempDir() + vol := filepath.Join(dir, "photos") + if err := os.MkdirAll(vol, 0o755); err != nil { + t.Fatal(err) + } + dbPath := filepath.Join(dir, "index.db") + cfgPath := filepath.Join(dir, "config.toml") + body := fmt.Sprintf("db = %q\n\n[agent]\n\n[volumes.photos]\npath = %q\n", dbPath, vol) + if err := os.WriteFile(cfgPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + _, err := runCLIExpectErr(t, "--config", cfgPath, "agent") + if !strings.Contains(err.Error(), "nothing to run") { + t.Fatalf("expected nothing-to-run error, got %v", err) + } +} diff --git a/config/agent.go b/config/agent.go index 16d02ef..77a146d 100644 --- a/config/agent.go +++ b/config/agent.go @@ -84,9 +84,9 @@ type rawPeerAuth struct { } func resolveAgent(r *rawAgent) (*Agent, error) { - if r.Listen == "" { - return nil, errors.New("listen is required") - } + // An empty listen is the listener-less mode (F35): the agent runs the + // background schedulers (index/sync/audit cadences) without binding an + // HTTP server, for cadence-only machines that never receive peer syncs. a := &Agent{Listen: r.Listen} if r.DB != "" { expanded, err := expandPath(r.DB) @@ -162,6 +162,12 @@ func resolveAgentTLS(r *rawTLS, a *Agent) error { func resolveAgentAuth(r *rawAuth, a *Agent) error { if r == nil || r.Token == nil { + if a.Listen == "" { + // Listener-less agent (F35): there is no HTTP surface to + // authenticate, so a bearer token is not required — and peer + // tokens would have no listener to guard either. + return nil + } return errors.New("auth.token is required (no agent without authentication)") } // resolveSecret takes a map[string]any and pulls the named key. We pass diff --git a/config/config_test.go b/config/config_test.go index 44d247f..a0b4f9f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -979,14 +979,21 @@ auth = { } } } -func TestLoadAgentMissingListen(t *testing.T) { - p := writeConfig(t, ` -[agent] -auth = { token = "x" } -`) - _, err := Load(p) - if err == nil || !strings.Contains(err.Error(), "listen is required") { - t.Fatalf("expected listen-required error, got %v", err) +// TestLoadAgentListenerLess covers F35: an [agent] block without `listen` +// is the listener-less mode — valid, scheduler-only. A bare block needs no +// auth, and an auth token without a listener is tolerated (unused). +func TestLoadAgentListenerLess(t *testing.T) { + for _, body := range []string{ + "\n[agent]\n", + "\n[agent]\nauth = { token = \"x\" }\n", + } { + cfg, err := Load(writeConfig(t, body)) + if err != nil { + t.Fatalf("Load(%q): %v", body, err) + } + if cfg.Agent == nil || cfg.Agent.Listen != "" { + t.Fatalf("expected listener-less agent (empty Listen), got %+v", cfg.Agent) + } } } diff --git a/design/reference-setup.md b/design/reference-setup.md index 5a06d82..993340f 100644 --- a/design/reference-setup.md +++ b/design/reference-setup.md @@ -187,9 +187,9 @@ db = "~/.squirrel/index.db" node_name = "laptop" [agent] -listen = "127.0.0.1:8443" # never receives; agent runs only for the cadences -[agent.auth] -token = { env = "SQUIRREL_AGENT_TOKEN" } +# No `listen`: this roaming laptop never receives peer syncs, so the agent +# runs the sync/index schedulers *without* an HTTP listener (F35). With no +# listener there is nothing to authenticate, so no [agent.auth] token either. [volumes.photos] path = "~/Pictures" diff --git a/docs/src/content/docs/guides/agent.md b/docs/src/content/docs/guides/agent.md index 87cecd4..594cf78 100644 --- a/docs/src/content/docs/guides/agent.md +++ b/docs/src/content/docs/guides/agent.md @@ -22,8 +22,32 @@ The agent requires an `[agent]` block in config. schedule. - **Hook firing** — fires per-volume [hooks](/squirrel/guides/hooks/) on change (after each successful index run) and on their `interval`. -- **HTTP server** — exposes a health endpoint and drives the state the - [TUI](/squirrel/guides/tui/) and desktop app read. +- **HTTP server** — exposes a health endpoint, serves peer syncs, and drives + the state the [TUI](/squirrel/guides/tui/) and desktop app read. Only when + `[agent] listen` is set (see below). + +## Listener-less (cadence-only) machines + +A machine that only *originates* content — a roaming laptop that pushes to a +hub and never receives peer syncs — has no reason to run an HTTP listener. Omit +`[agent] listen` and the agent runs its schedulers (index/sync/audit cadences) +**without binding a listener**; with no listener there is nothing to +authenticate, so `[agent.auth]` is optional too. + +```toml +[agent] +# no `listen`: schedulers only, no HTTP server + +[volumes.photos] +path = "~/Pictures" +sync_to = ["nas"] +sync_every = "1h" +``` + +When `listen` *is* set, behaviour is unchanged: the agent runs the schedulers +**and** the HTTP server (and then a bearer token is required). A listener-less +agent with no cadence and no `scan_interval` has nothing to do and refuses to +start, rather than idling silently. ## Database path precedence diff --git a/docs/src/content/docs/reference/configuration.md b/docs/src/content/docs/reference/configuration.md index 5ddf661..3fec920 100644 --- a/docs/src/content/docs/reference/configuration.md +++ b/docs/src/content/docs/reference/configuration.md @@ -136,3 +136,9 @@ Required to run [`squirrel agent`](/squirrel/guides/agent/). Configures the unattended cadence (`index_every`, `sync_every`), scheduled audits, and an optional agent-specific `db` path (which takes precedence over the top-level `db` when the agent runs). + +`listen` is optional. When set (e.g. `0.0.0.0:8443`), the agent binds an HTTP +server for peer syncs and the health endpoint, and a bearer token is required. +When omitted, the agent runs its schedulers only — the *listener-less* mode for +cadence-only machines that never receive peer syncs; `[agent.auth]` is then +optional. See [The agent](/squirrel/guides/agent/#listener-less-cadence-only-machines). From ee1f14271c44dd049a751cc1109a6fdebb3e7076 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:36:32 +0000 Subject: [PATCH 07/15] design: mark F35 fixed by #175 Strike the listener-less friction entry now that the cadence-only agent mode lands, per the friction log's PR-reference convention. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- design/friction-log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/friction-log.md b/design/friction-log.md index 496c68f..2c5a9df 100644 --- a/design/friction-log.md +++ b/design/friction-log.md @@ -416,7 +416,7 @@ bytes ever traverse, purely to enable durability pulls. *Partially addressed in #171: `squirrel config check` now stats and flags a missing node byte-path; load-time validation is still absent.* -**F35 · S3 — cadence-only machines must still run the full agent.** +**F35 · S3 — ~~cadence-only machines must still run the full agent.~~ (fixed in #175)** A machine that never receives (laptop) runs the HTTP listener and must configure `[agent] listen` + auth token anyway, because the scheduler lives inside the agent. A listener that exists to be unused From f29df8e6bb042d60ed2ef292f8f37605ffadec46 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:39:08 +0000 Subject: [PATCH 08/15] store: audit-preserving destination reset (F20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ResetDestination: an operator-driven verb that forgets a destination's recorded remote state (remote_objects, remote_packs, the durability vector, and push-freshness) so the next sync treats it as fresh and re-uploads. It is the supported recovery for a wrecked or repointed destination, replacing hand SQL or a cross-machine rename. The reset is audit-preserving: the runs table and the append-only destination_run_ids_history advance log are untouched, and the content/files rows are never modified. The reset itself is recorded as a kind='audit' run (reusing the destination-scoped audit shape, so no schema change) with a new free-text 'reset-destination' runs_audit transition carrying the cleared counts. Everything runs in one transaction — a reset is all-or-nothing. CountDestinationRecordedState backs the CLI's dry-run preview. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- store/destination_reset.go | 177 ++++++++++++++++++++++++++++ store/destination_reset_test.go | 201 ++++++++++++++++++++++++++++++++ store/runs_audit.go | 7 ++ 3 files changed, 385 insertions(+) create mode 100644 store/destination_reset.go create mode 100644 store/destination_reset_test.go diff --git a/store/destination_reset.go b/store/destination_reset.go new file mode 100644 index 0000000..198f731 --- /dev/null +++ b/store/destination_reset.go @@ -0,0 +1,177 @@ +package store + +import ( + "context" + "database/sql" + "fmt" +) + +// DestinationResetCounts reports how many recorded-state rows a destination +// reset cleared (or, on a dry run, would clear). The four categories are the +// full derived record squirrel keeps about a destination: the per-content +// and per-pack upload ledgers, the live durability vector, and the +// push-freshness maxima. They span every volume — a destination's recorded +// state is not volume-scoped — so the counts are totals across the index. +type DestinationResetCounts struct { + RemoteObjects int64 + RemotePacks int64 + VectorComponents int64 + FreshnessRows int64 +} + +// Total is the sum of all four categories, used as the reset run's +// file_count so the audit trail carries the magnitude at a glance. +func (c DestinationResetCounts) Total() int64 { + return c.RemoteObjects + c.RemotePacks + c.VectorComponents + c.FreshnessRows +} + +// Empty reports whether the destination has no recorded state at all — the +// "nothing to reset" case the CLI reports affirmatively rather than minting +// an empty run. +func (c DestinationResetCounts) Empty() bool { return c.Total() == 0 } + +// rowQuerier is the subset of *sql.DB and *sql.Tx that +// countDestinationRecordedState needs, so the same count runs read-only on +// the DB (the dry-run preview) and inside the reset transaction (the note +// the run records). +type rowQuerier interface { + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// CountDestinationRecordedState returns how much derived state the index +// holds for destination, without changing anything. It is the read-only +// question behind `squirrel destination reset --dry-run`. +func (s *Store) CountDestinationRecordedState(ctx context.Context, destination string) (DestinationResetCounts, error) { + if destination == "" { + return DestinationResetCounts{}, fmt.Errorf("CountDestinationRecordedState: destination must be non-empty") + } + return countDestinationRecordedState(ctx, s.db, destination) +} + +func countDestinationRecordedState(ctx context.Context, q rowQuerier, destination string) (DestinationResetCounts, error) { + var c DestinationResetCounts + counts := []struct { + dst *int64 + query string + }{ + {&c.RemoteObjects, `SELECT COUNT(*) FROM remote_objects WHERE destination = ?`}, + {&c.RemotePacks, `SELECT COUNT(*) FROM remote_packs WHERE destination = ?`}, + {&c.VectorComponents, `SELECT COUNT(*) FROM destination_run_ids WHERE destination = ?`}, + {&c.FreshnessRows, `SELECT COUNT(*) FROM destination_push_freshness WHERE destination = ?`}, + } + for _, item := range counts { + if err := q.QueryRowContext(ctx, item.query, destination).Scan(item.dst); err != nil { + return DestinationResetCounts{}, fmt.Errorf("count destination %q recorded state: %w", destination, err) + } + } + return c, nil +} + +// ResetDestination forgets everything the index records about a destination's +// remote state — its per-content and per-pack upload ledgers, its live +// durability vector, and its push-freshness maxima — so the next sync treats +// the destination as fresh and re-uploads. It is the supported recovery for a +// wrecked or repointed destination (friction log F20): before it, a fresh +// `root` still refused because the layout guard keyed on run history by +// destination name, leaving hand SQL or a config-wide rename as the only outs. +// +// The reset is audit-preserving. It never touches the runs table or the +// append-only destination_run_ids_history advance log, so the record of what +// was ever asserted durable survives; only the live derived state is cleared, +// exactly as RevokeDestinationRunIDsFromSource clears a peer's live +// assertions while leaving the history intact. The contents and files rows — +// the content squirrel must never lose track of — are untouched. +// +// The reset is itself recorded as a run: a kind='audit' row (the run kinds +// are fixed by a CHECK, so this reuses the destination-scoped audit shape +// BeginRemoteVerifyRun already uses — volume_id and destination NULL) whose +// runs_audit trail carries a 'reset-destination' note with the destination +// name and the cleared counts. Everything runs in one transaction: on any +// error nothing is cleared and no run is recorded, so a reset is all-or-nothing. +func (s *Store) ResetDestination(ctx context.Context, destination string) (int64, DestinationResetCounts, error) { + if destination == "" { + return 0, DestinationResetCounts{}, fmt.Errorf("ResetDestination: destination must be non-empty") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, DestinationResetCounts{}, fmt.Errorf("begin reset destination %q: %w", destination, err) + } + defer func() { _ = tx.Rollback() }() + + counts, err := countDestinationRecordedState(ctx, tx, destination) + if err != nil { + return 0, DestinationResetCounts{}, err + } + atNs := NowNs() + runID, err := insertResetRunTx(ctx, tx, atNs) + if err != nil { + return 0, DestinationResetCounts{}, err + } + if err := deleteDestinationRecordedStateTx(ctx, tx, destination); err != nil { + return 0, DestinationResetCounts{}, err + } + note := fmt.Sprintf("destination=%q objects=%d packs=%d vector=%d freshness=%d", + destination, counts.RemoteObjects, counts.RemotePacks, counts.VectorComponents, counts.FreshnessRows) + if err := appendRunAuditTx(ctx, tx, + RunAuditEntry{RunID: runID, Transition: TransitionResetDestination, Note: note}, atNs); err != nil { + return 0, DestinationResetCounts{}, err + } + if err := finishResetRunTx(ctx, tx, runID, atNs, counts.Total()); err != nil { + return 0, DestinationResetCounts{}, err + } + if err := tx.Commit(); err != nil { + return 0, DestinationResetCounts{}, fmt.Errorf("commit reset destination %q: %w", destination, err) + } + return runID, counts, nil +} + +// insertResetRunTx opens the kind='audit' run that records the reset. The +// runs CHECK keeps audit rows' volume_id and destination NULL (the destination +// travels in the 'reset-destination' runs_audit note instead), matching +// BeginRemoteVerifyRun's destination-scoped audit shape. +func insertResetRunTx(ctx context.Context, tx *sql.Tx, atNs int64) (int64, error) { + res, err := tx.ExecContext(ctx, ` + INSERT INTO runs (kind, volume_id, destination, started_at_ns, status, file_count) + VALUES ('audit', NULL, NULL, ?, 'running', 0) + `, atNs) + if err != nil { + return 0, fmt.Errorf("insert reset run: %w", err) + } + id, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("reset run last insert id: %w", err) + } + return id, nil +} + +// deleteDestinationRecordedStateTx clears the four derived-state tables for +// destination. The append-only destination_run_ids_history and the runs +// table are deliberately absent — the reset preserves the audit trail. +func deleteDestinationRecordedStateTx(ctx context.Context, tx *sql.Tx, destination string) error { + stmts := []string{ + `DELETE FROM remote_objects WHERE destination = ?`, + `DELETE FROM remote_packs WHERE destination = ?`, + `DELETE FROM destination_run_ids WHERE destination = ?`, + `DELETE FROM destination_push_freshness WHERE destination = ?`, + } + for _, stmt := range stmts { + if _, err := tx.ExecContext(ctx, stmt, destination); err != nil { + return fmt.Errorf("clear destination %q recorded state: %w", destination, err) + } + } + return nil +} + +// finishResetRunTx moves the reset run to success with the cleared total as +// its file_count and appends the 'finish' runs_audit line, mirroring +// FinishRun so the reset run reads like any other terminal run. +func finishResetRunTx(ctx context.Context, tx *sql.Tx, runID, atNs, total int64) error { + if _, err := tx.ExecContext(ctx, ` + UPDATE runs SET ended_at_ns = ?, status = 'success', file_count = ? + WHERE id = ? + `, atNs, total, runID); err != nil { + return fmt.Errorf("finish reset run %d: %w", runID, err) + } + return appendRunAuditTx(ctx, tx, + RunAuditEntry{RunID: runID, Transition: TransitionFinish, Note: RunStatusSuccess}, atNs) +} diff --git a/store/destination_reset_test.go b/store/destination_reset_test.go new file mode 100644 index 0000000..f3223a6 --- /dev/null +++ b/store/destination_reset_test.go @@ -0,0 +1,201 @@ +package store + +import ( + "context" + "testing" +) + +// seedDestinationState populates every category of derived state for +// destination on volume vID, plus one durability advance for a second +// destination, and returns the self node id. It leaves a runs row (the +// index run) and a destination_run_ids_history row behind so the +// audit-preservation assertions have something to check against. +func seedDestinationState(t *testing.T, s *Store, vID, runID int64, destination string) int64 { + t.Helper() + ctx := context.Background() + self, err := s.GetSelfNode(ctx) + if err != nil { + t.Fatalf("GetSelfNode: %v", err) + } + + objContent := packContentFixture(t, s, vID, runID, "obj.txt", 0xa1) + if err := s.InsertRemoteObject(ctx, RemoteObject{ + ContentID: objContent, Destination: destination, UploadedRunID: runID, + }); err != nil { + t.Fatalf("InsertRemoteObject: %v", err) + } + + packContent := packContentFixture(t, s, vID, runID, "packed.txt", 0xa2) + if err := s.InsertPacks(ctx, []PackWrite{{ + Pack: Pack{PackKey: packKey(0x11), SizeBytes: 10, MemberCount: 1, CreatedRunID: runID}, + Members: []PackMember{{ContentID: packContent, ByteOffset: 0, ByteLength: 1}}, + }}); err != nil { + t.Fatalf("InsertPacks: %v", err) + } + pack, err := s.GetPackByKey(ctx, packKey(0x11)) + if err != nil { + t.Fatalf("GetPackByKey: %v", err) + } + if err := s.InsertRemotePack(ctx, RemotePack{ + PackID: pack.ID, Destination: destination, UploadedRunID: runID, + }); err != nil { + t.Fatalf("InsertRemotePack: %v", err) + } + + if err := s.UpsertDestinationRunIDVerified(ctx, vID, destination, self.ID, 5, VerifyMethodBlake3, false); err != nil { + t.Fatalf("UpsertDestinationRunIDVerified: %v", err) + } + if err := s.UpsertDestinationPushFreshness(ctx, vID, destination, self.ID, 5); err != nil { + t.Fatalf("UpsertDestinationPushFreshness: %v", err) + } + // A second destination whose state must survive a reset of the first. + if err := s.UpsertDestinationRunIDVerified(ctx, vID, "other", self.ID, 7, VerifyMethodBlake3, false); err != nil { + t.Fatalf("UpsertDestinationRunIDVerified(other): %v", err) + } + return self.ID +} + +// TestResetDestinationClearsDerivedStateAuditPreservingly is the F20 core: +// reset forgets a destination's upload ledgers, vector, and freshness while +// leaving the runs table, the append-only advance log, other destinations, +// and the content rows intact — and records itself as an audit run. +func TestResetDestinationClearsDerivedStateAuditPreservingly(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + vID := makeVolume(t, s, "/v") + runID := makeRun(t, s, vID) + self := seedDestinationState(t, s, vID, runID, "arch") + + counts, err := s.CountDestinationRecordedState(ctx, "arch") + if err != nil { + t.Fatalf("CountDestinationRecordedState: %v", err) + } + if counts.RemoteObjects != 1 || counts.RemotePacks != 1 || counts.VectorComponents != 1 || counts.FreshnessRows != 1 { + t.Fatalf("pre-reset counts = %+v, want one of each", counts) + } + + _, cleared, err := s.ResetDestination(ctx, "arch") + if err != nil { + t.Fatalf("ResetDestination: %v", err) + } + if cleared != counts { + t.Fatalf("cleared = %+v, want %+v", cleared, counts) + } + + // Derived state is gone. + after, err := s.CountDestinationRecordedState(ctx, "arch") + if err != nil { + t.Fatalf("CountDestinationRecordedState after: %v", err) + } + if !after.Empty() { + t.Fatalf("post-reset counts = %+v, want empty", after) + } + vec, err := s.ListDestinationRunIDs(ctx, vID, "arch") + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + if len(vec) != 0 { + t.Fatalf("vector for arch = %+v, want empty", vec) + } + + // The append-only advance log survives — the reset is audit-preserving. + hist, err := s.ListDestinationRunIDHistory(ctx, vID, "arch") + if err != nil { + t.Fatalf("ListDestinationRunIDHistory: %v", err) + } + if len(hist) == 0 { + t.Fatalf("advance history for arch was cleared; reset must preserve it") + } + + // The other destination is untouched. + otherVec, err := s.ListDestinationRunIDs(ctx, vID, "other") + if err != nil { + t.Fatalf("ListDestinationRunIDs(other): %v", err) + } + if len(otherVec) != 1 || otherVec[0].OriginNodeID != self { + t.Fatalf("other vector = %+v, want one component for self", otherVec) + } + + // The content rows are never touched. + if _, err := s.GetByPath(ctx, vID, "obj.txt"); err != nil { + t.Fatalf("content row for obj.txt lost: %v", err) + } +} + +// TestResetDestinationRecordsAuditRun: the reset is a kind='audit' run that +// finishes success and carries a 'reset-destination' runs_audit note with +// the cleared counts, alongside the usual 'finish' transition. +func TestResetDestinationRecordsAuditRun(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + vID := makeVolume(t, s, "/v") + runID := makeRun(t, s, vID) + seedDestinationState(t, s, vID, runID, "arch") + + resetRun, cleared, err := s.ResetDestination(ctx, "arch") + if err != nil { + t.Fatalf("ResetDestination: %v", err) + } + run, err := s.GetRun(ctx, resetRun) + if err != nil { + t.Fatalf("GetRun: %v", err) + } + if run.Kind != RunKindAudit || run.Status != RunStatusSuccess || run.VolumeID.Valid || run.Destination.Valid { + t.Fatalf("reset run = %+v, want a success audit run with NULL volume+destination", run) + } + if run.FileCount != cleared.Total() { + t.Fatalf("run file_count = %d, want cleared total %d", run.FileCount, cleared.Total()) + } + + audits, err := s.ListRunAudit(ctx, resetRun) + if err != nil { + t.Fatalf("ListRunAudit: %v", err) + } + var sawReset, sawFinish bool + for _, a := range audits { + switch a.Transition { + case TransitionResetDestination: + sawReset = true + if !a.Note.Valid || a.Note.String == "" { + t.Fatalf("reset-destination note is empty: %+v", a) + } + case TransitionFinish: + sawFinish = true + } + } + if !sawReset || !sawFinish { + t.Fatalf("audit transitions = %+v, want both reset-destination and finish", audits) + } + + // The original index run is preserved — reset never prunes runs. + if _, err := s.GetRun(ctx, runID); err != nil { + t.Fatalf("original index run %d lost: %v", runID, err) + } +} + +// TestResetDestinationEmpty: a destination with no recorded state reports +// empty counts, so the CLI can report "nothing to reset" rather than mint a +// run. +func TestResetDestinationEmpty(t *testing.T) { + s := openTestStore(t) + counts, err := s.CountDestinationRecordedState(context.Background(), "never-used") + if err != nil { + t.Fatalf("CountDestinationRecordedState: %v", err) + } + if !counts.Empty() { + t.Fatalf("counts = %+v, want empty for an unused destination", counts) + } +} + +// TestResetDestinationRejectsEmptyName: both entry points refuse a blank +// destination rather than clearing everything with an empty-string match. +func TestResetDestinationRejectsEmptyName(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + if _, err := s.CountDestinationRecordedState(ctx, ""); err == nil { + t.Fatalf("CountDestinationRecordedState(\"\") succeeded, want error") + } + if _, _, err := s.ResetDestination(ctx, ""); err == nil { + t.Fatalf("ResetDestination(\"\") succeeded, want error") + } +} diff --git a/store/runs_audit.go b/store/runs_audit.go index e74fb48..14be1d4 100644 --- a/store/runs_audit.go +++ b/store/runs_audit.go @@ -33,6 +33,13 @@ const ( // CHECK keeps destination NULL on audit rows, so this entry is where // the audit trail names the verified destination. TransitionVerifyDestination = "verify-destination" + // TransitionResetDestination records a `squirrel destination reset`: + // the operator forgetting a destination's recorded upload and + // durability state (ResetDestination). It shares the destination-scoped + // kind='audit' run shape, so — like TransitionVerifyDestination — this + // note is where the audit trail names the reset destination and carries + // the cleared counts, the runs CHECK keeping destination NULL on the row. + TransitionResetDestination = "reset-destination" ) // RunAudit is one row of the insert-only runs_audit log: a single From 599dacf984b99058bd0eb204eec86bb16e4a3861 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:39:26 +0000 Subject: [PATCH 09/15] sync: recognise an empty remote root as a fresh start (F20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The packed and content-addressed layout guards keyed on run history by destination name: a recorded success whose manifest segment / pack map is absent was refused outright, so following the guard's own advice (a fresh root) still refused. Now, when the marker is missing, the guard probes the configured root; an empty root — a wiped or repointed destination, or one cleared by `squirrel destination reset` — is treated as a fresh start and re-uploaded. A non-empty root, or any error probing it, keeps the refusal (fail-closed). The refusal message now also points at `destination reset`. Adds Rclone.remoteRootEmpty (recursive lsf) and a shared freshStartOnEmptyRoot helper. The fake-rclone test shim learns `lsf`; the mirror-era guard tests now seed a remote file so they exercise a genuine layout conflict, and new tests cover the empty-root fresh start on both layouts. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- sync/content_addressed.go | 5 ++- sync/content_addressed_test.go | 67 ++++++++++++++++++++++++++++++++++ sync/packed.go | 5 ++- sync/packed_test.go | 33 +++++++++++++++++ sync/rclone.go | 21 +++++++++++ sync/sync.go | 16 ++++++++ 6 files changed, 145 insertions(+), 2 deletions(-) diff --git a/sync/content_addressed.go b/sync/content_addressed.go index bbe0ecc..0ffc32f 100644 --- a/sync/content_addressed.go +++ b/sync/content_addressed.go @@ -211,7 +211,10 @@ func (h *contentAddressedHandler) watermark(ctx context.Context, volID int64) (i } segURI := h.segmentURI(last.ID) if _, err := h.rcl.statRemote(ctx, segURI, checkersArgs(h.dest)...); err != nil { - return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no manifest segment at %s — its history does not look content-addressed; point the layout at a fresh destination or root instead of switching an existing one: %w", h.dest.Name, last.ID, segURI, err) + if freshStartOnEmptyRoot(ctx, h.rcl, h.dest) { + return 0, nil + } + return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no manifest segment at %s — its history does not look content-addressed; point the layout at a fresh destination or root, or (after wiping the remote root) run `squirrel destination reset %s`, instead of switching an existing one: %w", h.dest.Name, last.ID, segURI, h.dest.Name, err) } return last.ID, nil } diff --git a/sync/content_addressed_test.go b/sync/content_addressed_test.go index fbb3f6d..d5f29e4 100644 --- a/sync/content_addressed_test.go +++ b/sync/content_addressed_test.go @@ -64,6 +64,7 @@ while [ $# -gt 0 ]; do --hash-type) shift; hashtypes="$hashtypes $1" ;; --include) shift; includes="$includes $1" ;; --checkers) shift ;; + -R) ;; --*) ;; *) if [ -z "$a1" ]; then a1="$1"; else a2="$1"; fi ;; esac @@ -119,6 +120,22 @@ lsjson) printf ']\n' fi ;; +lsf) + # Directory listing for the layout guard's emptiness probe. Resolve the + # base directory ignoring any crypt filename suffix — we only need to + # know whether any file exists beneath the root — then list recursively. + case "$a1" in + *:*) + p="${a1#*:}" + case "$p" in + "${RCLONE_FAKE_STRIP:-//none//}"/*) p="${p#"${RCLONE_FAKE_STRIP}"/}" ;; + esac + dir="$RCLONE_FAKE_ROOT/$p" ;; + *) dir="$a1" ;; + esac + [ -d "$dir" ] && find "$dir" -type f + exit 0 + ;; *) echo "unexpected rclone subcommand: $cmd $*" >&2; exit 64 ;; esac ` @@ -622,6 +639,10 @@ func TestContentAddressedWatermarkGuard(t *testing.T) { if err := f.store.FinishRun(ctx, mirrorRun, store.RunStatusSuccess, "", 1); err != nil { t.Fatalf("finish mirror-era run: %v", err) } + // A real mirror era leaves the volume's files mirrored under the root, + // so the root is non-empty: a genuine layout conflict, distinct from a + // wiped destination (which the fresh-start recognition below handles). + seedRemoteFile(t, f, "pics", "a.txt") rep, err := f.sync(t) if err == nil || !strings.Contains(err.Error(), "does not look content-addressed") { @@ -632,6 +653,52 @@ func TestContentAddressedWatermarkGuard(t *testing.T) { } } +// seedRemoteFile writes one file under the fixture's remote root, standing +// in for the bytes a prior (mirror) era left there so the root is non-empty. +func seedRemoteFile(t *testing.T, f *caFixture, parts ...string) { + t.Helper() + p := f.remotePath(parts...) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("mkdir remote file dir: %v", err) + } + if err := os.WriteFile(p, []byte("mirror-era-bytes"), 0o644); err != nil { + t.Fatalf("seed remote file: %v", err) + } +} + +// TestContentAddressedFreshStartOnEmptyRoot is the F20 fix: when name-keyed +// run history records a prior success but the configured root is empty on the +// remote (a wiped or repointed destination, or one cleared by +// `squirrel destination reset`), the guard treats it as a fresh start and +// re-uploads instead of refusing. +func TestContentAddressedFreshStartOnEmptyRoot(t *testing.T) { + f := setupContentAddressedFixture(t) + f.write(t, "a.txt", "alpha") + f.index(t) + + ctx := context.Background() + staleRun, err := f.store.BeginRun(ctx, store.RunKindSync, f.volumeID(t), "offsite", false) + if err != nil { + t.Fatalf("seed stale run: %v", err) + } + if err := f.store.FinishRun(ctx, staleRun, store.RunStatusSuccess, "", 1); err != nil { + t.Fatalf("finish stale run: %v", err) + } + // Remote root left empty — no seedRemoteFile — so the guard reads it as + // a fresh start. + + rep, err := f.sync(t) + if err != nil { + t.Fatalf("fresh-start sync failed: %v (rep=%+v)", err, rep) + } + if rep.Status != store.RunStatusSuccess { + t.Fatalf("Status = %q, want success", rep.Status) + } + if _, err := os.Stat(f.remoteBlob(ObjectsDirName, blake3Hex("alpha"))); err != nil { + t.Fatalf("fresh start did not re-upload the object: %v", err) + } +} + func TestContentAddressedDryRunRefused(t *testing.T) { f := setupContentAddressedFixture(t) f.write(t, "a.txt", "alpha") diff --git a/sync/packed.go b/sync/packed.go index 88bca90..2f5d5a3 100644 --- a/sync/packed.go +++ b/sync/packed.go @@ -264,7 +264,10 @@ func (h *packedHandler) watermark(ctx context.Context, volID int64) (int64, erro } mapURI := h.mapURI(last.ID) if _, err := h.rcl.statRemote(ctx, mapURI, checkersArgs(h.dest)...); err != nil { - return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no pack placement map at %s — its history is not packed (a mirror or content-addressed root); point the layout at a fresh destination or root instead of switching an existing one: %w", h.dest.Name, last.ID, mapURI, err) + if freshStartOnEmptyRoot(ctx, h.rcl, h.dest) { + return 0, nil + } + return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no pack placement map at %s — its history is not packed (a mirror or content-addressed root); point the layout at a fresh destination or root, or (after wiping the remote root) run `squirrel destination reset %s`, instead of switching an existing one: %w", h.dest.Name, last.ID, mapURI, h.dest.Name, err) } return last.ID, nil } diff --git a/sync/packed_test.go b/sync/packed_test.go index ba285b3..10dcc6d 100644 --- a/sync/packed_test.go +++ b/sync/packed_test.go @@ -450,6 +450,9 @@ func TestPackedWatermarkGuardRefusesMirror(t *testing.T) { if err := f.store.FinishRun(ctx, mirrorRun, store.RunStatusSuccess, "", 1); err != nil { t.Fatalf("finish mirror run: %v", err) } + // A real mirror era leaves files under the root, so the root is + // non-empty — a genuine layout conflict, not a wiped destination. + seedRemoteFile(t, f, "pics", "a.txt") rep, err := f.sync(t) if err == nil || !strings.Contains(err.Error(), "not packed") { t.Fatalf("expected packed-history refusal, got %v", err) @@ -459,6 +462,36 @@ func TestPackedWatermarkGuardRefusesMirror(t *testing.T) { } } +// TestPackedFreshStartOnEmptyRoot is the packed analogue of the F20 fix: a +// prior success recorded under this destination name, but an empty remote +// root, is a fresh start rather than a refusal. +func TestPackedFreshStartOnEmptyRoot(t *testing.T) { + f := setupPackedFixture(t, "1MiB") + f.write(t, "a.txt", "tiny") + f.index(t) + + ctx := context.Background() + staleRun, err := f.store.BeginRun(ctx, store.RunKindSync, f.volumeID(t), "offsite", false) + if err != nil { + t.Fatalf("seed stale run: %v", err) + } + if err := f.store.FinishRun(ctx, staleRun, store.RunStatusSuccess, "", 1); err != nil { + t.Fatalf("finish stale run: %v", err) + } + // Remote root left empty — the guard reads it as a fresh start. + + rep, err := f.sync(t) + if err != nil { + t.Fatalf("fresh-start sync failed: %v (rep=%+v)", err, rep) + } + if rep.Status != store.RunStatusSuccess { + t.Fatalf("Status = %q, want success", rep.Status) + } + if _, err := os.Stat(f.remoteBlob(PacksDirName, fmt.Sprintf("map-%d", rep.RunID))); err != nil { + t.Fatalf("fresh start wrote no placement map: %v", err) + } +} + // TestPackedDryRunRefused: the packed push has no dry-run mode yet. func TestPackedDryRunRefused(t *testing.T) { f := setupPackedFixture(t, "1MiB") diff --git a/sync/rclone.go b/sync/rclone.go index 19316e7..f0d2c25 100644 --- a/sync/rclone.go +++ b/sync/rclone.go @@ -388,6 +388,27 @@ func (r *Rclone) deleteFile(ctx context.Context, fileURI string) error { return err } +// remoteRootEmpty reports whether rootURI holds no files, listing it +// recursively via `rclone lsf -R --files-only`. A missing root (rclone +// exits non-zero with empty output, as when the directory was never +// created) counts as empty, mirroring listSnapshots' treatment of an +// absent snapshot directory. The layout guards use it to recognise a +// wiped or repointed destination — including one whose recorded state was +// cleared by `squirrel destination reset` — as a fresh start rather than +// a layout conflict. A genuine listing error (non-empty stderr, empty +// stdout) surfaces so the caller can stay fail-closed and refuse. +func (r *Rclone) remoteRootEmpty(ctx context.Context, rootURI string, extraArgs ...string) (bool, error) { + args := append([]string{"lsf", "-R", "--files-only"}, extraArgs...) + out, err := r.runPlain(ctx, append(args, rootURI)...) + if err != nil { + if len(bytes.TrimSpace(out)) == 0 { + return true, nil + } + return false, err + } + return len(bytes.TrimSpace(out)) == 0, nil +} + // statRemote returns the size of the single object at uri via // `rclone lsjson --stat`. The content-addressed push uses it to confirm // presence and size of each uploaded object and manifest segment; diff --git a/sync/sync.go b/sync/sync.go index 24a1ca1..e8dbbc8 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -634,6 +634,22 @@ func remoteSubpathURI(dest *config.Destination, subpath string) string { } } +// freshStartOnEmptyRoot reports whether a missing layout marker (a manifest +// segment or pack placement map absent at the last recorded success) should +// be read as a fresh start rather than a layout conflict. It is true only +// when dest's configured root holds no files on the remote — a wiped or +// repointed destination, or one whose recorded state was cleared by +// `squirrel destination reset` — so the name-keyed run history no longer +// describes anything on disk and a fresh full push is safe (it skips +// nothing). A non-empty root, or any error probing it, returns false so the +// caller keeps its refusal: fail-closed, because refusing a real +// layout-switch is recoverable while a delta against a stale watermark +// silently skips content. +func freshStartOnEmptyRoot(ctx context.Context, rcl *Rclone, dest *config.Destination) bool { + empty, err := rcl.remoteRootEmpty(ctx, remoteSubpathURI(dest, ""), checkersArgs(dest)...) + return err == nil && empty +} + // destinationVolumeURI returns the rclone destination spec for the given // volume under dest. func destinationVolumeURI(dest *config.Destination, volumeName string) string { From 9ec30093fc9d43ca211b959258fec5e22f551d4f Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:39:36 +0000 Subject: [PATCH 10/15] cmd: add `destination reset` and signpost machine replacement (F20, F28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `squirrel destination reset <name>` is a weighty change command: it prints what it will clear, refuses without --yes, and offers --dry-run — mirroring ux-principle 2. Parent `destination` command and the `reset` subcommand live in separate files. A configured-but-unrecorded destination reports "nothing to reset" rather than minting an empty run. restore's node dead-end ("restore from node destinations is not supported") now points at the machine-replacement runbook — rebuilding an edge machine from its hub is a reverse peer push driven from the hub, not a restore. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/destination.go | 19 ++++ cmd/squirrel/destination_reset.go | 117 +++++++++++++++++++++ cmd/squirrel/destination_reset_test.go | 134 +++++++++++++++++++++++++ cmd/squirrel/restore.go | 8 +- cmd/squirrel/root.go | 1 + 5 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 cmd/squirrel/destination.go create mode 100644 cmd/squirrel/destination_reset.go create mode 100644 cmd/squirrel/destination_reset_test.go diff --git a/cmd/squirrel/destination.go b/cmd/squirrel/destination.go new file mode 100644 index 0000000..a9081b7 --- /dev/null +++ b/cmd/squirrel/destination.go @@ -0,0 +1,19 @@ +package main + +import "github.com/spf13/cobra" + +// newDestinationCmd returns the `squirrel destination` parent command +// grouping the destination-level recovery verbs. It has no behaviour of its +// own and prints help when invoked bare. +func newDestinationCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "destination", + Short: "Manage a destination's recorded state", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, + } + cmd.AddCommand(newDestinationResetCmd()) + return cmd +} diff --git a/cmd/squirrel/destination_reset.go b/cmd/squirrel/destination_reset.go new file mode 100644 index 0000000..88e649b --- /dev/null +++ b/cmd/squirrel/destination_reset.go @@ -0,0 +1,117 @@ +package main + +import ( + "fmt" + "io" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/store" +) + +// newDestinationResetCmd returns `squirrel destination reset <destination>`: +// the explicit operator verb that forgets a destination's recorded upload and +// durability state so the next sync treats it as fresh (friction log F20). +// It is a *change* command in the ux-principles sense — weighty and +// irreversible in effect — so it defaults to refusing, offers a --dry-run +// preview, and requires an explicit --yes to proceed. The runs table and the +// append-only durability advance log are preserved; the reset itself is +// recorded as an audit run. +func newDestinationResetCmd() *cobra.Command { + var ( + yes bool + dryRun bool + ) + cmd := &cobra.Command{ + Use: "reset <destination>", + Short: "Forget a destination's recorded upload and durability state (audit-preserving)", + Long: `Forget everything the index records about a destination's remote state: +its per-content and per-pack upload ledgers, its live durability vector, and +its push-freshness maxima. The next sync then treats the destination as fresh +and re-uploads. + +Use it to recover a wrecked or repointed destination — e.g. after wiping the +remote root, or pointing an existing destination name at a fresh root, where +the layout guard would otherwise refuse on name-keyed run history alone. + +This clears derived state only. The runs table and the append-only durability +advance log are untouched, so the audit trail survives; the reset is itself +recorded as an audit run. The remote bytes are not deleted — wipe or repoint +the remote root separately if you want the destination genuinely empty.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runDestinationReset(cmd, args[0], yes, dryRun) + }, + } + cmd.Flags().BoolVar(&yes, "yes", false, "confirm the reset; required to actually clear recorded state") + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "print what would be cleared without changing anything") + return cmd +} + +func runDestinationReset(cmd *cobra.Command, name string, yes, dryRun bool) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + if _, ok := cfg.Destinations[name]; !ok { + return fmt.Errorf("unknown destination %q (declare it in %s); destination reset clears a bucket destination's recorded state — a peer node's durability assertions are revoked separately", name, cfg.Path) + } + + s, err := openStore(cmd, cfg) + if err != nil { + return err + } + defer s.Close() + + counts, err := s.CountDestinationRecordedState(cmd.Context(), name) + if err != nil { + return err + } + out := cmd.OutOrStdout() + if counts.Empty() { + fmt.Fprintf(out, "destination %q has no recorded state — nothing to reset\n", name) + return nil + } + if dryRun { + printResetPreview(out, name, counts, true) + return nil + } + if !yes { + printResetPreview(out, name, counts, false) + return fmt.Errorf("refusing to clear recorded state without confirmation; re-run with --yes (or --dry-run to preview)") + } + + runID, cleared, err := s.ResetDestination(cmd.Context(), name) + if err != nil { + return err + } + printResetResult(out, name, cleared, runID) + return nil +} + +// printResetPreview renders the per-category counts that a reset would clear. +// dryRun toggles only the leading verb so the same table serves both the +// explicit --dry-run preview and the confirmation gate. +func printResetPreview(out io.Writer, name string, c store.DestinationResetCounts, dryRun bool) { + prefix := "reset would clear" + if !dryRun { + prefix = "this will clear" + } + fmt.Fprintf(out, "%s destination %q recorded state:\n", prefix, name) + printResetCounts(out, c) + fmt.Fprintln(out, " (runs history and the durability advance log are preserved; remote bytes are untouched)") +} + +// printResetResult renders the loud confirmation after a reset ran, naming +// the audit run that recorded it and what to do next. +func printResetResult(out io.Writer, name string, c store.DestinationResetCounts, runID int64) { + fmt.Fprintf(out, "reset destination %q (run %d):\n", name, runID) + printResetCounts(out, c) + fmt.Fprintf(out, "next: the next sync re-uploads to %q; for a content-addressed or packed layout, wipe or repoint the remote root so the layout guard sees a fresh start\n", name) +} + +func printResetCounts(out io.Writer, c store.DestinationResetCounts) { + fmt.Fprintf(out, " upload records: %d objects, %d packs\n", c.RemoteObjects, c.RemotePacks) + fmt.Fprintf(out, " durability vector: %d component(s)\n", c.VectorComponents) + fmt.Fprintf(out, " push freshness: %d row(s)\n", c.FreshnessRows) +} diff --git a/cmd/squirrel/destination_reset_test.go b/cmd/squirrel/destination_reset_test.go new file mode 100644 index 0000000..d8f9a5a --- /dev/null +++ b/cmd/squirrel/destination_reset_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/mbertschler/squirrel/store" +) + +// writeResetConfig writes a config with one local destination "arch" and a +// db path, returning the config path and the db path. +func writeResetConfig(t *testing.T) (configPath, dbPath string) { + t.Helper() + dir := t.TempDir() + dbPath = filepath.Join(dir, "index.db") + destDir := filepath.Join(dir, "arch") + body := fmt.Sprintf("db = %q\n\n[destinations.arch]\ntype = \"local\"\nroot = %q\n", dbPath, destDir) + configPath = writeCheckConfig(t, body) + return configPath, dbPath +} + +// seedResetVector opens the fixture DB directly and records one durability +// vector component for destination, so the reset has something to clear. +func seedResetVector(t *testing.T, dbPath, destination string) { + t.Helper() + ctx := context.Background() + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + defer s.Close() + v, err := s.GetOrCreateVolume(ctx, "/v") + if err != nil { + t.Fatalf("GetOrCreateVolume: %v", err) + } + self, err := s.GetSelfNode(ctx) + if err != nil { + t.Fatalf("GetSelfNode: %v", err) + } + if err := s.UpsertDestinationRunIDVerified(ctx, v.ID, destination, self.ID, 5, store.VerifyMethodBlake3, false); err != nil { + t.Fatalf("UpsertDestinationRunIDVerified: %v", err) + } +} + +// resetVectorCount reports how many vector components destination still has. +func resetVectorCount(t *testing.T, dbPath, destination string) int { + t.Helper() + ctx := context.Background() + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + defer s.Close() + v, err := s.GetOrCreateVolume(ctx, "/v") + if err != nil { + t.Fatalf("GetOrCreateVolume: %v", err) + } + vec, err := s.ListDestinationRunIDs(ctx, v.ID, destination) + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + return len(vec) +} + +// TestCLIDestinationResetUnknown: a destination not in config is refused +// before the store is touched. +func TestCLIDestinationResetUnknown(t *testing.T) { + cfgPath, _ := writeResetConfig(t) + out, err := runCLIExpectErr(t, "--config", cfgPath, "destination", "reset", "ghost", "--yes") + if !strings.Contains(out, "unknown destination") && !strings.Contains(err.Error(), "unknown destination") { + t.Fatalf("expected unknown-destination error, got out=%q err=%v", out, err) + } +} + +// TestCLIDestinationResetNothing: a configured destination with no recorded +// state reports "nothing to reset" affirmatively and mints no run. +func TestCLIDestinationResetNothing(t *testing.T) { + cfgPath, _ := writeResetConfig(t) + out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--yes") + if !strings.Contains(out, "nothing to reset") { + t.Fatalf("expected nothing-to-reset message:\n%s", out) + } +} + +// TestCLIDestinationResetDryRun: --dry-run previews the counts and clears +// nothing. +func TestCLIDestinationResetDryRun(t *testing.T) { + cfgPath, dbPath := writeResetConfig(t) + seedResetVector(t, dbPath, "arch") + + out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--dry-run") + if !strings.Contains(out, "would clear") { + t.Fatalf("expected dry-run preview:\n%s", out) + } + if n := resetVectorCount(t, dbPath, "arch"); n != 1 { + t.Fatalf("dry-run cleared state: vector count = %d, want 1", n) + } +} + +// TestCLIDestinationResetNeedsConfirmation: without --yes the command +// previews and refuses, changing nothing (principle 2: weighty change). +func TestCLIDestinationResetNeedsConfirmation(t *testing.T) { + cfgPath, dbPath := writeResetConfig(t) + seedResetVector(t, dbPath, "arch") + + out, err := runCLIExpectErr(t, "--config", cfgPath, "destination", "reset", "arch") + if !strings.Contains(err.Error(), "--yes") { + t.Fatalf("expected confirmation-required error, got %v", err) + } + if !strings.Contains(out, "this will clear") { + t.Fatalf("expected preview before refusal:\n%s", out) + } + if n := resetVectorCount(t, dbPath, "arch"); n != 1 { + t.Fatalf("refused reset still cleared state: vector count = %d, want 1", n) + } +} + +// TestCLIDestinationResetConfirmed: --yes clears the state and names the +// audit run that recorded it. +func TestCLIDestinationResetConfirmed(t *testing.T) { + cfgPath, dbPath := writeResetConfig(t) + seedResetVector(t, dbPath, "arch") + + out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--yes") + if !strings.Contains(out, "reset destination") || !strings.Contains(out, "run ") { + t.Fatalf("expected loud reset confirmation with run id:\n%s", out) + } + if n := resetVectorCount(t, dbPath, "arch"); n != 0 { + t.Fatalf("confirmed reset left state: vector count = %d, want 0", n) + } +} diff --git a/cmd/squirrel/restore.go b/cmd/squirrel/restore.go index deab82e..edb8339 100644 --- a/cmd/squirrel/restore.go +++ b/cmd/squirrel/restore.go @@ -195,9 +195,11 @@ func pickSingleRestoreDestination(cfg *config.Config, vol *config.Volume) (*conf } dest, ok := cfg.Destinations[vol.SyncTo[0]] if !ok { - // sync_to may name a node, not a bucket; restore doesn't yet - // pull from peer nodes, so surface that mismatch explicitly. - return nil, fmt.Errorf("volume %q syncs only to %q which is not a bucket destination — restore from node destinations is not supported", vol.Name, vol.SyncTo[0]) + // sync_to may name a node, not a bucket; restore doesn't pull from + // peer nodes. Rebuilding an edge machine from its hub is a reverse + // peer push driven from the hub — the same mechanism nas→htpc uses + // daily — not a restore, so point there rather than dead-ending. + return nil, fmt.Errorf("volume %q syncs only to peer node %q; restore pulls from bucket destinations, not nodes — to rebuild this machine from its hub, drive a reverse peer push from the hub (see the machine-replacement runbook in docs/guides/recovery)", vol.Name, vol.SyncTo[0]) } return dest, nil } diff --git a/cmd/squirrel/root.go b/cmd/squirrel/root.go index af77560..da59108 100644 --- a/cmd/squirrel/root.go +++ b/cmd/squirrel/root.go @@ -57,6 +57,7 @@ func newRootCmd() *cobra.Command { root.AddCommand(newDBCmd()) root.AddCommand(newConfigCmd()) root.AddCommand(newNodeCmd()) + root.AddCommand(newDestinationCmd()) return root } From bf1b7f43290c5325934aca6db2da1b990da543af Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:39:45 +0000 Subject: [PATCH 11/15] docs: recovery & disaster runbooks (F20, F28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New "Recovery & disaster runbooks" guide: resetting a wrecked destination, rebuilding a dead machine from its hub via a reverse peer push (re-pairing with `node pair`), and the manual DR that already works and stays — mirror restore, ride-along index-snapshot recovery, and packed/content-addressed recovery. Adds `squirrel destination reset` to the CLI reference and the guide to the sidebar. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- docs/astro.config.mjs | 1 + docs/src/content/docs/guides/recovery.md | 132 +++++++++++++++++++++++ docs/src/content/docs/reference/cli.md | 31 ++++++ 3 files changed, 164 insertions(+) create mode 100644 docs/src/content/docs/guides/recovery.md diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index 59f3d82..85a9acb 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -62,6 +62,7 @@ export default defineConfig({ { label: "Offsite verification", link: "/guides/verification/" }, { label: "Offloading", link: "/guides/offloading/" }, { label: "Restoring", link: "/guides/restore/" }, + { label: "Recovery & disaster runbooks", link: "/guides/recovery/" }, { label: "Hooks", link: "/guides/hooks/" }, { label: "Auditing for drift", link: "/guides/auditing/" }, { label: "Peer sync", link: "/guides/peer-sync/" }, diff --git a/docs/src/content/docs/guides/recovery.md b/docs/src/content/docs/guides/recovery.md new file mode 100644 index 0000000..09f70f9 --- /dev/null +++ b/docs/src/content/docs/guides/recovery.md @@ -0,0 +1,132 @@ +--- +title: Recovery & disaster runbooks +description: Reset a wrecked destination, rebuild a dead machine from its hub, and the manual disaster-recovery paths that already work — mirror restore, index-snapshot recovery, and packed/content-addressed recovery. +--- + +Recovery in squirrel is deliberately a set of **explicit, human-driven acts** +— never something the agent does on a cadence. Each one preserves the audit +trail: the runs table is never pruned, and no history is overwritten without +an opt-in operator command. This page collects the recovery paths. + +## Resetting a wrecked destination + +`squirrel destination reset <name>` forgets everything the index records about +a destination's remote state — its per-content and per-pack upload ledgers, its +live durability vector, and its push-freshness maxima — so the next sync treats +the destination as fresh and re-uploads. + +Use it when a destination's recorded state no longer matches reality and a sync +refuses to proceed. The classic case: after wiping a remote, or pointing an +existing destination name at a fresh `root`, the +[content-addressed](/squirrel/layouts/content-addressed/) or +[packed](/squirrel/layouts/packed/) layout guard refuses — its recorded history +under that destination name expects a manifest segment or placement map that is +no longer there. Before this verb, the only escape was hand-editing SQLite or +renaming the destination across every machine's config. + +```sh +squirrel destination reset s3archive --dry-run # preview what would clear +squirrel destination reset s3archive --yes # actually clear it +``` + +It is a **change** command, so it is weighty by design: it prints what it will +clear, and refuses without `--yes` (or a `--dry-run` preview). + +### What is and isn't cleared + +Cleared (derived state only): + +- `remote_objects` and `remote_packs` — the upload ledgers. +- the destination's durability vector and push-freshness rows. + +Preserved (the audit trail): + +- the **runs** table — every past sync, verify, and audit stays; the reset is + itself recorded as a new audit run. +- the append-only **durability advance log** — the record of what was ever + asserted durable survives, exactly as [revoking a peer](/squirrel/guides/peer-sync/) + leaves its history intact. +- the **content and files** rows — squirrel never loses track of content. + +The **remote bytes are not touched** — reset only clears squirrel's *record*. +If you want the destination genuinely empty, wipe or repoint the remote root +separately. Once the configured root is empty on the remote, the layout guard +recognises the fresh start on its own and the next sync re-uploads from scratch. + +:::note[Stop the agent first] +A running agent may kick a sync on its cadence mid-reset. Nothing is lost — a +content-addressed re-upload is idempotent — but pausing the agent keeps the +recovery legible. +::: + +## Rebuilding a machine from its hub + +When an edge machine dies (the most likely household disaster), rebuild it from +the hub with a **reverse peer push** — the same peer-sync mechanism the hub uses +to feed a receive-only node every day. There is no separate restore verb for +this: `squirrel restore` pulls from bucket destinations, not from peer nodes, +and the machinery to push a volume to a node already exists. + +The story, using the [reference setup](/squirrel/reference/configuration/) +(rebuilding `laptop` from `nas`): + +1. **Install and re-pair the replacement.** Install squirrel, generate its + agent cert, and re-establish the peer relationship with the hub. Use the + pairing helper so the token matrix is correct by construction: + + ```sh + squirrel agent cert # on the replacement: new cert + pin + squirrel node pair nas # emits matching config halves for both + ``` + + Paste each half into the named machine's config and fill any placeholders + (see [Peer sync](/squirrel/guides/peer-sync/)). The replacement runs an agent + that **listens** so the hub can dial it — the reverse of its normal + initiate-only role. + +2. **Point the hub at the replacement, temporarily.** On the hub, add the + replacement as a `[nodes.laptop]` entry and list it in the `sync_to` of the + volumes to restore. Then drive the push: + + ```sh + squirrel sync photos --to laptop + squirrel sync docs --to laptop + ``` + + The hub enumerates its master copy and pushes; the replacement receives and + re-hashes every path (`peer-blake3`), so the restore is content-verified end + to end. Both sides record correlated runs — the recovery is in the audit + trail like any other sync. + +3. **Return to steady state.** Once the replacement holds the volumes, remove + the temporary hub-side `sync_to`/`[nodes.laptop]` push config, and resume the + normal edge → hub direction (`laptop` initiating to `nas`). + +## Manual disaster recovery that already works + +These paths need no new tooling and are the right answer for whole-repository or +hub loss. They stay first-class. + +### Mirror restore + +For a [mirror](/squirrel/layouts/mirror/) destination (including an encrypted +one), [`squirrel restore`](/squirrel/guides/restore/) pulls the volume back +byte-for-byte, decrypting on the way down and verifying BLAKE3 where the +destination exposes hashes. + +### Recovering the index too + +squirrel rides an [index snapshot](/squirrel/configuration/index-snapshots/) +along to destination buckets under `.squirrel-index/`. When the hub itself dies, +a restore-from-cloud yields the data *and* the index that explains it — fetch +the ride-along snapshot and swap it in as the live index with +[`squirrel db restore`](/squirrel/reference/cli/#squirrel-db). `runs` and `query` +answer immediately against the recovered catalog. + +### Packed & content-addressed recovery + +The [content-addressed](/squirrel/layouts/content-addressed/) and +[packed](/squirrel/layouts/packed/) formats are deliberately simple enough to +[recover without squirrel](/squirrel/reference/formats/#disaster-recovery-without-squirrel) +— the manifest and pack formats are documented, so the bytes are recoverable +with standard tools even if squirrel is unavailable. diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index 07a992d..9d8d2ae 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -26,6 +26,7 @@ squirrel runs fail <id> squirrel hooks [--volume NAME] [--limit N] squirrel volumes squirrel restore <volume> [--from NAME] [--to PATH] [--shallow] [--dry-run] [--in-place] +squirrel destination reset <dest> [--yes] [--dry-run] squirrel audit [<volume>] [--deep | --folders] squirrel peer-sync history <volume> <peer> squirrel peer-sync pull-durability <volume> <peer> [--allow-rewind] @@ -229,6 +230,36 @@ because rclone crypt remotes don't expose content hashes. --- +## squirrel destination + +**Manage a destination's recorded state.** + +On its own it prints help. See [Recovery & disaster runbooks](/squirrel/guides/recovery/). + +### squirrel destination reset + +**Forget a destination's recorded upload and durability state (audit-preserving).** + +``` +squirrel destination reset <destination> +``` + +One positional — the destination name. Clears the `remote_objects`/`remote_packs` +upload ledgers, the durability vector, and the push-freshness rows for the +destination, so the next sync treats it as fresh and re-uploads. The runs table +and the append-only durability advance log are preserved, and the reset is +recorded as an audit run; the remote bytes are untouched. + +| Flag | Default | Meaning | +|---|---|---| +| `--yes` | `false` | Confirm the reset; required to actually clear state. | +| `--dry-run` | `false` | Print what would be cleared without changing anything. | + +Use it to recover a wrecked or repointed destination — see +[Resetting a wrecked destination](/squirrel/guides/recovery/#resetting-a-wrecked-destination). + +--- + ## squirrel audit **Walk a volume looking for out-of-band drift since the last index.** From 38650462d5f6d3d845a392421e58395353ebe5dc Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:40:55 +0000 Subject: [PATCH 12/15] design: mark F20 and F28 addressed by #176 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- design/friction-log.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/design/friction-log.md b/design/friction-log.md index 496c68f..0ded92f 100644 --- a/design/friction-log.md +++ b/design/friction-log.md @@ -205,7 +205,7 @@ TUI-side folding of consecutive no-ops would restore the audit trail's readability. (The `runs` help text also still says "List index runs"; it lists every kind.) -**F20 · S2 — recovering a wrecked destination has no supported path.** +**F20 · S2 — ~~recovering a wrecked destination has no supported path.~~ (`squirrel destination reset` + empty-root guard, #176)** After the F12 bug era the packed-layout guard refused every further s3archive sync ("its history is not packed … point the layout at a fresh destination or root"). But the guard keys on the pair's *run @@ -337,7 +337,7 @@ content-verified method and relay that — the schema already carries ## Checkpoint 8 — restore day -**F28 · S2 — a dead edge machine has no supported restore path.** The +**F28 · S2 — ~~a dead edge machine has no supported restore path.~~ (reverse-peer-push runbook + restore signpost, #176)** The laptop syncs only to the nas (a node), and `restore` refuses nodes outright ("restore from node destinations is not supported" — clear, at least). The machinery for the recovery *exists* — a reverse peer From 14e831a5c712722caf0edd774c8921c0fc86980c Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:42:13 +0000 Subject: [PATCH 13/15] Address Copilot review on #175 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - config/agent.go: the "token required" error now states the requirement is conditional on a listener — "auth.token is required when [agent] listen is set (... omit listen for a listener-less, scheduler-only agent)" — instead of the unconditional "no agent without authentication", which read as wrong now that listener-less mode exists. - cmd/squirrel/agent.go: the listener-less "nothing to run" fail-fast now lists hook.interval alongside sync_every/index_every/scan_interval, so the remediation matches the real scheduling rules (agentHasWork and the scheduler both count an interval hook as work). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/agent.go | 2 +- config/agent.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/squirrel/agent.go b/cmd/squirrel/agent.go index 8498156..e0e6130 100644 --- a/cmd/squirrel/agent.go +++ b/cmd/squirrel/agent.go @@ -88,7 +88,7 @@ func serveAgent(cmd *cobra.Command, cfg *config.Config, srv *agent.Server, logge // degradation, not a valid config. if cfg.Agent.Listen == "" { if !agentHasWork(cfg) { - return fmt.Errorf("listener-less agent has nothing to run: set [agent] listen to receive peer syncs, or a sync_every/index_every/scan_interval cadence, in %s", cfg.Path) + return fmt.Errorf("listener-less agent has nothing to run: set [agent] listen to receive peer syncs, or configure a cadence (a volume's sync_every, index_every, or hook.interval, or [agent] scan_interval) in %s", cfg.Path) } logSchedulerOnlyStartup(logger) return srv.RunSchedulers(cmd.Context()) diff --git a/config/agent.go b/config/agent.go index 77a146d..4cee31c 100644 --- a/config/agent.go +++ b/config/agent.go @@ -168,7 +168,7 @@ func resolveAgentAuth(r *rawAuth, a *Agent) error { // tokens would have no listener to guard either. return nil } - return errors.New("auth.token is required (no agent without authentication)") + return errors.New("auth.token is required when [agent] listen is set (the listener needs a bearer token; omit listen for a listener-less, scheduler-only agent)") } // resolveSecret takes a map[string]any and pulls the named key. We pass // a synthetic single-entry map so the same code handles plain strings From fbea42423b7ab48d9196bf329b0e429452da1045 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 11:48:57 +0000 Subject: [PATCH 14/15] sync,cmd: address Copilot review on #176 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FIX 1 (fail-closed): remoteRootEmpty no longer reads any lsf error with empty stdout as an empty root — runPlain carries stderr only in the error, so an auth/network failure was being mistaken for a wiped destination and letting the layout guard proceed. Now only rclone's canonical "directory not found" (or "object not found") counts as a fresh root; every other error surfaces so the guard fails closed. New test injects an auth-style lsf error (guard refuses) vs a not-found (fresh root). FIX 2: the restore node-restore signpost pointed at a repo source path (docs/guides/recovery) that doesn't exist for an installed binary; it now names the published docs route (guides/recovery). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- cmd/squirrel/restore.go | 2 +- sync/content_addressed_test.go | 38 +++++++++++++++++++++++++++++++--- sync/rclone.go | 35 +++++++++++++++++++++++-------- 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/cmd/squirrel/restore.go b/cmd/squirrel/restore.go index edb8339..f92c1fa 100644 --- a/cmd/squirrel/restore.go +++ b/cmd/squirrel/restore.go @@ -199,7 +199,7 @@ func pickSingleRestoreDestination(cfg *config.Config, vol *config.Volume) (*conf // peer nodes. Rebuilding an edge machine from its hub is a reverse // peer push driven from the hub — the same mechanism nas→htpc uses // daily — not a restore, so point there rather than dead-ending. - return nil, fmt.Errorf("volume %q syncs only to peer node %q; restore pulls from bucket destinations, not nodes — to rebuild this machine from its hub, drive a reverse peer push from the hub (see the machine-replacement runbook in docs/guides/recovery)", vol.Name, vol.SyncTo[0]) + return nil, fmt.Errorf("volume %q syncs only to peer node %q; restore pulls from bucket destinations, not nodes — to rebuild this machine from its hub, drive a reverse peer push from the hub (see the machine-replacement runbook: the Recovery guide, guides/recovery)", vol.Name, vol.SyncTo[0]) } return dest, nil } diff --git a/sync/content_addressed_test.go b/sync/content_addressed_test.go index d5f29e4..93d968e 100644 --- a/sync/content_addressed_test.go +++ b/sync/content_addressed_test.go @@ -121,9 +121,13 @@ lsjson) fi ;; lsf) - # Directory listing for the layout guard's emptiness probe. Resolve the - # base directory ignoring any crypt filename suffix — we only need to - # know whether any file exists beneath the root — then list recursively. + # Directory listing for the layout guard's emptiness probe. An injected + # error simulates a not-found (fresh root) or a real failure (auth etc.) + # so the guard's fail-closed classification can be tested. + if [ -n "$RCLONE_FAKE_LSF_ERROR" ]; then echo "$RCLONE_FAKE_LSF_ERROR" >&2; exit 1; fi + # Resolve the base directory ignoring any crypt filename suffix — we only + # need to know whether any file exists beneath the root — then list + # recursively. case "$a1" in *:*) p="${a1#*:}" @@ -198,6 +202,7 @@ func setupCAFixture(t *testing.T, destBlock, strip string) *caFixture { t.Setenv("RCLONE_FAKE_HASH_VALUE", "") t.Setenv("RCLONE_FAKE_HASH_PREFIX", "") t.Setenv("RCLONE_FAKE_CRYPT_SUFFIX", "") + t.Setenv("RCLONE_FAKE_LSF_ERROR", "") root := t.TempDir() volPath := filepath.Join(root, "src") @@ -653,6 +658,33 @@ func TestContentAddressedWatermarkGuard(t *testing.T) { } } +// TestRemoteRootEmptyClassifiesErrors guards the layout guard's fail-closed +// contract: only rclone's canonical not-found reads as an empty (fresh) +// root; any other lsf error must surface so the guard refuses rather than +// mistaking a broken probe (auth, network) for a wiped destination. +func TestRemoteRootEmptyClassifiesErrors(t *testing.T) { + f := setupContentAddressedFixture(t) + ctx := context.Background() + + // A canonical directory-not-found is a fresh root. + t.Setenv("RCLONE_FAKE_LSF_ERROR", "2020/01/01 ERROR : directory not found") + empty, err := f.rcl.remoteRootEmpty(ctx, "offsite:/data") + if err != nil || !empty { + t.Fatalf("not-found lsf = (%t, %v), want (true, nil)", empty, err) + } + + // An auth/network error must surface so the guard fails closed — never + // reported as an empty root just because stdout was empty. + t.Setenv("RCLONE_FAKE_LSF_ERROR", "Failed to create file system: 401 Unauthorized") + empty, err = f.rcl.remoteRootEmpty(ctx, "offsite:/data") + if err == nil { + t.Fatalf("auth lsf error swallowed (empty=%t); guard would proceed instead of refusing", empty) + } + if empty { + t.Fatalf("auth lsf error reported empty root; must be false") + } +} + // seedRemoteFile writes one file under the fixture's remote root, standing // in for the bytes a prior (mirror) era left there so the root is non-empty. func seedRemoteFile(t *testing.T, f *caFixture, parts ...string) { diff --git a/sync/rclone.go b/sync/rclone.go index f0d2c25..5bc34fd 100644 --- a/sync/rclone.go +++ b/sync/rclone.go @@ -389,19 +389,22 @@ func (r *Rclone) deleteFile(ctx context.Context, fileURI string) error { } // remoteRootEmpty reports whether rootURI holds no files, listing it -// recursively via `rclone lsf -R --files-only`. A missing root (rclone -// exits non-zero with empty output, as when the directory was never -// created) counts as empty, mirroring listSnapshots' treatment of an -// absent snapshot directory. The layout guards use it to recognise a -// wiped or repointed destination — including one whose recorded state was -// cleared by `squirrel destination reset` — as a fresh start rather than -// a layout conflict. A genuine listing error (non-empty stderr, empty -// stdout) surfaces so the caller can stay fail-closed and refuse. +// recursively via `rclone lsf -R --files-only`. Only rclone's canonical +// "directory not found" (a root that was never created) — accepting the +// "object not found" phrasing too — counts as an empty/fresh root. Every +// other error (auth, network, permission) surfaces so the layout guards +// stay fail-closed and refuse rather than mistaking a broken probe for a +// wiped destination: runPlain carries stderr only in the returned error, +// not in stdout, so an empty stdout alone must never be read as "empty". +// The guards use the empty result to recognise a wiped or repointed +// destination — including one whose recorded state was cleared by +// `squirrel destination reset` — as a fresh start rather than a layout +// conflict. func (r *Rclone) remoteRootEmpty(ctx context.Context, rootURI string, extraArgs ...string) (bool, error) { args := append([]string{"lsf", "-R", "--files-only"}, extraArgs...) out, err := r.runPlain(ctx, append(args, rootURI)...) if err != nil { - if len(bytes.TrimSpace(out)) == 0 { + if isRemoteNotFound(err) { return true, nil } return false, err @@ -409,6 +412,20 @@ func (r *Rclone) remoteRootEmpty(ctx context.Context, rootURI string, extraArgs return len(bytes.TrimSpace(out)) == 0, nil } +// isRemoteNotFound reports whether err is rclone's canonical +// "directory not found" (or "object not found") — the only listing failure +// the layout guard reads as an empty, fresh root. Any other error must +// surface so the guard fails closed. #150's isNotFoundErr is not on this +// branch, so the canonical-phrase check lives here; runPlain formats +// rclone's stderr into the error string, so matching err.Error() catches it. +func isRemoteNotFound(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "directory not found") || strings.Contains(msg, "object not found") +} + // statRemote returns the size of the single object at uri via // `rclone lsjson --stat`. The content-addressed push uses it to confirm // presence and size of each uploaded object and manifest segment; From e40ee5ab123a19a8c5bcbc9259c3e9e7197ea7e2 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Fri, 24 Jul 2026 17:38:36 +0000 Subject: [PATCH 15/15] agent,config: fix stale "required" doc comments for listener-less mode F35 made Listen optional and Token conditional on Listen, but three doc comments on the public Config/Agent structs still declared them unconditionally required, contradicting the relaxed validateConfig / resolveAgentAuth. Correct the comments so the documented contract matches the validated one. Doc-only; no behaviour change. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014Skm2Qm6wWUB7dD4CpPTz7 --- agent/agent.go | 8 +++++--- config/agent.go | 10 ++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index a22d2b2..2c08802 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -54,9 +54,11 @@ type SyncRunReport struct { Err error } -// Config configures one agent listener. Fields are validated by New; all -// of them are required except TLSCert/TLSKey (which must be set together -// or not at all — empty pair means plain HTTP). +// Config configures one agent. Fields are validated by New: Version is +// always required; Listen is optional (empty selects listener-less, +// scheduler-only mode, F35); Token is required only when Listen is set (an +// HTTP surface needs a bearer token); TLSCert/TLSKey must be set together +// or not at all — empty pair means plain HTTP. type Config struct { // Listen is the bind address passed to net.Listen, e.g. "0.0.0.0:8443". Listen string diff --git a/config/agent.go b/config/agent.go index 4cee31c..559faa0 100644 --- a/config/agent.go +++ b/config/agent.go @@ -17,7 +17,8 @@ const ( // callers try to start without it. type Agent struct { // Listen is the TCP address the agent binds to, e.g. "0.0.0.0:8443". - // Required. + // Optional: an empty value selects listener-less, scheduler-only mode + // (F35) — the agent runs its cadences without binding an HTTP server. Listen string // DB optionally overrides the top-level db for the agent process. The // agent binary resolves --db > Agent.DB > top-level db > default; an @@ -28,9 +29,10 @@ type Agent struct { // empty disables TLS (plain HTTP). TLSCert string TLSKey string - // Token is the resolved bearer token literal. Required: an agent - // without a token is an unauthenticated open port and we refuse to - // start one. + // Token is the resolved bearer token literal. Required only when Listen + // is set: an agent with a listener but no token is an unauthenticated + // open port and we refuse to start one. A listener-less agent has no + // HTTP surface and needs no token. Token string // PeerTokens maps a per-peer bearer token to the node name that // presents it, so the agent can recover an authenticated caller