diff --git a/README.md b/README.md index 0318624..8ddc6a6 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ password = { env = "OFFSITE_CRYPT_PASSWORD" } password2 = { env = "OFFSITE_CRYPT_SALT" } # salt — optional but recommended ``` -`password` and `password2` are **rclone-obscured** values, the same representation `rclone config` stores for its own crypt remotes — generate one with `rclone obscure `. Both accept a literal or `{ env = "VAR" }`. Squirrel renders two sections into its `rclone.conf` — the underlying remote plus a crypt remote wrapping it — and addresses all sync and restore transfers through the crypt remote. Keep the passwords safe: restoring from an encrypted destination requires them. +`password` and `password2` are **plaintext** (a literal or `{ env = "VAR" }`); squirrel obscures them into rclone's on-disk form when it renders `rclone.conf`, so you never run `rclone obscure` yourself. A config written before this behaviour, whose values are already obscured, keeps them verbatim by adding `obscured = true` to the crypt block. Squirrel renders two sections into its `rclone.conf` — the underlying remote plus a crypt remote wrapping it — and addresses all sync and restore transfers through the crypt remote. Keep the passwords safe: restoring from an encrypted destination requires them. Two properties to be aware of: diff --git a/agent/agent.go b/agent/agent.go index f0822c8..09752ce 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -54,9 +54,11 @@ type SyncRunReport struct { Err error } -// Config configures one agent listener. Fields are validated by New; all -// of them are required except TLSCert/TLSKey (which must be set together -// or not at all — empty pair means plain HTTP). +// Config configures one agent. Fields are validated by New: Version is +// always required; Listen is optional (empty selects listener-less, +// scheduler-only mode, F35); Token is required only when Listen is set (an +// HTTP surface needs a bearer token); TLSCert/TLSKey must be set together +// or not at all — empty pair means plain HTTP. type Config struct { // Listen is the bind address passed to net.Listen, e.g. "0.0.0.0:8443". Listen string @@ -182,16 +184,28 @@ func (s *Server) Handler() http.Handler { return s.handler } // dial. func (s *Server) HasTLS() bool { return s.cfg.TLSCert != "" } +// CertFingerprint returns the sha256: pin of the agent's configured TLS +// certificate — the value the startup banner prints so an operator can see +// what peers must pin. It errors when the agent serves plain HTTP (no cert) +// or the certificate file cannot be read. +func (s *Server) CertFingerprint() (string, error) { + if s.cfg.TLSCert == "" { + return "", errors.New("agent: no TLS certificate configured") + } + return FingerprintCertFile(s.cfg.TLSCert) +} + // Addr returns the configured listen address verbatim. For `:0`-style // binds the kernel-assigned port is only knowable from the net.Listener // the caller hands to Serve; this accessor is for the startup banner. func (s *Server) Addr() string { return s.cfg.Listen } func validateConfig(cfg Config) error { - if cfg.Listen == "" { - return errors.New("agent: Config.Listen is required") - } - if cfg.Token == "" { + // An empty Listen selects listener-less mode (F35): the agent runs only + // its background schedulers, so neither a bind address nor a bearer + // token is required. A token is required only when there is an HTTP + // surface to protect. + if cfg.Listen != "" && cfg.Token == "" { return errors.New("agent: Config.Token is required") } if (cfg.TLSCert == "") != (cfg.TLSKey == "") { diff --git a/agent/agent_test.go b/agent/agent_test.go index 9606f21..9f36fc0 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -64,7 +64,6 @@ func TestNewRejectsBadConfig(t *testing.T) { cfg Config want string }{ - {"no listen", Config{Token: "t", Version: "v"}, "Listen is required"}, {"no token", Config{Listen: ":0", Version: "v"}, "Token is required"}, {"half tls", Config{Listen: ":0", Token: "t", TLSCert: "c", Version: "v"}, "must be set together"}, {"no version", Config{Listen: ":0", Token: "t"}, "Version is required"}, @@ -79,6 +78,43 @@ func TestNewRejectsBadConfig(t *testing.T) { } } +// TestNewListenerLess covers F35: an empty Listen (and no Token) is a +// valid scheduler-only agent, and the server reports no listener/TLS. +func TestNewListenerLess(t *testing.T) { + srv, err := New(Config{Version: "v"}, openTestStore(t)) + if err != nil { + t.Fatalf("New (listener-less): %v", err) + } + if srv.Addr() != "" { + t.Fatalf("Addr = %q, want empty for a listener-less agent", srv.Addr()) + } + if srv.HasTLS() { + t.Fatal("listener-less agent unexpectedly reports TLS") + } +} + +// TestRunSchedulersReturnsOnCancel pins that the listener-less run path +// blocks until its context is cancelled and then returns cleanly, without +// ever binding a listener. +func TestRunSchedulersReturnsOnCancel(t *testing.T) { + srv, err := New(Config{Version: "v"}, openTestStore(t)) + if err != nil { + t.Fatalf("New: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- srv.RunSchedulers(ctx) }() + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("RunSchedulers: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("RunSchedulers did not return after context cancel") + } +} + // TestNewDefaultsLoggerToDiscard pins the contract that callers may // leave Config.Logger nil and the agent will still be safe to use: // New() substitutes a discard logger so the future scheduler can diff --git a/agent/cert.go b/agent/cert.go new file mode 100644 index 0000000..cdcc27f --- /dev/null +++ b/agent/cert.go @@ -0,0 +1,127 @@ +package agent + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "time" +) + +// SelfSignedCert bundles a freshly generated self-signed certificate, its +// private key (both PEM-encoded), and the sha256: fingerprint pin peers +// paste into their [nodes.X.tls] cert_fingerprint. +type SelfSignedCert struct { + CertPEM []byte + KeyPEM []byte + Fingerprint string +} + +// certValidity is how long a generated agent certificate stays valid. +// Generous because these certs are pinned by fingerprint, not chained to a +// CA: a peer that pins the fingerprint trusts it regardless of expiry, and +// regenerating would change the fingerprint and force every peer to re-pin. +const certValidity = 10 * 365 * 24 * time.Hour + +// GenerateSelfSignedCert mints an ECDSA P-256 self-signed certificate for a +// squirrel agent. nodeName becomes the subject common name and a DNS SAN; +// localhost and the loopback addresses are added so a locally-dialed agent +// still validates for clients that do check names. The peer-sync client +// pins the fingerprint and skips name checks, but nothing should depend on +// that. The returned Fingerprint is the sha256: pin over the certificate's +// DER bytes — the exact value the peer-sync verifier compares. +func GenerateSelfSignedCert(nodeName string) (SelfSignedCert, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return SelfSignedCert{}, fmt.Errorf("generate key: %w", err) + } + tmpl, err := certTemplate(nodeName) + if err != nil { + return SelfSignedCert{}, err + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return SelfSignedCert{}, fmt.Errorf("create certificate: %w", err) + } + keyPEM, err := encodeKeyPEM(key) + if err != nil { + return SelfSignedCert{}, err + } + return SelfSignedCert{ + CertPEM: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + KeyPEM: keyPEM, + Fingerprint: fingerprintDER(der), + }, nil +} + +// certTemplate builds the x509 template for a self-signed agent cert. +func certTemplate(nodeName string) (*x509.Certificate, error) { + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, fmt.Errorf("generate serial: %w", err) + } + cn := nodeName + if cn == "" { + cn = "squirrel-agent" + } + dns := []string{"localhost"} + if nodeName != "" { + dns = append([]string{nodeName}, dns...) + } + now := time.Now() + return &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: cn}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(certValidity), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: dns, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + }, nil +} + +func encodeKeyPEM(key *ecdsa.PrivateKey) ([]byte, error) { + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, fmt.Errorf("marshal key: %w", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), nil +} + +// fingerprintDER renders the sha256: pin over a certificate's DER bytes, +// matching the peer-sync client's VerifyConnection comparison. +func fingerprintDER(der []byte) string { + sum := sha256.Sum256(der) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// FingerprintCertFile reads a PEM certificate file and returns its sha256: +// pin — the value the agent logs at startup and peers put in +// [nodes.X.tls] cert_fingerprint. The first CERTIFICATE block (the leaf) +// wins; trailing chain blocks are ignored. +func FingerprintCertFile(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read certificate %s: %w", path, err) + } + for { + block, rest := pem.Decode(data) + if block == nil { + return "", fmt.Errorf("no CERTIFICATE block in %s", path) + } + if block.Type == "CERTIFICATE" { + return fingerprintDER(block.Bytes), nil + } + data = rest + } +} diff --git a/agent/cert_test.go b/agent/cert_test.go new file mode 100644 index 0000000..f734e86 --- /dev/null +++ b/agent/cert_test.go @@ -0,0 +1,111 @@ +package agent + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "os" + "path/filepath" + "regexp" + "testing" +) + +// fingerprintShape mirrors config.fingerprintRE so the generated pin is +// guaranteed pasteable into [nodes.X.tls] cert_fingerprint. +var fingerprintShape = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`) + +func TestGenerateSelfSignedCert(t *testing.T) { + c, err := GenerateSelfSignedCert("nas") + if err != nil { + t.Fatalf("GenerateSelfSignedCert: %v", err) + } + if !fingerprintShape.MatchString(c.Fingerprint) { + t.Fatalf("fingerprint %q does not match sha256:<64-hex>", c.Fingerprint) + } + // cert + key must form a usable TLS pair. + if _, err := tls.X509KeyPair(c.CertPEM, c.KeyPEM); err != nil { + t.Fatalf("cert/key do not form a valid pair: %v", err) + } + // The fingerprint must equal the SHA-256 over the leaf's DER — exactly + // what the peer-sync client compares in VerifyConnection. + block, _ := pem.Decode(c.CertPEM) + if block == nil { + t.Fatal("no PEM block in CertPEM") + } + leaf, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("parse certificate: %v", err) + } + sum := sha256.Sum256(leaf.Raw) + if want := "sha256:" + hex.EncodeToString(sum[:]); want != c.Fingerprint { + t.Fatalf("fingerprint = %q, want %q", c.Fingerprint, want) + } + if leaf.Subject.CommonName != "nas" { + t.Fatalf("CommonName = %q, want nas", leaf.Subject.CommonName) + } +} + +func TestGenerateSelfSignedCertUnique(t *testing.T) { + a, err := GenerateSelfSignedCert("nas") + if err != nil { + t.Fatalf("GenerateSelfSignedCert: %v", err) + } + b, err := GenerateSelfSignedCert("nas") + if err != nil { + t.Fatalf("GenerateSelfSignedCert: %v", err) + } + if a.Fingerprint == b.Fingerprint { + t.Fatal("two generations produced the same fingerprint") + } +} + +func TestFingerprintCertFile(t *testing.T) { + c, err := GenerateSelfSignedCert("laptop") + if err != nil { + t.Fatalf("GenerateSelfSignedCert: %v", err) + } + path := filepath.Join(t.TempDir(), "agent.crt") + if err := os.WriteFile(path, c.CertPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + got, err := FingerprintCertFile(path) + if err != nil { + t.Fatalf("FingerprintCertFile: %v", err) + } + if got != c.Fingerprint { + t.Fatalf("FingerprintCertFile = %q, want %q", got, c.Fingerprint) + } +} + +func TestFingerprintCertFileMissing(t *testing.T) { + if _, err := FingerprintCertFile(filepath.Join(t.TempDir(), "nope.crt")); err == nil { + t.Fatal("expected error for missing cert file") + } +} + +// TestServerCertFingerprint covers the accessor logAgentStartup uses to log +// the pin at startup: it returns the configured cert's fingerprint and +// errors for a plain-HTTP agent. +func TestServerCertFingerprint(t *testing.T) { + c, err := GenerateSelfSignedCert("nas") + if err != nil { + t.Fatalf("GenerateSelfSignedCert: %v", err) + } + path := filepath.Join(t.TempDir(), "agent.crt") + if err := os.WriteFile(path, c.CertPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + tlsSrv := &Server{cfg: Config{TLSCert: path}} + got, err := tlsSrv.CertFingerprint() + if err != nil { + t.Fatalf("CertFingerprint: %v", err) + } + if got != c.Fingerprint { + t.Fatalf("CertFingerprint = %q, want %q", got, c.Fingerprint) + } + if _, err := (&Server{cfg: Config{}}).CertFingerprint(); err == nil { + t.Fatal("expected error for a plain-HTTP agent with no cert") + } +} diff --git a/agent/serve.go b/agent/serve.go index 8d13d34..544336c 100644 --- a/agent/serve.go +++ b/agent/serve.go @@ -93,6 +93,26 @@ func (s *Server) Serve(ctx context.Context, ln net.Listener) error { } } +// RunSchedulers runs the background scan + cadence loops without binding +// an HTTP listener — the listener-less agent mode (F35) for cadence-only +// machines that never receive peer syncs. It blocks until ctx is +// cancelled, then waits for an in-flight loop tick to finish its volume, +// mirroring Serve's shutdown discipline minus the HTTP server. The two +// loops are gated exactly as under Serve (scan only when ScanInterval is +// set; scheduler only when a volume declares a cadence), so an agent with +// nothing scheduled runs no goroutines and simply waits for cancellation. +func (s *Server) RunSchedulers(ctx context.Context) error { + loopCtx, cancelLoops := context.WithCancel(ctx) + defer cancelLoops() + var loopWG sync.WaitGroup + s.startScanLoop(loopCtx, &loopWG, s.scanLogger()) + s.startSchedulerLoop(loopCtx, &loopWG) + <-ctx.Done() + cancelLoops() + loopWG.Wait() + return nil +} + // startScanLoop spins up the drift-detection scheduler in a sibling // goroutine, but only when ScanInterval is set. The WaitGroup lets // Serve block on the loop's clean exit during shutdown so a tick diff --git a/cmd/squirrel/agent.go b/cmd/squirrel/agent.go index ca1de3c..02f4d8a 100644 --- a/cmd/squirrel/agent.go +++ b/cmd/squirrel/agent.go @@ -19,9 +19,10 @@ import ( // newAgentCmd returns the `squirrel agent` cobra command. It starts the // HTTP server declared by the `[agent]` config block and blocks until -// the cobra context (wired to SIGINT/SIGTERM in main) is cancelled. +// the cobra context (wired to SIGINT/SIGTERM in main) is cancelled. The +// `cert` child is a one-shot bootstrap helper and does not start a server. func newAgentCmd() *cobra.Command { - return &cobra.Command{ + cmd := &cobra.Command{ Use: "agent", Short: "Run the squirrel agent (HTTP server + scheduled audits + cadence-driven index/sync)", Args: cobra.NoArgs, @@ -29,6 +30,8 @@ func newAgentCmd() *cobra.Command { return runAgent(cmd) }, } + cmd.AddCommand(newAgentCertCmd()) + return cmd } func runAgent(cmd *cobra.Command) error { @@ -68,6 +71,24 @@ func runAgent(cmd *cobra.Command) error { if err != nil { return err } + return serveAgent(cmd, cfg, srv, logger) +} + +// serveAgent dispatches the built server to its run path: the listener-less +// scheduler-only run (F35) when [agent] listen is empty, or the HTTP server +// otherwise. Split from runAgent so the setup phase stays compact. +func serveAgent(cmd *cobra.Command, cfg *config.Config, srv *agent.Server, logger *slog.Logger) error { + // Listener-less mode (F35): no `listen`, so run the schedulers without an + // HTTP server. Refuse an agent that would do nothing at all — a + // scheduler-only agent with no cadences and no scan is silent + // degradation, not a valid config. + if cfg.Agent.Listen == "" { + if !agentHasWork(cfg) { + return fmt.Errorf("listener-less agent has nothing to run: set [agent] listen to receive peer syncs, or configure a cadence (a volume's sync_every, index_every, or hook.interval, or [agent] scan_interval) in %s", cfg.Path) + } + logSchedulerOnlyStartup(logger) + return srv.RunSchedulers(cmd.Context()) + } // Bind first so a port-in-use (or any other listen failure) surfaces // as a CLI error and never logs a misleading "agent listening" line. // We also log the listener's resolved Addr so `:0` (and other @@ -81,6 +102,33 @@ func runAgent(cmd *cobra.Command) error { return srv.Serve(cmd.Context(), ln) } +// agentHasWork reports whether a listener-less agent has any background +// work: a drift-scan interval, or at least one volume with a scheduler +// cadence (sync, standalone index, or interval hook). It mirrors the +// agent scheduler's own "is anything scheduled" gate so the CLI refuses a +// do-nothing agent rather than letting it idle silently. +func agentHasWork(cfg *config.Config) bool { + if cfg.Agent.ScanInterval > 0 { + return true + } + for _, v := range cfg.Volumes { + if v.SyncEvery > 0 || v.IndexEvery > 0 { + return true + } + if v.Hook != nil && v.Hook.Interval > 0 { + return true + } + } + return false +} + +// logSchedulerOnlyStartup emits the listener-less counterpart of the +// "agent listening" banner so a journal shows the agent came up in +// scheduler-only mode with no bound port. +func logSchedulerOnlyStartup(logger *slog.Logger) { + logger.Info("agent scheduler running", "listener", "disabled", "version", version) +} + // openAgentStore extends the standard resolveDBPath precedence with the // agent-specific override: --db > cfg.Agent.DB > cfg.DB > default. We // can't reuse openStore directly because it doesn't know about the @@ -222,9 +270,21 @@ func logAgentStartup(logger *slog.Logger, srv *agent.Server, addr string) { if srv.HasTLS() { scheme = "https" } - logger.Info("agent listening", + attrs := []any{ "addr", addr, "scheme", scheme, "version", version, - ) + } + // Surfacing the fingerprint at startup (F1) lets an operator read the + // pin peers must put in [nodes.X.tls] straight from the agent's log, + // without a separate command. A read failure is non-fatal — the agent + // still serves — so it is logged as a warning attribute, not an error. + if srv.HasTLS() { + if fp, err := srv.CertFingerprint(); err == nil { + attrs = append(attrs, "cert_fingerprint", fp) + } else { + attrs = append(attrs, "cert_fingerprint_error", err.Error()) + } + } + logger.Info("agent listening", attrs...) } diff --git a/cmd/squirrel/agent_cert.go b/cmd/squirrel/agent_cert.go new file mode 100644 index 0000000..164224e --- /dev/null +++ b/cmd/squirrel/agent_cert.go @@ -0,0 +1,115 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/agent" + "github.com/mbertschler/squirrel/config" +) + +// newAgentCertCmd returns `squirrel agent cert`: a bootstrap helper that +// generates the agent's self-signed TLS certificate + key at the paths the +// [agent.tls] block configures and prints the sha256: pin peers put in +// their [nodes.X.tls] cert_fingerprint (F1). It is a deliberate change (it +// writes key material), not introspection, and refuses to clobber an +// existing certificate without --force: regenerating changes the +// fingerprint and breaks every peer that already pinned it. +func newAgentCertCmd() *cobra.Command { + var force bool + cmd := &cobra.Command{ + Use: "cert", + Short: "Generate the agent's self-signed TLS cert+key and print its sha256: pin", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return runAgentCert(cmd, force) + }, + } + cmd.Flags().BoolVar(&force, "force", false, "overwrite an existing cert/key (changes the fingerprint — every peer must re-pin)") + return cmd +} + +func runAgentCert(cmd *cobra.Command, force bool) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + certPath, keyPath, err := agentCertPaths(cfg) + if err != nil { + return err + } + if err := guardExistingCert(certPath, keyPath, force); err != nil { + return err + } + generated, err := agent.GenerateSelfSignedCert(cfg.NodeName) + if err != nil { + return err + } + if err := writeCertFiles(certPath, keyPath, generated); err != nil { + return err + } + printCertResult(cmd, cfg, certPath, keyPath, generated.Fingerprint) + return nil +} + +// agentCertPaths pulls the configured cert/key paths, erroring with a +// pointer at the config when the [agent] block or its tls paths are absent — +// there is nowhere to write otherwise. +func agentCertPaths(cfg *config.Config) (certPath, keyPath string, err error) { + if cfg.Agent == nil { + return "", "", fmt.Errorf("no [agent] block in %s", cfg.Path) + } + if cfg.Agent.TLSCert == "" || cfg.Agent.TLSKey == "" { + return "", "", fmt.Errorf("no [agent.tls] cert/key configured in %s — set both paths before generating", cfg.Path) + } + return cfg.Agent.TLSCert, cfg.Agent.TLSKey, nil +} + +// guardExistingCert refuses to overwrite either file unless force is set. +func guardExistingCert(certPath, keyPath string, force bool) error { + if force { + return nil + } + for _, p := range []string{certPath, keyPath} { + _, err := os.Stat(p) + switch { + case err == nil: + return fmt.Errorf("%s already exists — refusing to overwrite (pass --force to regenerate; every peer that pinned the old fingerprint must re-pin)", p) + case !errors.Is(err, os.ErrNotExist): + return fmt.Errorf("stat %s: %w", p, err) + } + } + return nil +} + +// writeCertFiles writes both files owner-only (0600) under 0700 parent +// directories. The certificate is not secret, but keeping it 0600 alongside +// the key is harmless — peers obtain the fingerprint from the pairing helper +// and the startup log, never by reading this file. +func writeCertFiles(certPath, keyPath string, c agent.SelfSignedCert) error { + for path, data := range map[string][]byte{certPath: c.CertPEM, keyPath: c.KeyPEM} { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("create directory for %s: %w", path, err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + } + return nil +} + +func printCertResult(cmd *cobra.Command, cfg *config.Config, certPath, keyPath, fingerprint string) { + out := cmd.OutOrStdout() + fmt.Fprintf(out, "wrote certificate %s\n", certPath) + fmt.Fprintf(out, "wrote private key %s\n\n", keyPath) + name := cfg.NodeName + if name == "" { + name = "<this-node>" + } + fmt.Fprintf(out, "Peers pin this node (%s) by adding to their config:\n\n", name) + fmt.Fprintf(out, "[nodes.%s.tls]\ncert_fingerprint = %q\n", name, fingerprint) +} diff --git a/cmd/squirrel/agent_cert_test.go b/cmd/squirrel/agent_cert_test.go new file mode 100644 index 0000000..d13bbf2 --- /dev/null +++ b/cmd/squirrel/agent_cert_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mbertschler/squirrel/agent" +) + +// writeAgentCertConfig writes a config with an [agent] block whose tls +// cert/key point at paths inside a fresh temp dir, and returns the config +// path plus the two target paths. +func writeAgentCertConfig(t *testing.T) (cfgPath, certPath, keyPath string) { + t.Helper() + dir := t.TempDir() + certPath = filepath.Join(dir, "agent.crt") + keyPath = filepath.Join(dir, "agent.key") + cfgPath = filepath.Join(dir, "config.toml") + body := fmt.Sprintf(""+ + "node_name = \"nas\"\n\n"+ + "[agent]\nlisten = \"127.0.0.1:8443\"\nauth = { token = \"tok\" }\n\n"+ + "[agent.tls]\ncert = %q\nkey = %q\n", certPath, keyPath) + if err := os.WriteFile(cfgPath, []byte(body), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return cfgPath, certPath, keyPath +} + +func TestCLIAgentCertGenerates(t *testing.T) { + cfgPath, certPath, keyPath := writeAgentCertConfig(t) + + out := runCLI(t, "--config", cfgPath, "agent", "cert") + for _, p := range []string{certPath, keyPath} { + if _, err := os.Stat(p); err != nil { + t.Fatalf("expected %s to be written: %v", p, err) + } + } + want, err := agent.FingerprintCertFile(certPath) + if err != nil { + t.Fatalf("FingerprintCertFile: %v", err) + } + if !strings.Contains(out, want) { + t.Fatalf("output missing generated fingerprint %q:\n%s", want, out) + } + if !strings.Contains(out, "[nodes.nas.tls]") || !strings.Contains(out, "cert_fingerprint") { + t.Fatalf("output missing pasteable pin snippet:\n%s", out) + } +} + +func TestCLIAgentCertRefusesOverwrite(t *testing.T) { + cfgPath, _, _ := writeAgentCertConfig(t) + runCLI(t, "--config", cfgPath, "agent", "cert") + + out, err := runCLIExpectErr(t, "--config", cfgPath, "agent", "cert") + if !strings.Contains(err.Error(), "already exists") { + t.Fatalf("expected refuse-overwrite error, got %v\n%s", err, out) + } +} + +func TestCLIAgentCertForceRegenerates(t *testing.T) { + cfgPath, certPath, _ := writeAgentCertConfig(t) + runCLI(t, "--config", cfgPath, "agent", "cert") + first, err := agent.FingerprintCertFile(certPath) + if err != nil { + t.Fatalf("fingerprint: %v", err) + } + runCLI(t, "--config", cfgPath, "agent", "cert", "--force") + second, err := agent.FingerprintCertFile(certPath) + if err != nil { + t.Fatalf("fingerprint: %v", err) + } + if first == second { + t.Fatal("--force did not regenerate the certificate (fingerprint unchanged)") + } +} + +func TestCLIAgentCertNoTLSConfigured(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.toml") + body := "[agent]\nlisten = \"127.0.0.1:8443\"\nauth = { token = \"tok\" }\n" + if err := os.WriteFile(cfgPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + _, err := runCLIExpectErr(t, "--config", cfgPath, "agent", "cert") + if !strings.Contains(err.Error(), "no [agent.tls] cert/key configured") { + t.Fatalf("expected no-tls error, got %v", err) + } +} diff --git a/cmd/squirrel/agent_listenerless_test.go b/cmd/squirrel/agent_listenerless_test.go new file mode 100644 index 0000000..8629c1e --- /dev/null +++ b/cmd/squirrel/agent_listenerless_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// TestCLIAgentListenerLessRunsScheduler covers F35 end-to-end: an [agent] +// block with no `listen` starts the scheduler without binding an HTTP +// listener, logs the scheduler-only banner, and shuts down cleanly on +// context cancel. The volume declares an index cadence (no sync_to), so no +// rclone binary is needed to bring the agent up. +func TestCLIAgentListenerLessRunsScheduler(t *testing.T) { + dir := t.TempDir() + vol := filepath.Join(dir, "photos") + if err := os.MkdirAll(vol, 0o755); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(vol, "a.jpg"), "x") + dbPath := filepath.Join(dir, "index.db") + cfgPath := filepath.Join(dir, "config.toml") + body := fmt.Sprintf("db = %q\n\n[agent]\n\n[volumes.photos]\npath = %q\nindex_every = \"1h\"\n", dbPath, vol) + if err := os.WriteFile(cfgPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + isolateConfig(t) + ctx, cancel := context.WithCancel(context.Background()) + buf := &syncBuf{} + root := newRootCmd() + root.SetOut(buf) + root.SetErr(buf) + root.SetArgs([]string{"--config", cfgPath, "agent"}) + done := make(chan error, 1) + go func() { done <- root.ExecuteContext(ctx) }() + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) && !strings.Contains(buf.String(), "agent scheduler running") { + time.Sleep(20 * time.Millisecond) + } + if !strings.Contains(buf.String(), "agent scheduler running") { + cancel() + t.Fatalf("scheduler-only banner never appeared:\n%s", buf.String()) + } + if strings.Contains(buf.String(), "agent listening") { + cancel() + t.Fatalf("listener-less agent unexpectedly bound a listener:\n%s", buf.String()) + } + + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("agent exited with error: %v\noutput:\n%s", err, buf.String()) + } + case <-time.After(5 * time.Second): + t.Fatalf("agent did not shut down within timeout\noutput:\n%s", buf.String()) + } +} + +// TestCLIAgentListenerLessNoWork guards the fail-fast: a listener-less +// agent with no cadence and no scan has nothing to run and must refuse to +// start rather than idle silently. +func TestCLIAgentListenerLessNoWork(t *testing.T) { + dir := t.TempDir() + vol := filepath.Join(dir, "photos") + if err := os.MkdirAll(vol, 0o755); err != nil { + t.Fatal(err) + } + dbPath := filepath.Join(dir, "index.db") + cfgPath := filepath.Join(dir, "config.toml") + body := fmt.Sprintf("db = %q\n\n[agent]\n\n[volumes.photos]\npath = %q\n", dbPath, vol) + if err := os.WriteFile(cfgPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + _, err := runCLIExpectErr(t, "--config", cfgPath, "agent") + if !strings.Contains(err.Error(), "nothing to run") { + t.Fatalf("expected nothing-to-run error, got %v", err) + } +} diff --git a/cmd/squirrel/config.go b/cmd/squirrel/config.go new file mode 100644 index 0000000..233b797 --- /dev/null +++ b/cmd/squirrel/config.go @@ -0,0 +1,15 @@ +package main + +import "github.com/spf13/cobra" + +// newConfigCmd returns the `squirrel config` parent command. Its children +// are introspection over the config file itself (currently `check`), kept +// distinct from the DB-facing question commands. +func newConfigCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "config", + Short: "Inspect the squirrel configuration", + } + cmd.AddCommand(newConfigCheckCmd()) + return cmd +} diff --git a/cmd/squirrel/config_check.go b/cmd/squirrel/config_check.go new file mode 100644 index 0000000..029a6be --- /dev/null +++ b/cmd/squirrel/config_check.go @@ -0,0 +1,285 @@ +package main + +import ( + "errors" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/config" +) + +// Per-check status markers. statusFail is fatal (the command exits +// non-zero); statusWarn is advisory (surfaced, but the config is usable); +// statusOK is a clean line. +const ( + statusOK = "ok" + statusWarn = "warn" + statusFail = "FAIL" +) + +// newConfigCheckCmd returns `squirrel config check`: parse + resolve the +// whole config (env vars included), stat volume paths and node byte-paths, +// validate offload policies against target capabilities, and print an +// affirmative summary (F4). It is strictly read-only — it never creates a +// path, cert, or database — per the "CLI is for change and for questions" +// principle: this is a question. +func newConfigCheckCmd() *cobra.Command { + return &cobra.Command{ + Use: "check", + Short: "Parse and resolve the config, stat its paths, and report what it declares", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return runConfigCheck(cmd) + }, + } +} + +// checkTally accumulates fatal/advisory counts across the report sections. +type checkTally struct { + fails int + warns int +} + +func (t *checkTally) add(status string) { + switch status { + case statusFail: + t.fails++ + case statusWarn: + t.warns++ + } +} + +func runConfigCheck(cmd *cobra.Command) error { + // requireConfig both loads and resolves: a parse error, an unknown + // field, or an unset { env = "VAR" } secret surfaces here as the + // check's failure, which is exactly the "did everything resolve?" + // question this command answers. + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "config: %s\n\n", cfg.Path) + + var tally checkTally + checkVolumes(cfg, out, &tally) + checkDestinations(cfg, out) + checkNodes(cfg, out, &tally) + checkOffloadPolicies(cfg, out, &tally) + + return printSummary(out, cfg, tally) +} + +// printCheckLine renders one aligned status line: " ok name detail". +func printCheckLine(out io.Writer, status, name, detail string) { + if detail == "" { + fmt.Fprintf(out, " %-4s %s\n", status, name) + return + } + fmt.Fprintf(out, " %-4s %s %s\n", status, name, detail) +} + +func checkVolumes(cfg *config.Config, out io.Writer, tally *checkTally) { + fmt.Fprintf(out, "volumes (%d)\n", len(cfg.Volumes)) + for _, name := range sortedKeys(cfg.Volumes) { + vol := cfg.Volumes[name] + status, detail := statVolumePath(vol.Path) + tally.add(status) + printCheckLine(out, status, name, joinDetail(vol.Path, detail)) + } +} + +func checkDestinations(cfg *config.Config, out io.Writer) { + fmt.Fprintf(out, "destinations (%d)\n", len(cfg.Destinations)) + for _, name := range sortedKeys(cfg.Destinations) { + d := cfg.Destinations[name] + parts := []string{d.Type, d.Layout} + if d.Crypt != nil { + parts = append(parts, "crypt") + } + // Destinations resolved (Load succeeded), so they are all ok here; + // their remote reachability is a sync-time concern, not something a + // read-only check probes. + printCheckLine(out, statusOK, name, strings.Join(parts, " ")) + } +} + +func checkNodes(cfg *config.Config, out io.Writer, tally *checkTally) { + fmt.Fprintf(out, "nodes (%d)\n", len(cfg.Nodes)) + for _, name := range sortedKeys(cfg.Nodes) { + n := cfg.Nodes[name] + status, detail := statNodeBytePath(n.Path) + tally.add(status) + endpoint := n.Endpoint.String() + printCheckLine(out, status, name, joinDetail(endpoint, "byte-path "+n.Path+detail)) + } +} + +func checkOffloadPolicies(cfg *config.Config, out io.Writer, tally *checkTally) { + names := offloadVolumeNames(cfg) + if len(names) == 0 { + return + } + fmt.Fprintf(out, "offload policies (%d)\n", len(names)) + for _, name := range names { + for _, target := range cfg.Volumes[name].OffloadRequires { + status, detail := offloadTargetStatus(cfg, target) + tally.add(status) + printCheckLine(out, status, name+" → "+target, detail) + } + } +} + +// offloadTargetStatus classifies one offload_requires target. A destination +// that can never yield offload-grade evidence (a mirror-layout crypt +// destination, F21) is a fatal misconfiguration — the gate would wait +// forever. A peer node yields peer-blake3 evidence directly. A name that is +// neither is an external target whose evidence must arrive relayed via a +// peer; that is legitimate, so it is reported without alarm. +func offloadTargetStatus(cfg *config.Config, target string) (status, detail string) { + if d, ok := cfg.Destinations[target]; ok { + if capable, reason := d.CanEverGateOffload(); !capable { + return statusFail, "never yields offload evidence: " + reason + } + return statusOK, "destination" + } + if _, ok := cfg.Nodes[target]; ok { + return statusOK, "peer node (peer-blake3 evidence)" + } + return statusOK, "external target — evidence must arrive relayed via a peer" +} + +func printSummary(out io.Writer, cfg *config.Config, tally checkTally) error { + fmt.Fprintln(out) + counts := fmt.Sprintf("%d volumes, %d destinations, %d nodes", + len(cfg.Volumes), len(cfg.Destinations), len(cfg.Nodes)) + switch { + case tally.fails > 0: + fmt.Fprintf(out, "%s — %s, %s\n", counts, + plural(tally.fails, "problem"), plural(tally.warns, "warning")) + return fmt.Errorf("config check found %d problem(s)", tally.fails) + case tally.warns > 0: + fmt.Fprintf(out, "%s — resolvable, %s\n", counts, plural(tally.warns, "warning")) + return nil + default: + fmt.Fprintf(out, "%s — all resolvable\n", counts) + return nil + } +} + +// statVolumePath stats a local volume root. A missing path or non-directory +// is fatal; an empty directory is advisory (F8: a typo'd or unmounted path +// looks identical to a genuinely new volume, so it must be flagged, not +// hard-failed). +func statVolumePath(path string) (status, detail string) { + info, err := os.Stat(path) + switch { + case errors.Is(err, os.ErrNotExist): + return statusFail, "does not exist" + case err != nil: + return statusFail, err.Error() + case !info.IsDir(): + return statusFail, "not a directory" + } + empty, err := dirIsEmpty(path) + if err != nil { + return statusWarn, "unreadable: " + err.Error() + } + if empty { + return statusWarn, "empty directory — new volume or wrong mount?" + } + return statusOK, "" +} + +// statNodeBytePath stats a node's rclone byte-path (F34). An rclone remote +// spec ("remote:path") is not a local path, so it is reported as unchecked +// rather than statted. A missing local mount is advisory: the share may +// legitimately be down when the check runs, but a silent absence is exactly +// what turns into undiagnosable transfer errors, so it is surfaced. The +// returned detail is a suffix appended after the path. +func statNodeBytePath(path string) (status, detail string) { + if isRcloneRemoteSpec(path) { + return statusOK, " (rclone remote spec — not checked)" + } + info, err := os.Stat(path) + switch { + case errors.Is(err, os.ErrNotExist): + return statusWarn, " (does not exist — mount not up?)" + case err != nil: + return statusWarn, " (" + err.Error() + ")" + case !info.IsDir(): + return statusWarn, " (not a directory)" + } + return statusOK, "" +} + +// isRcloneRemoteSpec reports whether p is an rclone "remote:path" reference +// rather than a filesystem path. An absolute path (leading /) is always a +// filesystem path; otherwise a leading "name:" segment marks a remote. +func isRcloneRemoteSpec(p string) bool { + if strings.HasPrefix(p, "/") { + return false + } + i := strings.IndexByte(p, ':') + return i > 0 +} + +// dirIsEmpty reports whether dir contains no entries, reading just one name +// rather than the whole listing. +func dirIsEmpty(dir string) (bool, error) { + f, err := os.Open(dir) + if err != nil { + return false, err + } + defer func() { _ = f.Close() }() + _, err = f.Readdirnames(1) + if errors.Is(err, io.EOF) { + return true, nil + } + return false, err +} + +// offloadVolumeNames returns the sorted names of volumes that declare an +// offload policy. +func offloadVolumeNames(cfg *config.Config) []string { + var names []string + for name, vol := range cfg.Volumes { + if len(vol.OffloadRequires) > 0 { + names = append(names, name) + } + } + sort.Strings(names) + return names +} + +// joinDetail appends a parenthetical/extra detail to a base string when the +// detail is non-empty. +func joinDetail(base, detail string) string { + if detail == "" { + return base + } + return base + " " + detail +} + +// plural renders "N thing" / "N things" for a count. +func plural(n int, thing string) string { + if n == 1 { + return fmt.Sprintf("%d %s", n, thing) + } + return fmt.Sprintf("%d %ss", n, thing) +} + +// sortedKeys returns the map keys sorted, for stable report ordering. +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/cmd/squirrel/config_check_test.go b/cmd/squirrel/config_check_test.go new file mode 100644 index 0000000..d8c9e37 --- /dev/null +++ b/cmd/squirrel/config_check_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// writeCheckConfig writes body to a config.toml in a fresh temp dir and +// returns its path. +func writeCheckConfig(t *testing.T, body string) string { + t.Helper() + configPath := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(configPath, []byte(body), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return configPath +} + +func TestCLIConfigCheckAffirmative(t *testing.T) { + dir := t.TempDir() + photos := filepath.Join(dir, "photos") + dest := filepath.Join(dir, "dest") + nodePath := filepath.Join(dir, "peer-mount") + for _, d := range []string{photos, dest, nodePath} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + writeTestFile(t, filepath.Join(photos, "a.jpg"), "x") + + body := "" + + "node_name = \"thisnode\"\n\n" + + "[volumes.photos]\npath = \"" + photos + "\"\nsync_to = [\"scratch\"]\n\n" + + "[destinations.scratch]\ntype = \"local\"\nroot = \"" + dest + "\"\n\n" + + "[nodes.peer]\nendpoint = \"https://peer.home:8443\"\npath = \"" + nodePath + "\"\n" + + "[nodes.peer.auth]\nbearer = \"tok\"\n" + cfgPath := writeCheckConfig(t, body) + + out := runCLI(t, "--config", cfgPath, "config", "check") + if !strings.Contains(out, "1 volumes, 1 destinations, 1 nodes — all resolvable") { + t.Fatalf("missing affirmative summary:\n%s", out) + } +} + +func TestCLIConfigCheckEmptyVolumeWarns(t *testing.T) { + dir := t.TempDir() + photos := filepath.Join(dir, "photos") + if err := os.MkdirAll(photos, 0o755); err != nil { + t.Fatal(err) + } + body := "[volumes.photos]\npath = \"" + photos + "\"\n" + cfgPath := writeCheckConfig(t, body) + + out := runCLI(t, "--config", cfgPath, "config", "check") + if !strings.Contains(out, "warn") || !strings.Contains(out, "empty") { + t.Fatalf("expected empty-volume warning:\n%s", out) + } + if !strings.Contains(out, "resolvable, 1 warning") { + t.Fatalf("expected warning summary:\n%s", out) + } +} + +func TestCLIConfigCheckMissingVolumeFails(t *testing.T) { + dir := t.TempDir() + missing := filepath.Join(dir, "not-there") + body := "[volumes.photos]\npath = \"" + missing + "\"\n" + cfgPath := writeCheckConfig(t, body) + + out, err := runCLIExpectErr(t, "--config", cfgPath, "config", "check") + if !strings.Contains(out, "does not exist") { + t.Fatalf("expected does-not-exist line:\n%s", out) + } + if !strings.Contains(err.Error(), "problem") { + t.Fatalf("expected problem error, got %v", err) + } +} + +// TestCLIConfigCheckOffloadMirrorCrypt exercises the offload_requires +// satisfiability check (F21): a mirror-layout crypt destination can never +// yield offload evidence, so naming it is a fatal misconfiguration. +// +// The rejection now lands at config load (#155's +// rejectUnsatisfiableOffloadRequires), ahead of `config check`'s own +// per-target classification, so what a user sees from `config check` is the +// load error. Either way the command must name the target, explain why it can +// never gate, and exit non-zero — that contract is what this pins. +func TestCLIConfigCheckOffloadMirrorCrypt(t *testing.T) { + dir := t.TempDir() + docs := filepath.Join(dir, "docs") + if err := os.MkdirAll(docs, 0o755); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(docs, "d.txt"), "x") + body := "" + + "[volumes.docs]\npath = \"" + docs + "\"\noffload_requires = [\"cloudbox\"]\n\n" + + "[destinations.cloudbox]\ntype = \"sftp\"\nhost = \"h\"\nuser = \"u\"\nroot = \"/data\"\n" + + "[destinations.cloudbox.crypt]\npassword = \"pw\"\n" + cfgPath := writeCheckConfig(t, body) + + out, err := runCLIExpectErr(t, "--config", cfgPath, "config", "check") + if !strings.Contains(out, "cloudbox") || !strings.Contains(out, "can never satisfy the durability gate") { + t.Fatalf("expected an unsatisfiable-offload problem naming cloudbox:\n%s", out) + } + if err == nil { + t.Fatal("expected a non-zero exit for an unsatisfiable offload_requires") + } +} + +// TestCLIConfigCheckReadOnly guards principle 2: `config check` must not +// mutate anything. The config declares a db path that must never be +// created by a check. +func TestCLIConfigCheckReadOnly(t *testing.T) { + dir := t.TempDir() + photos := filepath.Join(dir, "photos") + if err := os.MkdirAll(photos, 0o755); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(photos, "a.jpg"), "x") + dbPath := filepath.Join(dir, "index.db") + body := "db = \"" + dbPath + "\"\n\n[volumes.photos]\npath = \"" + photos + "\"\n" + cfgPath := writeCheckConfig(t, body) + + runCLI(t, "--config", cfgPath, "config", "check") + if _, err := os.Stat(dbPath); !os.IsNotExist(err) { + t.Fatalf("config check created the database at %s (must be read-only)", dbPath) + } +} diff --git a/cmd/squirrel/destination.go b/cmd/squirrel/destination.go new file mode 100644 index 0000000..a9081b7 --- /dev/null +++ b/cmd/squirrel/destination.go @@ -0,0 +1,19 @@ +package main + +import "github.com/spf13/cobra" + +// newDestinationCmd returns the `squirrel destination` parent command +// grouping the destination-level recovery verbs. It has no behaviour of its +// own and prints help when invoked bare. +func newDestinationCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "destination", + Short: "Manage a destination's recorded state", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, + } + cmd.AddCommand(newDestinationResetCmd()) + return cmd +} diff --git a/cmd/squirrel/destination_reset.go b/cmd/squirrel/destination_reset.go new file mode 100644 index 0000000..88e649b --- /dev/null +++ b/cmd/squirrel/destination_reset.go @@ -0,0 +1,117 @@ +package main + +import ( + "fmt" + "io" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/store" +) + +// newDestinationResetCmd returns `squirrel destination reset <destination>`: +// the explicit operator verb that forgets a destination's recorded upload and +// durability state so the next sync treats it as fresh (friction log F20). +// It is a *change* command in the ux-principles sense — weighty and +// irreversible in effect — so it defaults to refusing, offers a --dry-run +// preview, and requires an explicit --yes to proceed. The runs table and the +// append-only durability advance log are preserved; the reset itself is +// recorded as an audit run. +func newDestinationResetCmd() *cobra.Command { + var ( + yes bool + dryRun bool + ) + cmd := &cobra.Command{ + Use: "reset <destination>", + Short: "Forget a destination's recorded upload and durability state (audit-preserving)", + Long: `Forget everything the index records about a destination's remote state: +its per-content and per-pack upload ledgers, its live durability vector, and +its push-freshness maxima. The next sync then treats the destination as fresh +and re-uploads. + +Use it to recover a wrecked or repointed destination — e.g. after wiping the +remote root, or pointing an existing destination name at a fresh root, where +the layout guard would otherwise refuse on name-keyed run history alone. + +This clears derived state only. The runs table and the append-only durability +advance log are untouched, so the audit trail survives; the reset is itself +recorded as an audit run. The remote bytes are not deleted — wipe or repoint +the remote root separately if you want the destination genuinely empty.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runDestinationReset(cmd, args[0], yes, dryRun) + }, + } + cmd.Flags().BoolVar(&yes, "yes", false, "confirm the reset; required to actually clear recorded state") + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "print what would be cleared without changing anything") + return cmd +} + +func runDestinationReset(cmd *cobra.Command, name string, yes, dryRun bool) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + if _, ok := cfg.Destinations[name]; !ok { + return fmt.Errorf("unknown destination %q (declare it in %s); destination reset clears a bucket destination's recorded state — a peer node's durability assertions are revoked separately", name, cfg.Path) + } + + s, err := openStore(cmd, cfg) + if err != nil { + return err + } + defer s.Close() + + counts, err := s.CountDestinationRecordedState(cmd.Context(), name) + if err != nil { + return err + } + out := cmd.OutOrStdout() + if counts.Empty() { + fmt.Fprintf(out, "destination %q has no recorded state — nothing to reset\n", name) + return nil + } + if dryRun { + printResetPreview(out, name, counts, true) + return nil + } + if !yes { + printResetPreview(out, name, counts, false) + return fmt.Errorf("refusing to clear recorded state without confirmation; re-run with --yes (or --dry-run to preview)") + } + + runID, cleared, err := s.ResetDestination(cmd.Context(), name) + if err != nil { + return err + } + printResetResult(out, name, cleared, runID) + return nil +} + +// printResetPreview renders the per-category counts that a reset would clear. +// dryRun toggles only the leading verb so the same table serves both the +// explicit --dry-run preview and the confirmation gate. +func printResetPreview(out io.Writer, name string, c store.DestinationResetCounts, dryRun bool) { + prefix := "reset would clear" + if !dryRun { + prefix = "this will clear" + } + fmt.Fprintf(out, "%s destination %q recorded state:\n", prefix, name) + printResetCounts(out, c) + fmt.Fprintln(out, " (runs history and the durability advance log are preserved; remote bytes are untouched)") +} + +// printResetResult renders the loud confirmation after a reset ran, naming +// the audit run that recorded it and what to do next. +func printResetResult(out io.Writer, name string, c store.DestinationResetCounts, runID int64) { + fmt.Fprintf(out, "reset destination %q (run %d):\n", name, runID) + printResetCounts(out, c) + fmt.Fprintf(out, "next: the next sync re-uploads to %q; for a content-addressed or packed layout, wipe or repoint the remote root so the layout guard sees a fresh start\n", name) +} + +func printResetCounts(out io.Writer, c store.DestinationResetCounts) { + fmt.Fprintf(out, " upload records: %d objects, %d packs\n", c.RemoteObjects, c.RemotePacks) + fmt.Fprintf(out, " durability vector: %d component(s)\n", c.VectorComponents) + fmt.Fprintf(out, " push freshness: %d row(s)\n", c.FreshnessRows) +} diff --git a/cmd/squirrel/destination_reset_test.go b/cmd/squirrel/destination_reset_test.go new file mode 100644 index 0000000..d8f9a5a --- /dev/null +++ b/cmd/squirrel/destination_reset_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/mbertschler/squirrel/store" +) + +// writeResetConfig writes a config with one local destination "arch" and a +// db path, returning the config path and the db path. +func writeResetConfig(t *testing.T) (configPath, dbPath string) { + t.Helper() + dir := t.TempDir() + dbPath = filepath.Join(dir, "index.db") + destDir := filepath.Join(dir, "arch") + body := fmt.Sprintf("db = %q\n\n[destinations.arch]\ntype = \"local\"\nroot = %q\n", dbPath, destDir) + configPath = writeCheckConfig(t, body) + return configPath, dbPath +} + +// seedResetVector opens the fixture DB directly and records one durability +// vector component for destination, so the reset has something to clear. +func seedResetVector(t *testing.T, dbPath, destination string) { + t.Helper() + ctx := context.Background() + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + defer s.Close() + v, err := s.GetOrCreateVolume(ctx, "/v") + if err != nil { + t.Fatalf("GetOrCreateVolume: %v", err) + } + self, err := s.GetSelfNode(ctx) + if err != nil { + t.Fatalf("GetSelfNode: %v", err) + } + if err := s.UpsertDestinationRunIDVerified(ctx, v.ID, destination, self.ID, 5, store.VerifyMethodBlake3, false); err != nil { + t.Fatalf("UpsertDestinationRunIDVerified: %v", err) + } +} + +// resetVectorCount reports how many vector components destination still has. +func resetVectorCount(t *testing.T, dbPath, destination string) int { + t.Helper() + ctx := context.Background() + s, err := store.Open(dbPath) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + defer s.Close() + v, err := s.GetOrCreateVolume(ctx, "/v") + if err != nil { + t.Fatalf("GetOrCreateVolume: %v", err) + } + vec, err := s.ListDestinationRunIDs(ctx, v.ID, destination) + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + return len(vec) +} + +// TestCLIDestinationResetUnknown: a destination not in config is refused +// before the store is touched. +func TestCLIDestinationResetUnknown(t *testing.T) { + cfgPath, _ := writeResetConfig(t) + out, err := runCLIExpectErr(t, "--config", cfgPath, "destination", "reset", "ghost", "--yes") + if !strings.Contains(out, "unknown destination") && !strings.Contains(err.Error(), "unknown destination") { + t.Fatalf("expected unknown-destination error, got out=%q err=%v", out, err) + } +} + +// TestCLIDestinationResetNothing: a configured destination with no recorded +// state reports "nothing to reset" affirmatively and mints no run. +func TestCLIDestinationResetNothing(t *testing.T) { + cfgPath, _ := writeResetConfig(t) + out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--yes") + if !strings.Contains(out, "nothing to reset") { + t.Fatalf("expected nothing-to-reset message:\n%s", out) + } +} + +// TestCLIDestinationResetDryRun: --dry-run previews the counts and clears +// nothing. +func TestCLIDestinationResetDryRun(t *testing.T) { + cfgPath, dbPath := writeResetConfig(t) + seedResetVector(t, dbPath, "arch") + + out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--dry-run") + if !strings.Contains(out, "would clear") { + t.Fatalf("expected dry-run preview:\n%s", out) + } + if n := resetVectorCount(t, dbPath, "arch"); n != 1 { + t.Fatalf("dry-run cleared state: vector count = %d, want 1", n) + } +} + +// TestCLIDestinationResetNeedsConfirmation: without --yes the command +// previews and refuses, changing nothing (principle 2: weighty change). +func TestCLIDestinationResetNeedsConfirmation(t *testing.T) { + cfgPath, dbPath := writeResetConfig(t) + seedResetVector(t, dbPath, "arch") + + out, err := runCLIExpectErr(t, "--config", cfgPath, "destination", "reset", "arch") + if !strings.Contains(err.Error(), "--yes") { + t.Fatalf("expected confirmation-required error, got %v", err) + } + if !strings.Contains(out, "this will clear") { + t.Fatalf("expected preview before refusal:\n%s", out) + } + if n := resetVectorCount(t, dbPath, "arch"); n != 1 { + t.Fatalf("refused reset still cleared state: vector count = %d, want 1", n) + } +} + +// TestCLIDestinationResetConfirmed: --yes clears the state and names the +// audit run that recorded it. +func TestCLIDestinationResetConfirmed(t *testing.T) { + cfgPath, dbPath := writeResetConfig(t) + seedResetVector(t, dbPath, "arch") + + out := runCLI(t, "--config", cfgPath, "destination", "reset", "arch", "--yes") + if !strings.Contains(out, "reset destination") || !strings.Contains(out, "run ") { + t.Fatalf("expected loud reset confirmation with run id:\n%s", out) + } + if n := resetVectorCount(t, dbPath, "arch"); n != 0 { + t.Fatalf("confirmed reset left state: vector count = %d, want 0", n) + } +} diff --git a/cmd/squirrel/node.go b/cmd/squirrel/node.go new file mode 100644 index 0000000..27091ae --- /dev/null +++ b/cmd/squirrel/node.go @@ -0,0 +1,14 @@ +package main + +import "github.com/spf13/cobra" + +// newNodeCmd returns the `squirrel node` parent command grouping the +// peer-relationship bootstrap helpers. +func newNodeCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "node", + Short: "Manage peer node relationships", + } + cmd.AddCommand(newNodePairCmd()) + return cmd +} diff --git a/cmd/squirrel/node_pair.go b/cmd/squirrel/node_pair.go new file mode 100644 index 0000000..886c880 --- /dev/null +++ b/cmd/squirrel/node_pair.go @@ -0,0 +1,167 @@ +package main + +import ( + "crypto/rand" + "encoding/base64" + "fmt" + "io" + "regexp" + + "github.com/spf13/cobra" + + "github.com/mbertschler/squirrel/agent" + "github.com/mbertschler/squirrel/config" +) + +// nodeNameRE mirrors config's volume/destination/node name rule so a bad +// peer name is rejected before it is templated into a TOML key. Kept local +// (config does not export its regexp) and deliberately identical. +var nodeNameRE = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]*$`) + +// nodePairOpts carries the peer-side facts this node cannot know on its own, +// supplied via flags; each empty value renders as a clearly-marked +// placeholder the operator fills in. +type nodePairOpts struct { + localEndpoint string + peerEndpoint string + peerPath string + peerFingerprint string +} + +// newNodePairCmd returns `squirrel node pair <peer>`: it generates the two +// bearer tokens a bidirectional peer relationship needs and emits the +// matching config halves for *both* machines, with each token already placed +// in the two slots that must agree. This kills the F3 token-matrix error +// class (four cross-referenced bindings written by hand). It prints only — +// it never edits either config — so the operator reviews and pastes. +func newNodePairCmd() *cobra.Command { + var opts nodePairOpts + cmd := &cobra.Command{ + Use: "pair <peer>", + Short: "Emit matching config halves (tokens, endpoints, fingerprints) for a peer relationship", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runNodePair(cmd, args[0], opts) + }, + } + cmd.Flags().StringVar(&opts.localEndpoint, "local-endpoint", "", "this node's agent endpoint as the peer dials it (e.g. https://nas.home:8443)") + cmd.Flags().StringVar(&opts.peerEndpoint, "peer-endpoint", "", "the peer's agent endpoint") + cmd.Flags().StringVar(&opts.peerPath, "peer-path", "", "the byte-path where this node mounts the peer's data") + cmd.Flags().StringVar(&opts.peerFingerprint, "peer-fingerprint", "", "the peer's TLS cert fingerprint (sha256:...)") + return cmd +} + +func runNodePair(cmd *cobra.Command, peer string, opts nodePairOpts) error { + cfg, err := requireConfig(cmd) + if err != nil { + return err + } + local := cfg.NodeName + if local == "" { + return fmt.Errorf("node_name is not set in %s — set it before pairing (it names this node to peers)", cfg.Path) + } + if !nodeNameRE.MatchString(peer) { + return fmt.Errorf("peer name %q is invalid (must match %s)", peer, nodeNameRE) + } + if peer == local { + return fmt.Errorf("peer name %q equals this node's name — a node cannot pair with itself", peer) + } + // tokenLtoP is presented when the local node calls the peer; tokenPtoL + // the reverse. Each appears in exactly two slots, cross-matched below. + tokenLtoP, err := newPairToken() + if err != nil { + return err + } + tokenPtoL, err := newPairToken() + if err != nil { + return err + } + printNodePair(cmd.OutOrStdout(), cfg, peer, tokenLtoP, tokenPtoL, opts) + return nil +} + +func printNodePair(out io.Writer, cfg *config.Config, peer, tokenLtoP, tokenPtoL string, opts nodePairOpts) { + local := cfg.NodeName + fmt.Fprintf(out, "# Node pairing: %s <-> %s\n", local, peer) + fmt.Fprintln(out, "# Two bearer tokens were generated and placed so the four bindings already") + fmt.Fprintln(out, "# cross-match. Paste each half into the named machine's config, fill any") + fmt.Fprintln(out, "# <...> placeholders, then restart both agents.") + fmt.Fprintln(out) + + fmt.Fprintf(out, "# ===== on %s (this machine) — add to %s =====\n\n", local, cfg.Path) + writeNodeBlock(out, peer, placeholder(opts.peerEndpoint, "https://<"+peer+"-host>:8443"), + placeholder(opts.peerPath, "<path where "+local+" mounts "+peer+"'s data>"), + tokenLtoP, placeholder(opts.peerFingerprint, "sha256:<"+peer+"'s fingerprint — run `squirrel agent cert` on "+peer+">")) + fmt.Fprintf(out, "\n[agent.auth.peers.%s]\nbearer = %q\n\n", peer, tokenPtoL) + + fmt.Fprintf(out, "# ===== on %s (the peer) — add to its config =====\n\n", peer) + writeNodeBlock(out, local, placeholder(opts.localEndpoint, localEndpointGuess(cfg)), + "<path where "+peer+" mounts "+local+"'s data>", + tokenPtoL, localFingerprint(cfg)) + fmt.Fprintf(out, "\n[agent.auth.peers.%s]\nbearer = %q\n", local, tokenLtoP) +} + +// writeNodeBlock renders one [nodes.<name>] block with its auth + tls +// sub-tables. +func writeNodeBlock(out io.Writer, name, endpoint, path, bearer, fingerprint string) { + fmt.Fprintf(out, "[nodes.%s]\n", name) + fmt.Fprintf(out, "endpoint = %q\n", endpoint) + fmt.Fprintf(out, "path = %q\n", path) + fmt.Fprintf(out, "[nodes.%s.auth]\n", name) + fmt.Fprintf(out, "bearer = %q\n", bearer) + fmt.Fprintf(out, "[nodes.%s.tls]\n", name) + fmt.Fprintf(out, "cert_fingerprint = %q\n", fingerprint) +} + +// localFingerprint returns this node's cert fingerprint when a cert already +// exists at the configured path, else a placeholder pointing at `agent cert`. +func localFingerprint(cfg *config.Config) string { + if cfg.Agent == nil || cfg.Agent.TLSCert == "" { + return "sha256:<this node has no [agent.tls] cert configured>" + } + fp, err := agent.FingerprintCertFile(cfg.Agent.TLSCert) + if err != nil { + return "sha256:<run `squirrel agent cert` on " + cfg.NodeName + " first>" + } + return fp +} + +// localEndpointGuess suggests this node's dialable endpoint from the agent's +// listen port; the host is unknowable from a bind address, so it stays a +// placeholder the operator completes. +func localEndpointGuess(cfg *config.Config) string { + port := "8443" + if cfg.Agent != nil && cfg.Agent.Listen != "" { + if _, p, ok := splitHostPort(cfg.Agent.Listen); ok && p != "" { + port = p + } + } + return "https://<" + cfg.NodeName + "-host>:" + port +} + +// splitHostPort splits "host:port" without erroring on a bare ":port" or an +// unbracketed IPv6, which net.SplitHostPort rejects; we only need the port. +func splitHostPort(addr string) (host, port string, ok bool) { + for i := len(addr) - 1; i >= 0; i-- { + if addr[i] == ':' { + return addr[:i], addr[i+1:], true + } + } + return addr, "", false +} + +func placeholder(value, fallback string) string { + if value != "" { + return value + } + return fallback +} + +// newPairToken mints a 256-bit URL-safe bearer token. +func newPairToken() (string, error) { + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return "", fmt.Errorf("generate pairing token: %w", err) + } + return base64.RawURLEncoding.EncodeToString(buf), nil +} diff --git a/cmd/squirrel/node_pair_test.go b/cmd/squirrel/node_pair_test.go new file mode 100644 index 0000000..949d6b6 --- /dev/null +++ b/cmd/squirrel/node_pair_test.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mbertschler/squirrel/agent" +) + +// bearerValues pulls the quoted value from every `bearer = "..."` line in +// the pairing output. +func bearerValues(out string) []string { + var vals []string + for _, line := range strings.Split(out, "\n") { + trimmed := strings.TrimSpace(line) + if rest, ok := strings.CutPrefix(trimmed, "bearer = "); ok { + vals = append(vals, strings.Trim(rest, `"`)) + } + } + return vals +} + +func TestCLINodePairEmitsBothHalves(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.toml") + if err := os.WriteFile(cfgPath, []byte("node_name = \"nas\"\n"), 0o600); err != nil { + t.Fatal(err) + } + out := runCLI(t, "--config", cfgPath, "node", "pair", "laptop") + + for _, want := range []string{ + "# ===== on nas", "# ===== on laptop", + "[nodes.laptop]", "[nodes.nas]", + "[agent.auth.peers.laptop]", "[agent.auth.peers.nas]", + } { + if !strings.Contains(out, want) { + t.Fatalf("pairing output missing %q:\n%s", want, out) + } + } + + // The four bearer bindings must reduce to exactly two distinct tokens, + // each used twice — the cross-match that kills the F3 error class. + vals := bearerValues(out) + if len(vals) != 4 { + t.Fatalf("expected 4 bearer lines, got %d:\n%s", len(vals), out) + } + counts := map[string]int{} + for _, v := range vals { + if len(v) < 20 { + t.Fatalf("bearer token looks too short: %q", v) + } + counts[v]++ + } + if len(counts) != 2 { + t.Fatalf("expected 2 distinct tokens, got %d: %v", len(counts), counts) + } + for tok, n := range counts { + if n != 2 { + t.Fatalf("token %q appears %d times, want 2 (cross-match broken)", tok, n) + } + } +} + +func TestCLINodePairRequiresNodeName(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.toml") + if err := os.WriteFile(cfgPath, []byte("[volumes.x]\npath = \"/tmp\"\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err := runCLIExpectErr(t, "--config", cfgPath, "node", "pair", "laptop") + if !strings.Contains(err.Error(), "node_name is not set") { + t.Fatalf("expected node_name error, got %v", err) + } +} + +func TestCLINodePairRejectsSelf(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.toml") + if err := os.WriteFile(cfgPath, []byte("node_name = \"nas\"\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err := runCLIExpectErr(t, "--config", cfgPath, "node", "pair", "nas") + if !strings.Contains(err.Error(), "cannot pair with itself") { + t.Fatalf("expected self-pair error, got %v", err) + } +} + +// TestCLINodePairEmbedsLocalFingerprint checks that when this node already +// has a cert, its real fingerprint is baked into the peer's half rather than +// a placeholder. +func TestCLINodePairEmbedsLocalFingerprint(t *testing.T) { + dir := t.TempDir() + certPath := filepath.Join(dir, "agent.crt") + keyPath := filepath.Join(dir, "agent.key") + cfgPath := filepath.Join(dir, "config.toml") + body := fmt.Sprintf(""+ + "node_name = \"nas\"\n\n"+ + "[agent]\nlisten = \"127.0.0.1:8443\"\nauth = { token = \"tok\" }\n\n"+ + "[agent.tls]\ncert = %q\nkey = %q\n", certPath, keyPath) + if err := os.WriteFile(cfgPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + runCLI(t, "--config", cfgPath, "agent", "cert") + fp, err := agent.FingerprintCertFile(certPath) + if err != nil { + t.Fatalf("fingerprint: %v", err) + } + out := runCLI(t, "--config", cfgPath, "node", "pair", "laptop") + if !strings.Contains(out, fp) { + t.Fatalf("pairing output missing this node's real fingerprint %q:\n%s", fp, out) + } +} diff --git a/cmd/squirrel/restore.go b/cmd/squirrel/restore.go index deab82e..f92c1fa 100644 --- a/cmd/squirrel/restore.go +++ b/cmd/squirrel/restore.go @@ -195,9 +195,11 @@ func pickSingleRestoreDestination(cfg *config.Config, vol *config.Volume) (*conf } dest, ok := cfg.Destinations[vol.SyncTo[0]] if !ok { - // sync_to may name a node, not a bucket; restore doesn't yet - // pull from peer nodes, so surface that mismatch explicitly. - return nil, fmt.Errorf("volume %q syncs only to %q which is not a bucket destination — restore from node destinations is not supported", vol.Name, vol.SyncTo[0]) + // sync_to may name a node, not a bucket; restore doesn't pull from + // peer nodes. Rebuilding an edge machine from its hub is a reverse + // peer push driven from the hub — the same mechanism nas→htpc uses + // daily — not a restore, so point there rather than dead-ending. + return nil, fmt.Errorf("volume %q syncs only to peer node %q; restore pulls from bucket destinations, not nodes — to rebuild this machine from its hub, drive a reverse peer push from the hub (see the machine-replacement runbook: the Recovery guide, guides/recovery)", vol.Name, vol.SyncTo[0]) } return dest, nil } diff --git a/cmd/squirrel/root.go b/cmd/squirrel/root.go index 003c593..b327e77 100644 --- a/cmd/squirrel/root.go +++ b/cmd/squirrel/root.go @@ -55,6 +55,9 @@ func newRootCmd() *cobra.Command { root.AddCommand(newPeerSyncCmd()) root.AddCommand(newTUICmd()) root.AddCommand(newDBCmd()) + root.AddCommand(newConfigCmd()) + root.AddCommand(newNodeCmd()) + root.AddCommand(newDestinationCmd()) root.AddCommand(newVersionCmd()) return root } diff --git a/config/agent.go b/config/agent.go index 16d02ef..559faa0 100644 --- a/config/agent.go +++ b/config/agent.go @@ -17,7 +17,8 @@ const ( // callers try to start without it. type Agent struct { // Listen is the TCP address the agent binds to, e.g. "0.0.0.0:8443". - // Required. + // Optional: an empty value selects listener-less, scheduler-only mode + // (F35) — the agent runs its cadences without binding an HTTP server. Listen string // DB optionally overrides the top-level db for the agent process. The // agent binary resolves --db > Agent.DB > top-level db > default; an @@ -28,9 +29,10 @@ type Agent struct { // empty disables TLS (plain HTTP). TLSCert string TLSKey string - // Token is the resolved bearer token literal. Required: an agent - // without a token is an unauthenticated open port and we refuse to - // start one. + // Token is the resolved bearer token literal. Required only when Listen + // is set: an agent with a listener but no token is an unauthenticated + // open port and we refuse to start one. A listener-less agent has no + // HTTP surface and needs no token. Token string // PeerTokens maps a per-peer bearer token to the node name that // presents it, so the agent can recover an authenticated caller @@ -84,9 +86,9 @@ type rawPeerAuth struct { } func resolveAgent(r *rawAgent) (*Agent, error) { - if r.Listen == "" { - return nil, errors.New("listen is required") - } + // An empty listen is the listener-less mode (F35): the agent runs the + // background schedulers (index/sync/audit cadences) without binding an + // HTTP server, for cadence-only machines that never receive peer syncs. a := &Agent{Listen: r.Listen} if r.DB != "" { expanded, err := expandPath(r.DB) @@ -162,7 +164,13 @@ func resolveAgentTLS(r *rawTLS, a *Agent) error { func resolveAgentAuth(r *rawAuth, a *Agent) error { if r == nil || r.Token == nil { - return errors.New("auth.token is required (no agent without authentication)") + if a.Listen == "" { + // Listener-less agent (F35): there is no HTTP surface to + // authenticate, so a bearer token is not required — and peer + // tokens would have no listener to guard either. + return nil + } + return errors.New("auth.token is required when [agent] listen is set (the listener needs a bearer token; omit listen for a listener-less, scheduler-only agent)") } // resolveSecret takes a map[string]any and pulls the named key. We pass // a synthetic single-entry map so the same code handles plain strings diff --git a/config/config.go b/config/config.go index 41a229d..7ae0c1e 100644 --- a/config/config.go +++ b/config/config.go @@ -231,12 +231,15 @@ const ( // filename_encryption is fixed off, keeping the destination tree layout // identical to an unencrypted destination. type Crypt struct { - // Password is the content-encryption password in rclone-obscured form, - // the same representation rclone's own crypt config stores (generate - // one with `rclone obscure`). Accepts a literal or { env = "VAR" }. + // Password is the content-encryption password already in rclone-obscured + // form — the representation rclone's own crypt config stores and rclone + // renders into rclone.conf. The config accepts plaintext (a literal or + // { env = "VAR" }) and squirrel obscures it at load time; a config that + // still supplies a pre-obscured value sets `obscured = true` so it is + // stored here verbatim. Password string - // Password2 is the salt, also rclone-obscured. Optional but - // recommended, matching rclone's crypt config. + // Password2 is the salt, in the same obscured form as Password. + // Optional but recommended, matching rclone's crypt config. Password2 string } diff --git a/config/config_test.go b/config/config_test.go index 2083c96..8ca751a 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -615,6 +615,7 @@ root = "/data" password = "transport-pw" [destinations.offsite.crypt] +obscured = true password = { env = "CRYPT_PASSWORD" } password2 = "obscured-salt" `) @@ -648,6 +649,7 @@ root = "/data" password = "transport-pw" [destinations.offsite.crypt] +obscured = true password = "obscured-pw" password2 = "obscured-salt" `) @@ -718,6 +720,7 @@ user = "u" root = "/data" [destinations.offsite.crypt] +obscured = true password = "obscured-pw" `) cfg, err := Load(p) @@ -1031,14 +1034,21 @@ auth = { } } } -func TestLoadAgentMissingListen(t *testing.T) { - p := writeConfig(t, ` -[agent] -auth = { token = "x" } -`) - _, err := Load(p) - if err == nil || !strings.Contains(err.Error(), "listen is required") { - t.Fatalf("expected listen-required error, got %v", err) +// TestLoadAgentListenerLess covers F35: an [agent] block without `listen` +// is the listener-less mode — valid, scheduler-only. A bare block needs no +// auth, and an auth token without a listener is tolerated (unused). +func TestLoadAgentListenerLess(t *testing.T) { + for _, body := range []string{ + "\n[agent]\n", + "\n[agent]\nauth = { token = \"x\" }\n", + } { + cfg, err := Load(writeConfig(t, body)) + if err != nil { + t.Fatalf("Load(%q): %v", body, err) + } + if cfg.Agent == nil || cfg.Agent.Listen != "" { + t.Fatalf("expected listener-less agent (empty Listen), got %+v", cfg.Agent) + } } } diff --git a/config/crypt_plaintext_test.go b/config/crypt_plaintext_test.go new file mode 100644 index 0000000..717e53d --- /dev/null +++ b/config/crypt_plaintext_test.go @@ -0,0 +1,79 @@ +package config + +import ( + "strings" + "testing" +) + +// TestLoadCryptObscuresPlaintext is the default path (F2): a crypt password +// is supplied in plaintext and squirrel obscures it into rclone's form at +// load time, so the rendered value is not the plaintext and reveals back to +// it. +func TestLoadCryptObscuresPlaintext(t *testing.T) { + cfg, err := Load(writeConfig(t, ` +[destinations.offsite] +type = "sftp" +host = "h" +user = "u" +root = "/data" + +[destinations.offsite.crypt] +password = "hunter2" +password2 = "the-salt" +`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + c := cfg.Destinations["offsite"].Crypt + if c.Password == "hunter2" { + t.Fatalf("crypt password was stored plaintext, not obscured") + } + if got := reveal(t, c.Password); got != "hunter2" { + t.Fatalf("obscured password reveals to %q, want %q", got, "hunter2") + } + if got := reveal(t, c.Password2); got != "the-salt" { + t.Fatalf("obscured salt reveals to %q, want %q", got, "the-salt") + } +} + +// TestLoadCryptObscuredMarkerKeepsVerbatim covers the migration path: with +// `obscured = true` the pre-obscured values are stored (and rendered) +// verbatim. +func TestLoadCryptObscuredMarkerKeepsVerbatim(t *testing.T) { + cfg, err := Load(writeConfig(t, ` +[destinations.offsite] +type = "sftp" +host = "h" +user = "u" +root = "/data" + +[destinations.offsite.crypt] +obscured = true +password = "already-obscured" +password2 = "already-salt" +`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + c := cfg.Destinations["offsite"].Crypt + if c.Password != "already-obscured" || c.Password2 != "already-salt" { + t.Fatalf("obscured marker did not keep values verbatim: %+v", c) + } +} + +func TestLoadCryptObscuredMustBeBool(t *testing.T) { + _, err := Load(writeConfig(t, ` +[destinations.offsite] +type = "sftp" +host = "h" +user = "u" +root = "/data" + +[destinations.offsite.crypt] +obscured = "yes" +password = "hunter2" +`)) + if err == nil || !strings.Contains(err.Error(), "crypt: obscured must be a boolean") { + t.Fatalf("expected obscured-must-be-bool error, got %v", err) + } +} diff --git a/config/destinations.go b/config/destinations.go index 316d4e0..d772593 100644 --- a/config/destinations.go +++ b/config/destinations.go @@ -393,9 +393,13 @@ func optionalZstdLevel(raw map[string]any, key string, def int) (int, error) { } // resolveCrypt validates the optional `crypt` sub-table of a destination. -// A missing key yields nil (no encryption overlay). The two password -// fields go through the same secret resolution as destination credentials; -// password is required, password2 (the salt) is optional. +// A missing key yields nil (no encryption overlay). The two password fields +// go through the same secret resolution as destination credentials; +// password is required, password2 (the salt) is optional. Values are +// plaintext by default and squirrel obscures them into rclone's wire form +// at resolution time (retiring the manual `rclone obscure` step); a +// `obscured = true` marker keeps a pre-obscured value verbatim for configs +// written before this behaviour. func resolveCrypt(raw map[string]any, typ string) (*Crypt, error) { v, ok := raw["crypt"] if !ok { @@ -411,25 +415,63 @@ func resolveCrypt(raw map[string]any, typ string) (*Crypt, error) { if !ok { return nil, errors.New("crypt must be a table, e.g. [destinations.<name>.crypt]") } - password, err := resolveSecret(table, "password") - if err != nil { - return nil, fmt.Errorf("crypt: %w", err) - } - if password == "" { - return nil, errors.New("crypt.password is required (rclone-obscured; generate with `rclone obscure`)") - } - password2, err := resolveSecret(table, "password2") + alreadyObscured, err := cryptObscuredFlag(table) if err != nil { return nil, fmt.Errorf("crypt: %w", err) } for k := range table { - if k != "password" && k != "password2" { + if k != "password" && k != "password2" && k != "obscured" { return nil, fmt.Errorf("crypt: unknown field %q", k) } } + password, err := resolveCryptSecret(table, "password", alreadyObscured) + if err != nil { + return nil, err + } + if password == "" { + return nil, errors.New("crypt.password is required (plaintext by default; set `obscured = true` to supply a pre-obscured value)") + } + password2, err := resolveCryptSecret(table, "password2", alreadyObscured) + if err != nil { + return nil, err + } return &Crypt{Password: password, Password2: password2}, nil } +// cryptObscuredFlag reads the optional `obscured` marker. Its default — +// false — means the password fields carry plaintext squirrel obscures +// itself; true means they already hold rclone-obscured values and must be +// passed through verbatim (the migration path for configs written before +// squirrel obscured on their behalf). +func cryptObscuredFlag(table map[string]any) (bool, error) { + v, ok := table["obscured"] + if !ok { + return false, nil + } + b, ok := v.(bool) + if !ok { + // The caller wraps this with the "crypt:" prefix every other + // resolveCrypt error carries, so the message stays bare here. + return false, errors.New("obscured must be a boolean") + } + return b, nil +} + +// resolveCryptSecret resolves one crypt password field (literal or +// { env = "VAR" }) and, unless it is flagged already-obscured, obscures the +// plaintext into rclone's wire form. An empty/absent field stays empty — an +// absent salt is legal, and an absent password is caught by the caller. +func resolveCryptSecret(table map[string]any, key string, alreadyObscured bool) (string, error) { + val, err := resolveSecret(table, key) + if err != nil { + return "", fmt.Errorf("crypt: %w", err) + } + if val == "" || alreadyObscured { + return val, nil + } + return rcloneObscure(val), nil +} + // validateCryptRemoteNames rejects a config where one destination's crypt // remote name is itself a declared destination — both would render an // rclone.conf section under the same name, and rclone would resolve the diff --git a/config/rclone_obscure_test.go b/config/rclone_obscure_test.go index bb40b47..64f4e40 100644 --- a/config/rclone_obscure_test.go +++ b/config/rclone_obscure_test.go @@ -4,6 +4,7 @@ import ( "crypto/aes" "crypto/cipher" "encoding/base64" + "os/exec" "strings" "testing" ) @@ -65,6 +66,25 @@ func TestRcloneObscureGolden(t *testing.T) { } } +// TestRcloneObscureMatchesRclone cross-checks the output against the real +// `rclone reveal`, proving squirrel speaks rclone's dialect rather than only +// agreeing with the Go reveal reimplemented above. Skipped when rclone is not +// installed. +func TestRcloneObscureMatchesRclone(t *testing.T) { + bin, err := exec.LookPath("rclone") + if err != nil { + t.Skip("rclone not on PATH") + } + const plaintext = "correct horse battery staple" + out, err := exec.Command(bin, "reveal", rcloneObscure(plaintext)).Output() + if err != nil { + t.Fatalf("rclone reveal: %v", err) + } + if got := strings.TrimRight(string(out), "\n"); got != plaintext { + t.Fatalf("rclone reveal = %q, want %q", got, plaintext) + } +} + // rcloneOptionNames lists, per rclone-backed destination type, the option // names rclone's backend actually recognises that squirrel renders into // rclone.conf. Each set was verified against rclone's backend source (the diff --git a/design/friction-log.md b/design/friction-log.md index a2d94ea..b69f8f6 100644 --- a/design/friction-log.md +++ b/design/friction-log.md @@ -21,7 +21,7 @@ rclone 1.74.1, SeaweedFS 3.80, kopia 0.21.1, testbed of 2026-07-23. ## Checkpoint 1 — bootstrap day -**F1 · S2 — TLS cert + fingerprint setup is entirely DIY.** Pinning is +**F1 · S2 — ~~TLS cert + fingerprint setup is entirely DIY.~~ (fixed in #171)** Pinning is the documented trust anchor for LAN agents, but there is no help producing its ingredients: the operator must know the right openssl incantations for cert+key *and* for the `sha256:<hex of DER>` pin @@ -29,13 +29,13 @@ incantations for cert+key *and* for the `sha256:<hex of DER>` pin nor any command covers this. A `squirrel agent cert` (generate) + printed fingerprint at agent startup would remove a whole error class. -**F2 · S3 — crypt passwords must be pre-obscured with rclone by hand.** +**F2 · S3 — ~~crypt passwords must be pre-obscured with rclone by hand.~~ (fixed in #171)** Squirrel owns rclone.conf and hides rclone everywhere else, but crypt config leaks the `rclone obscure` step (4 manual invocations for two crypt destinations). Accepting plaintext-via-env and obscuring internally would match the "squirrel owns the rclone surface" stance. -**F3 · S2 — the peer-token matrix is easy to wire wrong.** One +**F3 · S2 — ~~the peer-token matrix is easy to wire wrong.~~ (`squirrel node pair`, #171)** One nas↔htpc relationship needs four token bindings across two files (each side's `[agent.auth.peers.X]` entry must equal the *other* side's `[nodes.Y].auth.bearer`). Nothing validates the pairing until @@ -43,7 +43,7 @@ a sync fails at runtime with 401. Writing these four configs, the cross-referencing was the single most error-prone part — and there is no `squirrel node add` / pairing flow to generate matching halves. -**F4 · S2 — a freshly configured machine reports nothing.** The +**F4 · S2 — ~~a freshly configured machine reports nothing.~~ (`squirrel config check`, #171)** The natural first command after writing config (`squirrel volumes`) prints an empty list with exit 0: it reads the *database*, not the config, so declared-but-never-indexed volumes are invisible. Config-parse success @@ -211,7 +211,7 @@ TUI-side folding of consecutive no-ops would restore the audit trail's readability. (The `runs` help text also still says "List index runs"; it lists every kind.) -**F20 · S2 — recovering a wrecked destination has no supported path.** +**F20 · S2 — ~~recovering a wrecked destination has no supported path.~~ (`squirrel destination reset` + empty-root guard, #176)** After the F12 bug era the packed-layout guard refused every further s3archive sync ("its history is not packed … point the layout at a fresh destination or root"). But the guard keys on the pair's *run @@ -343,7 +343,7 @@ content-verified method and relay that — the schema already carries ## Checkpoint 8 — restore day -**F28 · S2 — a dead edge machine has no supported restore path.** The +**F28 · S2 — ~~a dead edge machine has no supported restore path.~~ (reverse-peer-push runbook + restore signpost, #176)** The laptop syncs only to the nas (a node), and `restore` refuses nodes outright ("restore from node destinations is not supported" — clear, at least). The machinery for the recovery *exists* — a reverse peer @@ -421,8 +421,10 @@ mount or rclone-style prefix that squirrel neither validates at load time nor mentions when it's wrong (bytes just fail to land); the htpc even needs a `[nodes.nas]` entry with a mandatory `path` that no bytes ever traverse, purely to enable durability pulls. +*Partially addressed in #171: `squirrel config check` now stats and +flags a missing node byte-path; load-time validation is still absent.* -**F35 · S3 — cadence-only machines must still run the full agent.** +**F35 · S3 — ~~cadence-only machines must still run the full agent.~~ (fixed in #175)** A machine that never receives (laptop) runs the HTTP listener and must configure `[agent] listen` + auth token anyway, because the scheduler lives inside the agent. A listener that exists to be unused diff --git a/design/reference-setup.md b/design/reference-setup.md index 92d4c85..85f5ebf 100644 --- a/design/reference-setup.md +++ b/design/reference-setup.md @@ -197,9 +197,9 @@ db = "~/.squirrel/index.db" node_name = "laptop" [agent] -listen = "127.0.0.1:8443" # never receives; agent runs only for the cadences -[agent.auth] -token = { env = "SQUIRREL_AGENT_TOKEN" } +# No `listen`: this roaming laptop never receives peer syncs, so the agent +# runs the sync/index schedulers *without* an HTTP listener (F35). With no +# listener there is nothing to authenticate, so no [agent.auth] token either. [volumes.photos] path = "~/Pictures" diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index 59f3d82..85a9acb 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -62,6 +62,7 @@ export default defineConfig({ { label: "Offsite verification", link: "/guides/verification/" }, { label: "Offloading", link: "/guides/offloading/" }, { label: "Restoring", link: "/guides/restore/" }, + { label: "Recovery & disaster runbooks", link: "/guides/recovery/" }, { label: "Hooks", link: "/guides/hooks/" }, { label: "Auditing for drift", link: "/guides/auditing/" }, { label: "Peer sync", link: "/guides/peer-sync/" }, diff --git a/docs/src/content/docs/guides/agent.md b/docs/src/content/docs/guides/agent.md index 87cecd4..594cf78 100644 --- a/docs/src/content/docs/guides/agent.md +++ b/docs/src/content/docs/guides/agent.md @@ -22,8 +22,32 @@ The agent requires an `[agent]` block in config. schedule. - **Hook firing** — fires per-volume [hooks](/squirrel/guides/hooks/) on change (after each successful index run) and on their `interval`. -- **HTTP server** — exposes a health endpoint and drives the state the - [TUI](/squirrel/guides/tui/) and desktop app read. +- **HTTP server** — exposes a health endpoint, serves peer syncs, and drives + the state the [TUI](/squirrel/guides/tui/) and desktop app read. Only when + `[agent] listen` is set (see below). + +## Listener-less (cadence-only) machines + +A machine that only *originates* content — a roaming laptop that pushes to a +hub and never receives peer syncs — has no reason to run an HTTP listener. Omit +`[agent] listen` and the agent runs its schedulers (index/sync/audit cadences) +**without binding a listener**; with no listener there is nothing to +authenticate, so `[agent.auth]` is optional too. + +```toml +[agent] +# no `listen`: schedulers only, no HTTP server + +[volumes.photos] +path = "~/Pictures" +sync_to = ["nas"] +sync_every = "1h" +``` + +When `listen` *is* set, behaviour is unchanged: the agent runs the schedulers +**and** the HTTP server (and then a bearer token is required). A listener-less +agent with no cadence and no `scan_interval` has nothing to do and refuses to +start, rather than idling silently. ## Database path precedence diff --git a/docs/src/content/docs/guides/recovery.md b/docs/src/content/docs/guides/recovery.md new file mode 100644 index 0000000..09f70f9 --- /dev/null +++ b/docs/src/content/docs/guides/recovery.md @@ -0,0 +1,132 @@ +--- +title: Recovery & disaster runbooks +description: Reset a wrecked destination, rebuild a dead machine from its hub, and the manual disaster-recovery paths that already work — mirror restore, index-snapshot recovery, and packed/content-addressed recovery. +--- + +Recovery in squirrel is deliberately a set of **explicit, human-driven acts** +— never something the agent does on a cadence. Each one preserves the audit +trail: the runs table is never pruned, and no history is overwritten without +an opt-in operator command. This page collects the recovery paths. + +## Resetting a wrecked destination + +`squirrel destination reset <name>` forgets everything the index records about +a destination's remote state — its per-content and per-pack upload ledgers, its +live durability vector, and its push-freshness maxima — so the next sync treats +the destination as fresh and re-uploads. + +Use it when a destination's recorded state no longer matches reality and a sync +refuses to proceed. The classic case: after wiping a remote, or pointing an +existing destination name at a fresh `root`, the +[content-addressed](/squirrel/layouts/content-addressed/) or +[packed](/squirrel/layouts/packed/) layout guard refuses — its recorded history +under that destination name expects a manifest segment or placement map that is +no longer there. Before this verb, the only escape was hand-editing SQLite or +renaming the destination across every machine's config. + +```sh +squirrel destination reset s3archive --dry-run # preview what would clear +squirrel destination reset s3archive --yes # actually clear it +``` + +It is a **change** command, so it is weighty by design: it prints what it will +clear, and refuses without `--yes` (or a `--dry-run` preview). + +### What is and isn't cleared + +Cleared (derived state only): + +- `remote_objects` and `remote_packs` — the upload ledgers. +- the destination's durability vector and push-freshness rows. + +Preserved (the audit trail): + +- the **runs** table — every past sync, verify, and audit stays; the reset is + itself recorded as a new audit run. +- the append-only **durability advance log** — the record of what was ever + asserted durable survives, exactly as [revoking a peer](/squirrel/guides/peer-sync/) + leaves its history intact. +- the **content and files** rows — squirrel never loses track of content. + +The **remote bytes are not touched** — reset only clears squirrel's *record*. +If you want the destination genuinely empty, wipe or repoint the remote root +separately. Once the configured root is empty on the remote, the layout guard +recognises the fresh start on its own and the next sync re-uploads from scratch. + +:::note[Stop the agent first] +A running agent may kick a sync on its cadence mid-reset. Nothing is lost — a +content-addressed re-upload is idempotent — but pausing the agent keeps the +recovery legible. +::: + +## Rebuilding a machine from its hub + +When an edge machine dies (the most likely household disaster), rebuild it from +the hub with a **reverse peer push** — the same peer-sync mechanism the hub uses +to feed a receive-only node every day. There is no separate restore verb for +this: `squirrel restore` pulls from bucket destinations, not from peer nodes, +and the machinery to push a volume to a node already exists. + +The story, using the [reference setup](/squirrel/reference/configuration/) +(rebuilding `laptop` from `nas`): + +1. **Install and re-pair the replacement.** Install squirrel, generate its + agent cert, and re-establish the peer relationship with the hub. Use the + pairing helper so the token matrix is correct by construction: + + ```sh + squirrel agent cert # on the replacement: new cert + pin + squirrel node pair nas # emits matching config halves for both + ``` + + Paste each half into the named machine's config and fill any placeholders + (see [Peer sync](/squirrel/guides/peer-sync/)). The replacement runs an agent + that **listens** so the hub can dial it — the reverse of its normal + initiate-only role. + +2. **Point the hub at the replacement, temporarily.** On the hub, add the + replacement as a `[nodes.laptop]` entry and list it in the `sync_to` of the + volumes to restore. Then drive the push: + + ```sh + squirrel sync photos --to laptop + squirrel sync docs --to laptop + ``` + + The hub enumerates its master copy and pushes; the replacement receives and + re-hashes every path (`peer-blake3`), so the restore is content-verified end + to end. Both sides record correlated runs — the recovery is in the audit + trail like any other sync. + +3. **Return to steady state.** Once the replacement holds the volumes, remove + the temporary hub-side `sync_to`/`[nodes.laptop]` push config, and resume the + normal edge → hub direction (`laptop` initiating to `nas`). + +## Manual disaster recovery that already works + +These paths need no new tooling and are the right answer for whole-repository or +hub loss. They stay first-class. + +### Mirror restore + +For a [mirror](/squirrel/layouts/mirror/) destination (including an encrypted +one), [`squirrel restore`](/squirrel/guides/restore/) pulls the volume back +byte-for-byte, decrypting on the way down and verifying BLAKE3 where the +destination exposes hashes. + +### Recovering the index too + +squirrel rides an [index snapshot](/squirrel/configuration/index-snapshots/) +along to destination buckets under `.squirrel-index/`. When the hub itself dies, +a restore-from-cloud yields the data *and* the index that explains it — fetch +the ride-along snapshot and swap it in as the live index with +[`squirrel db restore`](/squirrel/reference/cli/#squirrel-db). `runs` and `query` +answer immediately against the recovered catalog. + +### Packed & content-addressed recovery + +The [content-addressed](/squirrel/layouts/content-addressed/) and +[packed](/squirrel/layouts/packed/) formats are deliberately simple enough to +[recover without squirrel](/squirrel/reference/formats/#disaster-recovery-without-squirrel) +— the manifest and pack formats are documented, so the bytes are recoverable +with standard tools even if squirrel is unavailable. diff --git a/docs/src/content/docs/layouts/encrypted.md b/docs/src/content/docs/layouts/encrypted.md index 88aa71c..e42ee53 100644 --- a/docs/src/content/docs/layouts/encrypted.md +++ b/docs/src/content/docs/layouts/encrypted.md @@ -13,9 +13,15 @@ password = { env = "OFFSITE_CRYPT_PASSWORD" } password2 = { env = "OFFSITE_CRYPT_SALT" } # salt — optional but recommended ``` -`password` and `password2` are **rclone-obscured** values — the same -representation `rclone config` stores for its own crypt remotes. Generate one -with `rclone obscure <plaintext>`. Both accept a literal or `{ env = "VAR" }`. +`password` and `password2` are **plaintext** — a literal or `{ env = "VAR" }`. +Squirrel obscures them into rclone's on-disk representation when it renders +`rclone.conf`, so you no longer run `rclone obscure` yourself. + +:::note[Migrating an already-obscured config] +Configs written before this behaviour hold pre-obscured values. Add +`obscured = true` to the `crypt` block to keep them verbatim, or replace them +with the plaintext and drop the marker. +::: Squirrel renders two sections into its `rclone.conf` — the underlying remote plus a crypt remote wrapping it — and addresses all sync and restore transfers diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index 07a992d..9d8d2ae 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -26,6 +26,7 @@ squirrel runs fail <id> squirrel hooks [--volume NAME] [--limit N] squirrel volumes squirrel restore <volume> [--from NAME] [--to PATH] [--shallow] [--dry-run] [--in-place] +squirrel destination reset <dest> [--yes] [--dry-run] squirrel audit [<volume>] [--deep | --folders] squirrel peer-sync history <volume> <peer> squirrel peer-sync pull-durability <volume> <peer> [--allow-rewind] @@ -229,6 +230,36 @@ because rclone crypt remotes don't expose content hashes. --- +## squirrel destination + +**Manage a destination's recorded state.** + +On its own it prints help. See [Recovery & disaster runbooks](/squirrel/guides/recovery/). + +### squirrel destination reset + +**Forget a destination's recorded upload and durability state (audit-preserving).** + +``` +squirrel destination reset <destination> +``` + +One positional — the destination name. Clears the `remote_objects`/`remote_packs` +upload ledgers, the durability vector, and the push-freshness rows for the +destination, so the next sync treats it as fresh and re-uploads. The runs table +and the append-only durability advance log are preserved, and the reset is +recorded as an audit run; the remote bytes are untouched. + +| Flag | Default | Meaning | +|---|---|---| +| `--yes` | `false` | Confirm the reset; required to actually clear state. | +| `--dry-run` | `false` | Print what would be cleared without changing anything. | + +Use it to recover a wrecked or repointed destination — see +[Resetting a wrecked destination](/squirrel/guides/recovery/#resetting-a-wrecked-destination). + +--- + ## squirrel audit **Walk a volume looking for out-of-band drift since the last index.** diff --git a/docs/src/content/docs/reference/configuration.md b/docs/src/content/docs/reference/configuration.md index 4e595e9..3fec920 100644 --- a/docs/src/content/docs/reference/configuration.md +++ b/docs/src/content/docs/reference/configuration.md @@ -96,8 +96,9 @@ destination). See [Encrypted (crypt)](/squirrel/layouts/encrypted/). | Key | Meaning | |---|---| -| `password` | rclone-obscured value (`rclone obscure <plaintext>`). Secret. | -| `password2` | rclone-obscured salt; optional but recommended. Secret. | +| `password` | Plaintext encryption password; squirrel obscures it at render time. Secret. | +| `password2` | Plaintext salt; optional but recommended. Secret. | +| `obscured` | `true` if `password`/`password2` are already rclone-obscured (renders them verbatim; default `false`). | Filenames are **not** encrypted (`filename_encryption = off`, fixed by design). @@ -135,3 +136,9 @@ Required to run [`squirrel agent`](/squirrel/guides/agent/). Configures the unattended cadence (`index_every`, `sync_every`), scheduled audits, and an optional agent-specific `db` path (which takes precedence over the top-level `db` when the agent runs). + +`listen` is optional. When set (e.g. `0.0.0.0:8443`), the agent binds an HTTP +server for peer syncs and the health endpoint, and a bearer token is required. +When omitted, the agent runs its schedulers only — the *listener-less* mode for +cadence-only machines that never receive peer syncs; `[agent.auth]` is then +optional. See [The agent](/squirrel/guides/agent/#listener-less-cadence-only-machines). diff --git a/store/destination_reset.go b/store/destination_reset.go new file mode 100644 index 0000000..198f731 --- /dev/null +++ b/store/destination_reset.go @@ -0,0 +1,177 @@ +package store + +import ( + "context" + "database/sql" + "fmt" +) + +// DestinationResetCounts reports how many recorded-state rows a destination +// reset cleared (or, on a dry run, would clear). The four categories are the +// full derived record squirrel keeps about a destination: the per-content +// and per-pack upload ledgers, the live durability vector, and the +// push-freshness maxima. They span every volume — a destination's recorded +// state is not volume-scoped — so the counts are totals across the index. +type DestinationResetCounts struct { + RemoteObjects int64 + RemotePacks int64 + VectorComponents int64 + FreshnessRows int64 +} + +// Total is the sum of all four categories, used as the reset run's +// file_count so the audit trail carries the magnitude at a glance. +func (c DestinationResetCounts) Total() int64 { + return c.RemoteObjects + c.RemotePacks + c.VectorComponents + c.FreshnessRows +} + +// Empty reports whether the destination has no recorded state at all — the +// "nothing to reset" case the CLI reports affirmatively rather than minting +// an empty run. +func (c DestinationResetCounts) Empty() bool { return c.Total() == 0 } + +// rowQuerier is the subset of *sql.DB and *sql.Tx that +// countDestinationRecordedState needs, so the same count runs read-only on +// the DB (the dry-run preview) and inside the reset transaction (the note +// the run records). +type rowQuerier interface { + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// CountDestinationRecordedState returns how much derived state the index +// holds for destination, without changing anything. It is the read-only +// question behind `squirrel destination reset --dry-run`. +func (s *Store) CountDestinationRecordedState(ctx context.Context, destination string) (DestinationResetCounts, error) { + if destination == "" { + return DestinationResetCounts{}, fmt.Errorf("CountDestinationRecordedState: destination must be non-empty") + } + return countDestinationRecordedState(ctx, s.db, destination) +} + +func countDestinationRecordedState(ctx context.Context, q rowQuerier, destination string) (DestinationResetCounts, error) { + var c DestinationResetCounts + counts := []struct { + dst *int64 + query string + }{ + {&c.RemoteObjects, `SELECT COUNT(*) FROM remote_objects WHERE destination = ?`}, + {&c.RemotePacks, `SELECT COUNT(*) FROM remote_packs WHERE destination = ?`}, + {&c.VectorComponents, `SELECT COUNT(*) FROM destination_run_ids WHERE destination = ?`}, + {&c.FreshnessRows, `SELECT COUNT(*) FROM destination_push_freshness WHERE destination = ?`}, + } + for _, item := range counts { + if err := q.QueryRowContext(ctx, item.query, destination).Scan(item.dst); err != nil { + return DestinationResetCounts{}, fmt.Errorf("count destination %q recorded state: %w", destination, err) + } + } + return c, nil +} + +// ResetDestination forgets everything the index records about a destination's +// remote state — its per-content and per-pack upload ledgers, its live +// durability vector, and its push-freshness maxima — so the next sync treats +// the destination as fresh and re-uploads. It is the supported recovery for a +// wrecked or repointed destination (friction log F20): before it, a fresh +// `root` still refused because the layout guard keyed on run history by +// destination name, leaving hand SQL or a config-wide rename as the only outs. +// +// The reset is audit-preserving. It never touches the runs table or the +// append-only destination_run_ids_history advance log, so the record of what +// was ever asserted durable survives; only the live derived state is cleared, +// exactly as RevokeDestinationRunIDsFromSource clears a peer's live +// assertions while leaving the history intact. The contents and files rows — +// the content squirrel must never lose track of — are untouched. +// +// The reset is itself recorded as a run: a kind='audit' row (the run kinds +// are fixed by a CHECK, so this reuses the destination-scoped audit shape +// BeginRemoteVerifyRun already uses — volume_id and destination NULL) whose +// runs_audit trail carries a 'reset-destination' note with the destination +// name and the cleared counts. Everything runs in one transaction: on any +// error nothing is cleared and no run is recorded, so a reset is all-or-nothing. +func (s *Store) ResetDestination(ctx context.Context, destination string) (int64, DestinationResetCounts, error) { + if destination == "" { + return 0, DestinationResetCounts{}, fmt.Errorf("ResetDestination: destination must be non-empty") + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, DestinationResetCounts{}, fmt.Errorf("begin reset destination %q: %w", destination, err) + } + defer func() { _ = tx.Rollback() }() + + counts, err := countDestinationRecordedState(ctx, tx, destination) + if err != nil { + return 0, DestinationResetCounts{}, err + } + atNs := NowNs() + runID, err := insertResetRunTx(ctx, tx, atNs) + if err != nil { + return 0, DestinationResetCounts{}, err + } + if err := deleteDestinationRecordedStateTx(ctx, tx, destination); err != nil { + return 0, DestinationResetCounts{}, err + } + note := fmt.Sprintf("destination=%q objects=%d packs=%d vector=%d freshness=%d", + destination, counts.RemoteObjects, counts.RemotePacks, counts.VectorComponents, counts.FreshnessRows) + if err := appendRunAuditTx(ctx, tx, + RunAuditEntry{RunID: runID, Transition: TransitionResetDestination, Note: note}, atNs); err != nil { + return 0, DestinationResetCounts{}, err + } + if err := finishResetRunTx(ctx, tx, runID, atNs, counts.Total()); err != nil { + return 0, DestinationResetCounts{}, err + } + if err := tx.Commit(); err != nil { + return 0, DestinationResetCounts{}, fmt.Errorf("commit reset destination %q: %w", destination, err) + } + return runID, counts, nil +} + +// insertResetRunTx opens the kind='audit' run that records the reset. The +// runs CHECK keeps audit rows' volume_id and destination NULL (the destination +// travels in the 'reset-destination' runs_audit note instead), matching +// BeginRemoteVerifyRun's destination-scoped audit shape. +func insertResetRunTx(ctx context.Context, tx *sql.Tx, atNs int64) (int64, error) { + res, err := tx.ExecContext(ctx, ` + INSERT INTO runs (kind, volume_id, destination, started_at_ns, status, file_count) + VALUES ('audit', NULL, NULL, ?, 'running', 0) + `, atNs) + if err != nil { + return 0, fmt.Errorf("insert reset run: %w", err) + } + id, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("reset run last insert id: %w", err) + } + return id, nil +} + +// deleteDestinationRecordedStateTx clears the four derived-state tables for +// destination. The append-only destination_run_ids_history and the runs +// table are deliberately absent — the reset preserves the audit trail. +func deleteDestinationRecordedStateTx(ctx context.Context, tx *sql.Tx, destination string) error { + stmts := []string{ + `DELETE FROM remote_objects WHERE destination = ?`, + `DELETE FROM remote_packs WHERE destination = ?`, + `DELETE FROM destination_run_ids WHERE destination = ?`, + `DELETE FROM destination_push_freshness WHERE destination = ?`, + } + for _, stmt := range stmts { + if _, err := tx.ExecContext(ctx, stmt, destination); err != nil { + return fmt.Errorf("clear destination %q recorded state: %w", destination, err) + } + } + return nil +} + +// finishResetRunTx moves the reset run to success with the cleared total as +// its file_count and appends the 'finish' runs_audit line, mirroring +// FinishRun so the reset run reads like any other terminal run. +func finishResetRunTx(ctx context.Context, tx *sql.Tx, runID, atNs, total int64) error { + if _, err := tx.ExecContext(ctx, ` + UPDATE runs SET ended_at_ns = ?, status = 'success', file_count = ? + WHERE id = ? + `, atNs, total, runID); err != nil { + return fmt.Errorf("finish reset run %d: %w", runID, err) + } + return appendRunAuditTx(ctx, tx, + RunAuditEntry{RunID: runID, Transition: TransitionFinish, Note: RunStatusSuccess}, atNs) +} diff --git a/store/destination_reset_test.go b/store/destination_reset_test.go new file mode 100644 index 0000000..f3223a6 --- /dev/null +++ b/store/destination_reset_test.go @@ -0,0 +1,201 @@ +package store + +import ( + "context" + "testing" +) + +// seedDestinationState populates every category of derived state for +// destination on volume vID, plus one durability advance for a second +// destination, and returns the self node id. It leaves a runs row (the +// index run) and a destination_run_ids_history row behind so the +// audit-preservation assertions have something to check against. +func seedDestinationState(t *testing.T, s *Store, vID, runID int64, destination string) int64 { + t.Helper() + ctx := context.Background() + self, err := s.GetSelfNode(ctx) + if err != nil { + t.Fatalf("GetSelfNode: %v", err) + } + + objContent := packContentFixture(t, s, vID, runID, "obj.txt", 0xa1) + if err := s.InsertRemoteObject(ctx, RemoteObject{ + ContentID: objContent, Destination: destination, UploadedRunID: runID, + }); err != nil { + t.Fatalf("InsertRemoteObject: %v", err) + } + + packContent := packContentFixture(t, s, vID, runID, "packed.txt", 0xa2) + if err := s.InsertPacks(ctx, []PackWrite{{ + Pack: Pack{PackKey: packKey(0x11), SizeBytes: 10, MemberCount: 1, CreatedRunID: runID}, + Members: []PackMember{{ContentID: packContent, ByteOffset: 0, ByteLength: 1}}, + }}); err != nil { + t.Fatalf("InsertPacks: %v", err) + } + pack, err := s.GetPackByKey(ctx, packKey(0x11)) + if err != nil { + t.Fatalf("GetPackByKey: %v", err) + } + if err := s.InsertRemotePack(ctx, RemotePack{ + PackID: pack.ID, Destination: destination, UploadedRunID: runID, + }); err != nil { + t.Fatalf("InsertRemotePack: %v", err) + } + + if err := s.UpsertDestinationRunIDVerified(ctx, vID, destination, self.ID, 5, VerifyMethodBlake3, false); err != nil { + t.Fatalf("UpsertDestinationRunIDVerified: %v", err) + } + if err := s.UpsertDestinationPushFreshness(ctx, vID, destination, self.ID, 5); err != nil { + t.Fatalf("UpsertDestinationPushFreshness: %v", err) + } + // A second destination whose state must survive a reset of the first. + if err := s.UpsertDestinationRunIDVerified(ctx, vID, "other", self.ID, 7, VerifyMethodBlake3, false); err != nil { + t.Fatalf("UpsertDestinationRunIDVerified(other): %v", err) + } + return self.ID +} + +// TestResetDestinationClearsDerivedStateAuditPreservingly is the F20 core: +// reset forgets a destination's upload ledgers, vector, and freshness while +// leaving the runs table, the append-only advance log, other destinations, +// and the content rows intact — and records itself as an audit run. +func TestResetDestinationClearsDerivedStateAuditPreservingly(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + vID := makeVolume(t, s, "/v") + runID := makeRun(t, s, vID) + self := seedDestinationState(t, s, vID, runID, "arch") + + counts, err := s.CountDestinationRecordedState(ctx, "arch") + if err != nil { + t.Fatalf("CountDestinationRecordedState: %v", err) + } + if counts.RemoteObjects != 1 || counts.RemotePacks != 1 || counts.VectorComponents != 1 || counts.FreshnessRows != 1 { + t.Fatalf("pre-reset counts = %+v, want one of each", counts) + } + + _, cleared, err := s.ResetDestination(ctx, "arch") + if err != nil { + t.Fatalf("ResetDestination: %v", err) + } + if cleared != counts { + t.Fatalf("cleared = %+v, want %+v", cleared, counts) + } + + // Derived state is gone. + after, err := s.CountDestinationRecordedState(ctx, "arch") + if err != nil { + t.Fatalf("CountDestinationRecordedState after: %v", err) + } + if !after.Empty() { + t.Fatalf("post-reset counts = %+v, want empty", after) + } + vec, err := s.ListDestinationRunIDs(ctx, vID, "arch") + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + if len(vec) != 0 { + t.Fatalf("vector for arch = %+v, want empty", vec) + } + + // The append-only advance log survives — the reset is audit-preserving. + hist, err := s.ListDestinationRunIDHistory(ctx, vID, "arch") + if err != nil { + t.Fatalf("ListDestinationRunIDHistory: %v", err) + } + if len(hist) == 0 { + t.Fatalf("advance history for arch was cleared; reset must preserve it") + } + + // The other destination is untouched. + otherVec, err := s.ListDestinationRunIDs(ctx, vID, "other") + if err != nil { + t.Fatalf("ListDestinationRunIDs(other): %v", err) + } + if len(otherVec) != 1 || otherVec[0].OriginNodeID != self { + t.Fatalf("other vector = %+v, want one component for self", otherVec) + } + + // The content rows are never touched. + if _, err := s.GetByPath(ctx, vID, "obj.txt"); err != nil { + t.Fatalf("content row for obj.txt lost: %v", err) + } +} + +// TestResetDestinationRecordsAuditRun: the reset is a kind='audit' run that +// finishes success and carries a 'reset-destination' runs_audit note with +// the cleared counts, alongside the usual 'finish' transition. +func TestResetDestinationRecordsAuditRun(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + vID := makeVolume(t, s, "/v") + runID := makeRun(t, s, vID) + seedDestinationState(t, s, vID, runID, "arch") + + resetRun, cleared, err := s.ResetDestination(ctx, "arch") + if err != nil { + t.Fatalf("ResetDestination: %v", err) + } + run, err := s.GetRun(ctx, resetRun) + if err != nil { + t.Fatalf("GetRun: %v", err) + } + if run.Kind != RunKindAudit || run.Status != RunStatusSuccess || run.VolumeID.Valid || run.Destination.Valid { + t.Fatalf("reset run = %+v, want a success audit run with NULL volume+destination", run) + } + if run.FileCount != cleared.Total() { + t.Fatalf("run file_count = %d, want cleared total %d", run.FileCount, cleared.Total()) + } + + audits, err := s.ListRunAudit(ctx, resetRun) + if err != nil { + t.Fatalf("ListRunAudit: %v", err) + } + var sawReset, sawFinish bool + for _, a := range audits { + switch a.Transition { + case TransitionResetDestination: + sawReset = true + if !a.Note.Valid || a.Note.String == "" { + t.Fatalf("reset-destination note is empty: %+v", a) + } + case TransitionFinish: + sawFinish = true + } + } + if !sawReset || !sawFinish { + t.Fatalf("audit transitions = %+v, want both reset-destination and finish", audits) + } + + // The original index run is preserved — reset never prunes runs. + if _, err := s.GetRun(ctx, runID); err != nil { + t.Fatalf("original index run %d lost: %v", runID, err) + } +} + +// TestResetDestinationEmpty: a destination with no recorded state reports +// empty counts, so the CLI can report "nothing to reset" rather than mint a +// run. +func TestResetDestinationEmpty(t *testing.T) { + s := openTestStore(t) + counts, err := s.CountDestinationRecordedState(context.Background(), "never-used") + if err != nil { + t.Fatalf("CountDestinationRecordedState: %v", err) + } + if !counts.Empty() { + t.Fatalf("counts = %+v, want empty for an unused destination", counts) + } +} + +// TestResetDestinationRejectsEmptyName: both entry points refuse a blank +// destination rather than clearing everything with an empty-string match. +func TestResetDestinationRejectsEmptyName(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + if _, err := s.CountDestinationRecordedState(ctx, ""); err == nil { + t.Fatalf("CountDestinationRecordedState(\"\") succeeded, want error") + } + if _, _, err := s.ResetDestination(ctx, ""); err == nil { + t.Fatalf("ResetDestination(\"\") succeeded, want error") + } +} diff --git a/store/runs_audit.go b/store/runs_audit.go index e74fb48..14be1d4 100644 --- a/store/runs_audit.go +++ b/store/runs_audit.go @@ -33,6 +33,13 @@ const ( // CHECK keeps destination NULL on audit rows, so this entry is where // the audit trail names the verified destination. TransitionVerifyDestination = "verify-destination" + // TransitionResetDestination records a `squirrel destination reset`: + // the operator forgetting a destination's recorded upload and + // durability state (ResetDestination). It shares the destination-scoped + // kind='audit' run shape, so — like TransitionVerifyDestination — this + // note is where the audit trail names the reset destination and carries + // the cleared counts, the runs CHECK keeping destination NULL on the row. + TransitionResetDestination = "reset-destination" ) // RunAudit is one row of the insert-only runs_audit log: a single diff --git a/sync/content_addressed.go b/sync/content_addressed.go index a7cc933..74b7d87 100644 --- a/sync/content_addressed.go +++ b/sync/content_addressed.go @@ -251,7 +251,10 @@ func (h *contentAddressedHandler) watermark(ctx context.Context, volID int64) (i } segURI := h.segmentURI(last.ID) if _, err := h.rcl.statRemote(ctx, segURI, checkersArgs(h.dest)...); err != nil { - return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no manifest segment at %s — its history does not look content-addressed; point the layout at a fresh destination or root instead of switching an existing one: %w", h.dest.Name, last.ID, segURI, err) + if freshStartOnEmptyRoot(ctx, h.rcl, h.dest) { + return 0, nil + } + return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no manifest segment at %s — its history does not look content-addressed; point the layout at a fresh destination or root, or (after wiping the remote root) run `squirrel destination reset %s`, instead of switching an existing one: %w", h.dest.Name, last.ID, segURI, h.dest.Name, err) } return last.ID, nil } diff --git a/sync/content_addressed_test.go b/sync/content_addressed_test.go index 73dbd08..1777a24 100644 --- a/sync/content_addressed_test.go +++ b/sync/content_addressed_test.go @@ -68,6 +68,7 @@ while [ $# -gt 0 ]; do --hash-type) shift; hashtypes="$hashtypes $1" ;; --include) shift; includes="$includes $1" ;; --checkers) shift ;; + -R) ;; --*) ;; *) if [ -z "$a1" ]; then a1="$1"; else a2="$1"; fi ;; esac @@ -140,6 +141,35 @@ lsjson) printf ']\n' fi ;; +lsf) + # Directory listing for the layout guard's emptiness probe. An injected + # error simulates a not-found (fresh root) or a real failure (auth etc.) + # so the guard's fail-closed classification can be tested. + if [ -n "$RCLONE_FAKE_LSF_ERROR" ]; then echo "$RCLONE_FAKE_LSF_ERROR" >&2; exit 1; fi + # Resolve the base directory, then list it recursively. + lsfsfx="" + case "$a1" in + *:*) + case "${a1%%:*}" in + *-crypt) lsfsfx="$RCLONE_FAKE_CRYPT_SUFFIX" ;; + esac + p="${a1#*:}" + case "$p" in + "${RCLONE_FAKE_STRIP:-//none//}"/*) p="${p#"${RCLONE_FAKE_STRIP}"/}" ;; + esac + dir="$RCLONE_FAKE_ROOT/$p" ;; + *) dir="$a1" ;; + esac + # Listed through a crypt overlay, rclone reports decrypted names — without + # the data suffix it appends on the underlying remote. Strip it so callers + # that match on a file's name (the emptiness probe skipping volume + # markers) see what real rclone would show them. + [ -d "$dir" ] && find "$dir" -type f | while IFS= read -r n; do + [ -n "$lsfsfx" ] && n="${n%"$lsfsfx"}" + printf '%s\n' "$n" + done + exit 0 + ;; *) echo "unexpected rclone subcommand: $cmd $*" >&2; exit 64 ;; esac ` @@ -205,6 +235,7 @@ func setupCAFixture(t *testing.T, destBlock, strip string) *caFixture { t.Setenv("RCLONE_FAKE_HASH_VALUE", "") t.Setenv("RCLONE_FAKE_HASH_PREFIX", "") t.Setenv("RCLONE_FAKE_CRYPT_SUFFIX", "") + t.Setenv("RCLONE_FAKE_LSF_ERROR", "") t.Setenv("RCLONE_FAKE_CAT_FAIL", "") t.Setenv("RCLONE_FAKE_STAT_FAIL", "") t.Setenv("RCLONE_FAKE_BUCKET_ABSENCE", "") @@ -671,6 +702,10 @@ func TestContentAddressedWatermarkGuard(t *testing.T) { if err := f.store.FinishRun(ctx, mirrorRun, store.RunStatusSuccess, "", 1); err != nil { t.Fatalf("finish mirror-era run: %v", err) } + // A real mirror era leaves the volume's files mirrored under the root, + // so the root is non-empty: a genuine layout conflict, distinct from a + // wiped destination (which the fresh-start recognition below handles). + seedRemoteFile(t, f, "pics", "a.txt") rep, err := f.sync(t) if err == nil || !strings.Contains(err.Error(), "does not look content-addressed") { @@ -681,6 +716,79 @@ func TestContentAddressedWatermarkGuard(t *testing.T) { } } +// TestRemoteRootEmptyClassifiesErrors guards the layout guard's fail-closed +// contract: only rclone's canonical not-found reads as an empty (fresh) +// root; any other lsf error must surface so the guard refuses rather than +// mistaking a broken probe (auth, network) for a wiped destination. +func TestRemoteRootEmptyClassifiesErrors(t *testing.T) { + f := setupContentAddressedFixture(t) + ctx := context.Background() + + // A canonical directory-not-found is a fresh root. + t.Setenv("RCLONE_FAKE_LSF_ERROR", "2020/01/01 ERROR : directory not found") + empty, err := f.rcl.remoteRootEmpty(ctx, "offsite:/data") + if err != nil || !empty { + t.Fatalf("not-found lsf = (%t, %v), want (true, nil)", empty, err) + } + + // An auth/network error must surface so the guard fails closed — never + // reported as an empty root just because stdout was empty. + t.Setenv("RCLONE_FAKE_LSF_ERROR", "Failed to create file system: 401 Unauthorized") + empty, err = f.rcl.remoteRootEmpty(ctx, "offsite:/data") + if err == nil { + t.Fatalf("auth lsf error swallowed (empty=%t); guard would proceed instead of refusing", empty) + } + if empty { + t.Fatalf("auth lsf error reported empty root; must be false") + } +} + +// seedRemoteFile writes one file under the fixture's remote root, standing +// in for the bytes a prior (mirror) era left there so the root is non-empty. +func seedRemoteFile(t *testing.T, f *caFixture, parts ...string) { + t.Helper() + p := f.remotePath(parts...) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("mkdir remote file dir: %v", err) + } + if err := os.WriteFile(p, []byte("mirror-era-bytes"), 0o644); err != nil { + t.Fatalf("seed remote file: %v", err) + } +} + +// TestContentAddressedFreshStartOnEmptyRoot is the F20 fix: when name-keyed +// run history records a prior success but the configured root is empty on the +// remote (a wiped or repointed destination, or one cleared by +// `squirrel destination reset`), the guard treats it as a fresh start and +// re-uploads instead of refusing. +func TestContentAddressedFreshStartOnEmptyRoot(t *testing.T) { + f := setupContentAddressedFixture(t) + f.write(t, "a.txt", "alpha") + f.index(t) + + ctx := context.Background() + staleRun, err := f.store.BeginRun(ctx, store.RunKindSync, f.volumeID(t), "offsite", false) + if err != nil { + t.Fatalf("seed stale run: %v", err) + } + if err := f.store.FinishRun(ctx, staleRun, store.RunStatusSuccess, "", 1); err != nil { + t.Fatalf("finish stale run: %v", err) + } + // Remote root left empty — no seedRemoteFile — so the guard reads it as + // a fresh start. + + rep, err := f.sync(t) + if err != nil { + t.Fatalf("fresh-start sync failed: %v (rep=%+v)", err, rep) + } + if rep.Status != store.RunStatusSuccess { + t.Fatalf("Status = %q, want success", rep.Status) + } + if _, err := os.Stat(f.remoteBlob(ObjectsDirName, blake3Hex("alpha"))); err != nil { + t.Fatalf("fresh start did not re-upload the object: %v", err) + } +} + // TestContentAddressedDryRunPreview: --dry-run reports the object count // and bytes a real push would upload, transfers nothing, and writes no // runs row — the mirror dry-run contract (RunID == 0, no new rows, an @@ -782,6 +890,10 @@ func TestContentAddressedDryRunRefusesLayoutFlip(t *testing.T) { if err := f.store.FinishRun(ctx, mirrorRun, store.RunStatusSuccess, "", 1); err != nil { t.Fatalf("finish mirror-era run: %v", err) } + // Leave mirror-era bytes at the root so this is a genuine layout + // conflict rather than a wiped destination, which the fresh-start + // recognition legitimately lets through. + seedRemoteFile(t, f, "pics", "a.txt") rep, err := RunPair(ctx, f.store, Tools{Rclone: f.rcl}, f.pair, Options{DryRun: true}) if err == nil || !strings.Contains(err.Error(), "does not look content-addressed") { diff --git a/sync/packed.go b/sync/packed.go index d4a96dd..6e73c7a 100644 --- a/sync/packed.go +++ b/sync/packed.go @@ -343,7 +343,10 @@ func (h *packedHandler) watermark(ctx context.Context, volID int64) (int64, erro } mapURI := h.mapURI(last.ID) if _, err := h.rcl.statRemote(ctx, mapURI, checkersArgs(h.dest)...); err != nil { - return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no pack placement map at %s — its history is not packed (a mirror or content-addressed root); point the layout at a fresh destination or root instead of switching an existing one: %w", h.dest.Name, last.ID, mapURI, err) + if freshStartOnEmptyRoot(ctx, h.rcl, h.dest) { + return 0, nil + } + return 0, fmt.Errorf("destination %q: the last successful sync (run %d) left no pack placement map at %s — its history is not packed (a mirror or content-addressed root); point the layout at a fresh destination or root, or (after wiping the remote root) run `squirrel destination reset %s`, instead of switching an existing one: %w", h.dest.Name, last.ID, mapURI, h.dest.Name, err) } return last.ID, nil } diff --git a/sync/packed_test.go b/sync/packed_test.go index d888142..0e197ce 100644 --- a/sync/packed_test.go +++ b/sync/packed_test.go @@ -453,6 +453,9 @@ func TestPackedWatermarkGuardRefusesMirror(t *testing.T) { if err := f.store.FinishRun(ctx, mirrorRun, store.RunStatusSuccess, "", 1); err != nil { t.Fatalf("finish mirror run: %v", err) } + // A real mirror era leaves files under the root, so the root is + // non-empty — a genuine layout conflict, not a wiped destination. + seedRemoteFile(t, f, "pics", "a.txt") rep, err := f.sync(t) if err == nil || !strings.Contains(err.Error(), "not packed") { t.Fatalf("expected packed-history refusal, got %v", err) @@ -462,6 +465,36 @@ func TestPackedWatermarkGuardRefusesMirror(t *testing.T) { } } +// TestPackedFreshStartOnEmptyRoot is the packed analogue of the F20 fix: a +// prior success recorded under this destination name, but an empty remote +// root, is a fresh start rather than a refusal. +func TestPackedFreshStartOnEmptyRoot(t *testing.T) { + f := setupPackedFixture(t, "1MiB") + f.write(t, "a.txt", "tiny") + f.index(t) + + ctx := context.Background() + staleRun, err := f.store.BeginRun(ctx, store.RunKindSync, f.volumeID(t), "offsite", false) + if err != nil { + t.Fatalf("seed stale run: %v", err) + } + if err := f.store.FinishRun(ctx, staleRun, store.RunStatusSuccess, "", 1); err != nil { + t.Fatalf("finish stale run: %v", err) + } + // Remote root left empty — the guard reads it as a fresh start. + + rep, err := f.sync(t) + if err != nil { + t.Fatalf("fresh-start sync failed: %v (rep=%+v)", err, rep) + } + if rep.Status != store.RunStatusSuccess { + t.Fatalf("Status = %q, want success", rep.Status) + } + if _, err := os.Stat(f.remoteBlob(PacksDirName, fmt.Sprintf("map-%d", rep.RunID))); err != nil { + t.Fatalf("fresh start wrote no placement map: %v", err) + } +} + // TestPackedDryRunPreview: --dry-run reports the object-vs-pack routing a // real push would take — large content as would-be objects on // RcloneResult, small content as the pack-side PackPreview — without @@ -539,6 +572,10 @@ func TestPackedDryRunRefusesLayoutFlip(t *testing.T) { if err := f.store.FinishRun(ctx, mirrorRun, store.RunStatusSuccess, "", 1); err != nil { t.Fatalf("finish mirror run: %v", err) } + // Leave mirror-era bytes at the root so this is a genuine layout + // conflict rather than a wiped destination, which the fresh-start + // recognition legitimately lets through. + seedRemoteFile(t, f, "pics", "a.txt") rep, err := RunPair(ctx, f.store, Tools{Rclone: f.rcl}, f.pair, Options{DryRun: true}) if err == nil || !strings.Contains(err.Error(), "not packed") { diff --git a/sync/rclone.go b/sync/rclone.go index bfa6ccc..64caa7c 100644 --- a/sync/rclone.go +++ b/sync/rclone.go @@ -15,6 +15,7 @@ import ( "io" "os" "os/exec" + "path" "path/filepath" "regexp" "sort" @@ -24,6 +25,7 @@ import ( "github.com/mbertschler/squirrel/config" "github.com/mbertschler/squirrel/runevents" + "github.com/mbertschler/squirrel/volmark" ) // MinRcloneVersion is the lowest rclone version this binary supports. @@ -450,6 +452,56 @@ func (r *Rclone) deleteFile(ctx context.Context, fileURI string) error { return err } +// remoteRootEmpty reports whether rootURI holds no files, listing it +// recursively via `rclone lsf -R --files-only`. Only rclone's canonical +// "directory not found" (a root that was never created) — accepting the +// "object not found" phrasing too — counts as an empty/fresh root. Every +// other error (auth, network, permission) surfaces so the layout guards +// stay fail-closed and refuse rather than mistaking a broken probe for a +// wiped destination: runPlain carries stderr only in the returned error, +// not in stdout, so an empty stdout alone must never be read as "empty". +// The guards use the empty result to recognise a wiped or repointed +// destination — including one whose recorded state was cleared by +// `squirrel destination reset` — as a fresh start rather than a layout +// conflict. +// +// Per-volume markers (volmark.MarkerName) do not count as content. A root +// that was wiped and re-bootstrapped carries a marker again — the marker gate +// writes one on `--init` before any layout guard runs — so counting markers +// would make the fresh-start recognition unreachable in exactly the situation +// it exists for. Anything else present, including a single stray file, still +// reads as non-empty and keeps the caller's refusal. +func (r *Rclone) remoteRootEmpty(ctx context.Context, rootURI string, extraArgs ...string) (bool, error) { + args := append([]string{"lsf", "-R", "--files-only"}, extraArgs...) + out, err := r.runPlain(ctx, append(args, rootURI)...) + if err != nil { + if isRemoteNotFound(err) { + return true, nil + } + return false, err + } + for _, line := range strings.Split(string(out), "\n") { + if name := strings.TrimSpace(line); name != "" && path.Base(name) != volmark.MarkerName { + return false, nil + } + } + return true, nil +} + +// isRemoteNotFound reports whether err is rclone's canonical +// "directory not found" (or "object not found") — the only listing failure +// the layout guard reads as an empty, fresh root. Any other error must +// surface so the guard fails closed. #150's isNotFoundErr is not on this +// branch, so the canonical-phrase check lives here; runPlain formats +// rclone's stderr into the error string, so matching err.Error() catches it. +func isRemoteNotFound(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "directory not found") || strings.Contains(msg, "object not found") +} + // statRemote returns the size of the single object at uri via // `rclone lsjson --stat`. The content-addressed push uses it to confirm // presence and size of each uploaded object and manifest segment; diff --git a/sync/rclone_test.go b/sync/rclone_test.go index ffc03fd..560fdfa 100644 --- a/sync/rclone_test.go +++ b/sync/rclone_test.go @@ -344,6 +344,7 @@ root = "/data" password = "transport-pw" [destinations.offsite.crypt] +obscured = true password = "obscured-pw" password2 = "obscured-salt" `) diff --git a/sync/sync.go b/sync/sync.go index ef84d68..89a6eef 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -745,6 +745,22 @@ func remoteSubpathURI(dest *config.Destination, subpath string) string { } } +// freshStartOnEmptyRoot reports whether a missing layout marker (a manifest +// segment or pack placement map absent at the last recorded success) should +// be read as a fresh start rather than a layout conflict. It is true only +// when dest's configured root holds no files on the remote — a wiped or +// repointed destination, or one whose recorded state was cleared by +// `squirrel destination reset` — so the name-keyed run history no longer +// describes anything on disk and a fresh full push is safe (it skips +// nothing). A non-empty root, or any error probing it, returns false so the +// caller keeps its refusal: fail-closed, because refusing a real +// layout-switch is recoverable while a delta against a stale watermark +// silently skips content. +func freshStartOnEmptyRoot(ctx context.Context, rcl *Rclone, dest *config.Destination) bool { + empty, err := rcl.remoteRootEmpty(ctx, remoteSubpathURI(dest, ""), checkersArgs(dest)...) + return err == nil && empty +} + // destinationVolumeURI returns the rclone destination spec for the given // volume under dest. func destinationVolumeURI(dest *config.Destination, volumeName string) string {