diff --git a/agent/agent.go b/agent/agent.go index 09752ce..fd7202b 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -54,6 +54,42 @@ type SyncRunReport struct { Err error } +// VerifyRunner is the function shape the scheduler delegates one +// destination's verify pass to (F32). It runs the same re-check as +// `squirrel verify ` and records a kind='audit' run, so the +// agent package needs no edge to the sync package. A zero RunID with a nil +// Err means the destination had nothing recorded to verify (a no-op, no run +// row written); a non-zero RunID carries the audit run's outcome. Status is +// one of store.RunStatus*. +type VerifyRunner func(ctx context.Context, destName string) VerifyRunReport + +// VerifyRunReport is the VerifyRunner result surfaced to the scheduler's +// kicked/finished/error log. +type VerifyRunReport struct { + RunID int64 + Status string + Err error +} + +// DurabilityPuller is the function shape the scheduler delegates one +// (volume, peer) durability pull to (F33). It runs the same metadata-only +// merge as `squirrel peer-sync pull-durability` — never rewinding a +// watermark, because the agent does not escalate — and records a +// kind='audit' run. Counts ride along for the finished log line. +type DurabilityPuller func(ctx context.Context, vol *config.Volume, peerName string) DurabilityPullReport + +// DurabilityPullReport is the DurabilityPuller result surfaced to the +// scheduler's kicked/finished/error log. +type DurabilityPullReport struct { + RunID int64 + Status string + Err error + Fetched int + Applied int + Dropped int + Rewinds int +} + // Config configures one agent. Fields are validated by New: Version is // always required; Listen is optional (empty selects listener-less, // scheduler-only mode, F35); Token is required only when Listen is set (an @@ -89,12 +125,19 @@ type Config struct { // which is what tests of the auth/health surface want. Volumes map[string]*config.Volume // Destinations maps destination name → resolved config-side - // destination. The durability endpoint reads it to advertise, per - // destination this node syncs a volume to, whether it can ever gate - // offload — so a peer gating on a relayed required target can fail fast - // when this node's destination is structurally incapable (#145). A nil - // map simply advertises no capabilities, degrading a peer's pre-check to - // the per-file gate; it never affects the sync endpoints. + // destination, read by two consumers. + // + // The durability endpoint advertises, per destination this node syncs a + // volume to, whether it can ever gate offload — so a peer gating on a + // relayed required target can fail fast when this node's destination is + // structurally incapable (#145). A nil map simply advertises no + // capabilities, degrading a peer's pre-check to the per-file gate; it + // never affects the sync endpoints. + // + // The scheduler consults it for the content-addressed/packed + // destinations that carry a verify cadence (per-destination + // verify_every, or the agent-level VerifyEvery default). Nil disables + // the verify cadence. Destinations map[string]*config.Destination // SyncRunner is the cadence scheduler's (#39) hook for invoking // one (volume, destination) sync. The CLI wires this to a closure @@ -108,6 +151,24 @@ type Config struct { // peer-sync receiver fixture, and a direct agent→sync edge would // close the cycle. SyncRunner SyncRunner + // Nodes maps peer node name → resolved config-side node. The + // scheduler consults it for the per-node pull_durability_every + // cadence. Nil disables the durability-pull cadence. + Nodes map[string]*config.Node + // VerifyEvery is the fleet-wide default verify cadence applied to every + // verifiable destination that declares no verify_every of its own + // (cfg.Agent.VerifyEvery). Zero means no default; a per-destination + // cadence still applies. + VerifyEvery time.Duration + // VerifyRunner is the scheduler's hook for running one destination's + // verify pass (F32). Nil disables verify-kicking. The CLI wires it to a + // closure over sync.VerifyRemote; an agent whose config has no + // verifiable destination with a cadence ignores it. + VerifyRunner VerifyRunner + // DurabilityPuller is the scheduler's hook for running one (volume, + // peer) durability pull (F33). Nil disables pull-kicking. The CLI wires + // it to a closure over sync.PullDurability. + DurabilityPuller DurabilityPuller // SchedulerTick overrides the scheduler's evaluation period. // Zero falls back to DefaultSchedulerTick. Tests pin it to a // small value; production rarely needs to touch it. diff --git a/agent/durability.go b/agent/durability.go index afc0dc8..a422cfb 100644 --- a/agent/durability.go +++ b/agent/durability.go @@ -63,6 +63,10 @@ func (r *peerSyncRouter) durabilityResponse(ctx context.Context, volumeName stri if err != nil { return syncproto.DurabilityResponse{}, fmt.Errorf("list push freshness: %w", err) } + // The effective verify cadence per destination, relayed so a puller can + // apply the fingerprint-verified cadence coupling against this + // responder's own re-confirmation schedule (see DurabilityComponent). + cadences := resolveVerifyCadences(r.srv.cfg.Destinations, r.srv.cfg.VerifyEvery) names := make(map[int64]string, 4) resolve := func(nodeID int64) (string, error) { if name, ok := names[nodeID]; ok { @@ -86,12 +90,13 @@ func (r *peerSyncRouter) durabilityResponse(ctx context.Context, volumeName stri return syncproto.DurabilityResponse{}, err } resp.Components = append(resp.Components, syncproto.DurabilityComponent{ - Destination: row.Destination, - OriginNode: name, - OriginRun: row.OriginRunID, - UpdatedAtNs: row.UpdatedAtNs, - VerifyMethod: row.VerifyMethod, - VerifiedAtNs: row.VerifiedAtNs.Int64, // zero when NULL (unknown) + Destination: row.Destination, + OriginNode: name, + OriginRun: row.OriginRunID, + UpdatedAtNs: row.UpdatedAtNs, + VerifyMethod: row.VerifyMethod, + VerifiedAtNs: row.VerifiedAtNs.Int64, // zero when NULL (unknown) + VerifyEveryNs: int64(cadences[row.Destination]), }) } for _, row := range fresh { diff --git a/agent/durability_test.go b/agent/durability_test.go index c0ab8b9..e85811a 100644 --- a/agent/durability_test.go +++ b/agent/durability_test.go @@ -7,8 +7,10 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" "github.com/mbertschler/squirrel/syncproto" ) @@ -93,6 +95,54 @@ func TestDurabilityEndpointListsComponents(t *testing.T) { } } +// TestDurabilityEndpointRelaysVerifyCadence: the responder relays its +// effective verify cadence per destination (a per-destination verify_every +// or the [agent] default) so a puller can apply the fingerprint-verified +// cadence coupling against the responder's own re-confirmation schedule. A +// destination with no cadence (a mirror, or one that declares none) relays +// zero, which the puller reads as fail-closed (issue #155). +func TestDurabilityEndpointRelaysVerifyCadence(t *testing.T) { + ctx := context.Background() + vol := &config.Volume{Name: "pics", Path: t.TempDir()} + srv := newTestServer(t, Config{ + Volumes: map[string]*config.Volume{vol.Name: vol}, + Destinations: map[string]*config.Destination{ + "s3archive": {Layout: config.LayoutPacked, VerifyEvery: 168 * time.Hour}, + "mirror": {Layout: config.LayoutMirror}, + }, + }) + + v, err := srv.store.CreateVolume(ctx, vol.Name, vol.Path) + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + self, err := srv.store.GetSelfNode(ctx) + if err != nil { + t.Fatalf("GetSelfNode: %v", err) + } + if err := srv.store.UpsertDestinationRunIDVerified(ctx, v.ID, "s3archive", self.ID, 7, store.VerifyMethodFingerprint, false); err != nil { + t.Fatalf("seed s3archive: %v", err) + } + if err := srv.store.UpsertDestinationRunIDVerified(ctx, v.ID, "mirror", self.ID, 3, store.VerifyMethodBlake3, false); err != nil { + t.Fatalf("seed mirror: %v", err) + } + + var resp syncproto.DurabilityResponse + if code := postDurability(t, srv, syncproto.DurabilityRequest{Volume: "pics"}, &resp); code != http.StatusOK { + t.Fatalf("status = %d, want 200", code) + } + got := map[string]int64{} + for _, c := range resp.Components { + got[c.Destination] = c.VerifyEveryNs + } + if got["s3archive"] != int64(168*time.Hour) { + t.Fatalf("s3archive verify_every_ns = %d, want %d", got["s3archive"], int64(168*time.Hour)) + } + if got["mirror"] != 0 { + t.Fatalf("mirror verify_every_ns = %d, want 0 (no cadence)", got["mirror"]) + } +} + // TestDurabilityEndpointAdvertisesCapabilities: the endpoint advertises, // per destination the node syncs the volume to, whether it can ever gate // offload — a crypt mirror as incapable (with a reason), a packed diff --git a/agent/scheduler.go b/agent/scheduler.go index 17d3883..9c95b9e 100644 --- a/agent/scheduler.go +++ b/agent/scheduler.go @@ -39,6 +39,20 @@ type scheduler struct { // tests that construct a bare scheduler; hookRunner methods tolerate a // nil receiver so the firing sites need no extra guard. hooks *hookRunner + // verifyRun and durabilityPull are the injected hooks for the two + // non-volume cadences (F32/F33); nil disables the respective cadence. + // verifyEvery maps a verifiable destination name to its resolved verify + // cadence (per-destination verify_every or the [agent] default); + // pullEvery maps a peer name to its pull_durability_every. lastVerify + // and lastPull are in-memory watermarks (destination name, and + // volume+peer key, → last attempt) — see runVerify for why they need + // not persist. + verifyRun VerifyRunner + durabilityPull DurabilityPuller + verifyEvery map[string]time.Duration + pullEvery map[string]time.Duration + lastVerify map[string]time.Time + lastPull map[string]time.Time } // lockHolder is the subset of the peer-sync router the scheduler uses @@ -58,21 +72,60 @@ func newScheduler(srv *Server, tickEvery time.Duration, now func() time.Time) *s now = time.Now } return &scheduler{ - store: srv.store, - volumes: srv.cfg.Volumes, - syncRun: srv.cfg.SyncRunner, - logger: srv.cfg.Logger, - locks: srv.router, - tickEvery: tickEvery, - now: now, - hooks: newHookRunner(srv.store, srv.cfg.Logger), + store: srv.store, + volumes: srv.cfg.Volumes, + syncRun: srv.cfg.SyncRunner, + logger: srv.cfg.Logger, + locks: srv.router, + tickEvery: tickEvery, + now: now, + hooks: newHookRunner(srv.store, srv.cfg.Logger), + verifyRun: srv.cfg.VerifyRunner, + durabilityPull: srv.cfg.DurabilityPuller, + verifyEvery: resolveVerifyCadences(srv.cfg.Destinations, srv.cfg.VerifyEvery), + pullEvery: resolvePullCadences(srv.cfg.Nodes), + lastVerify: map[string]time.Time{}, + lastPull: map[string]time.Time{}, } } +// resolveVerifyCadences maps each verifiable (content-addressed or packed) +// destination to its effective verify cadence: the destination's own +// verify_every when set, otherwise the [agent] verify_every default. Only +// destinations with a positive resulting cadence are included, so a config +// with neither knob set yields an empty map and no verify activity. +func resolveVerifyCadences(dests map[string]*config.Destination, def time.Duration) map[string]time.Duration { + out := make(map[string]time.Duration) + for name, d := range dests { + if d.Layout != config.LayoutContentAddressed && d.Layout != config.LayoutPacked { + continue + } + eff := d.VerifyEvery + if eff == 0 { + eff = def + } + if eff > 0 { + out[name] = eff + } + } + return out +} + +// resolvePullCadences maps each peer node declaring pull_durability_every to +// that cadence. Nodes without the knob are omitted, so a config with no pull +// cadence yields an empty map and no durability-pull activity. +func resolvePullCadences(nodes map[string]*config.Node) map[string]time.Duration { + out := make(map[string]time.Duration) + for name, n := range nodes { + if n.PullDurabilityEvery > 0 { + out[name] = n.PullDurabilityEvery + } + } + return out +} + // anyScheduledVolume reports whether any configured volume has at -// least one cadence knob set. Serve skips starting the scheduler -// goroutine when this returns false so the agent has no idle goroutine -// when nothing is scheduled. +// least one cadence knob set. func (s *scheduler) anyScheduledVolume() bool { for _, v := range s.volumes { if volumeHasCadence(v) { @@ -82,6 +135,47 @@ func (s *scheduler) anyScheduledVolume() bool { return false } +// anyScheduledWork reports whether the scheduler has anything to do at all: +// a volume cadence, a destination verify cadence (F32), or a peer durability +// cadence (F33). Serve skips starting the scheduler goroutine when this +// returns false so the agent has no idle goroutine when nothing is +// scheduled. The verify/durability cadences are decisive on their own — a +// receive-only node (the reference htpc) declares no volume cadence yet must +// still run the scheduler purely to keep its offload-gate evidence fresh. +func (s *scheduler) anyScheduledWork() bool { + return s.anyScheduledVolume() || len(s.verifyEvery) > 0 || len(s.pullEvery) > 0 +} + +// ScheduledWorkInConfig answers anyScheduledWork's question straight from a +// config, before any Server exists: does this config give the schedulers +// anything to do — a drift scan, a volume cadence, a destination verify +// cadence (F32), or a peer durability-pull cadence (F33)? +// +// The CLI consults it to refuse a listener-less agent that would idle +// silently (F35). It reads the same cadences through the same resolvers as +// the scheduler's own gate, so the two cannot drift: a receive-only machine +// whose only work is a pull_durability_every has real work and must start, +// even though it declares no volume cadence at all. +func ScheduledWorkInConfig(cfg *config.Config) bool { + if cfg == nil { + return false + } + var scanInterval, verifyDefault time.Duration + if cfg.Agent != nil { + scanInterval, verifyDefault = cfg.Agent.ScanInterval, cfg.Agent.VerifyEvery + } + if scanInterval > 0 { + return true + } + for _, v := range cfg.Volumes { + if volumeHasCadence(v) { + return true + } + } + return len(resolveVerifyCadences(cfg.Destinations, verifyDefault)) > 0 || + len(resolvePullCadences(cfg.Nodes)) > 0 +} + // volumeHasCadence reports whether a volume opts into any scheduler-driven // cadence: a sync, a standalone index, or an interval hook. The interval // hook counts on its own — a volume can want periodic verification of its @@ -128,6 +222,14 @@ func (s *scheduler) tick(ctx context.Context) { } s.evaluateVolume(ctx, v) } + // The verify and durability-pull cadences key on destinations and peer + // nodes rather than volumes, so they run after the per-volume loop as + // their own phases. Both are read-only / metadata-only and take no + // volume lock: verify touches only the remote-object bookkeeping, and a + // durability pull only merges peer-supplied vectors — neither races the + // per-volume index/sync/audit work the loop above coordinates. + s.evaluateVerify(ctx) + s.evaluateDurabilityPulls(ctx) } func sortedVolumeNames(m map[string]*config.Volume) []string { @@ -511,3 +613,156 @@ func (s *scheduler) runSync(ctx context.Context, vol *config.Volume, volumeID in "run_id", rep.RunID, "err", rep.Err.Error()) } } + +// evaluateVerify walks every destination with a resolved verify cadence +// (F32) and kicks its verify pass when due, in name order for deterministic +// logs. A nil verifyRun (no rclone wired, or no verifiable destination) +// disables the phase. The pass is the same one `squirrel verify` runs and +// records its own kind='audit' run; the agent never bootstraps or writes to +// the destination, so the marker/init boundary is respected. +func (s *scheduler) evaluateVerify(ctx context.Context) { + if s.verifyRun == nil { + return + } + for _, destName := range sortedCadenceNames(s.verifyEvery) { + if ctx.Err() != nil { + return + } + if !s.verifyDue(destName) { + continue + } + s.runVerify(ctx, destName) + } +} + +// verifyDue reports whether `now - last_verify >= cadence` for the +// destination. A destination not yet verified this process lifetime is due. +func (s *scheduler) verifyDue(destName string) bool { + last, ok := s.lastVerify[destName] + if !ok { + return true + } + return s.now().Sub(last) >= s.verifyEvery[destName] +} + +// runVerify kicks one destination's verify pass and emits the +// kicked/finished/error log triple. The in-memory watermark is advanced up +// front so a failed or empty pass consumes the cadence window exactly like +// the volume cadences (no special retry — the next window re-evaluates). It +// need not persist: verify is read-only and idempotent, so re-running once +// after an agent restart is harmless and, for evidence freshness, desirable. +func (s *scheduler) runVerify(ctx context.Context, destName string) { + s.lastVerify[destName] = s.now() + s.logger.Info("scheduler.kicked", "kind", "verify", "destination", destName) + start := s.now() + rep := s.verifyRun(ctx, destName) + duration := s.now().Sub(start) + status := rep.Status + if status == "" { + // No recorded objects or packs: nothing to re-check and no audit + // run written (run_id stays 0). Reported distinctly so the log + // doesn't imply a verification actually ran. + status = "no-op" + } + s.logger.Info("scheduler.finished", + "kind", "verify", "destination", destName, + "run_id", rep.RunID, "status", status, + "duration_ms", duration.Milliseconds()) + if rep.Err != nil { + s.logger.Error("scheduler.error", + "kind", "verify", "destination", destName, + "run_id", rep.RunID, "err", rep.Err.Error()) + } +} + +// evaluateDurabilityPulls walks every peer with a pull_durability_every +// cadence (F33) and, for each locally-configured volume with a stake in the +// peer's evidence, kicks a durability pull when due. A nil durabilityPull +// disables the phase. Peers and volumes are visited in name order so the log +// is deterministic. +func (s *scheduler) evaluateDurabilityPulls(ctx context.Context) { + if s.durabilityPull == nil { + return + } + for _, peer := range sortedCadenceNames(s.pullEvery) { + for _, volName := range sortedVolumeNames(s.volumes) { + if ctx.Err() != nil { + return + } + vol := s.volumes[volName] + if !volumeHasDurabilityStake(vol) { + continue + } + if !s.pullDue(volName, peer) { + continue + } + s.runDurabilityPull(ctx, vol, peer) + } + } +} + +// volumeHasDurabilityStake reports whether a durability pull could advance +// anything for this volume. The merge keeps only components for destinations +// the volume references (offload_requires ∪ sync_to — see +// sync.acceptedDestinations), so a volume naming neither has nothing to gain +// and is skipped. This is what lets the reference htpc pull its media +// volume's relayed s3archive evidence while skipping the photos volume it +// merely plays back. +func volumeHasDurabilityStake(v *config.Volume) bool { + return len(v.OffloadRequires) > 0 || len(v.SyncTo) > 0 +} + +// pullDue reports whether `now - last_pull >= cadence` for the (volume, +// peer) pair. A pair not yet pulled this process lifetime is due. +func (s *scheduler) pullDue(volume, peer string) bool { + last, ok := s.lastPull[pullKey(volume, peer)] + if !ok { + return true + } + return s.now().Sub(last) >= s.pullEvery[peer] +} + +// runDurabilityPull kicks one (volume, peer) durability pull and emits the +// kicked/finished/error log triple. Watermark advanced up front, same policy +// and rationale as runVerify. The injected puller never rewinds a watermark +// — the agent does not escalate. +func (s *scheduler) runDurabilityPull(ctx context.Context, vol *config.Volume, peer string) { + s.lastPull[pullKey(vol.Name, peer)] = s.now() + s.logger.Info("scheduler.kicked", "kind", "pull-durability", "volume", vol.Name, "peer", peer) + start := s.now() + rep := s.durabilityPull(ctx, vol, peer) + duration := s.now().Sub(start) + status := rep.Status + if status == "" { + status = store.RunStatusFailed + } + s.logger.Info("scheduler.finished", + "kind", "pull-durability", "volume", vol.Name, "peer", peer, + "run_id", rep.RunID, "status", status, + "fetched", rep.Fetched, "applied", rep.Applied, "dropped", rep.Dropped, + "rewinds", rep.Rewinds, + "duration_ms", duration.Milliseconds()) + if rep.Err != nil { + s.logger.Error("scheduler.error", + "kind", "pull-durability", "volume", vol.Name, "peer", peer, + "run_id", rep.RunID, "err", rep.Err.Error()) + } +} + +// pullKey is the composite watermark key for a (volume, peer) durability +// pull. The NUL separator can't appear in a validated volume or node name, +// so distinct pairs never collide. +func pullKey(volume, peer string) string { + return volume + "\x00" + peer +} + +// sortedCadenceNames returns the keys of a name→cadence map in sorted order, +// so the verify and durability phases visit their subjects deterministically. +func sortedCadenceNames(m map[string]time.Duration) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/agent/scheduler_cadence_test.go b/agent/scheduler_cadence_test.go new file mode 100644 index 0000000..9cc38ad --- /dev/null +++ b/agent/scheduler_cadence_test.go @@ -0,0 +1,317 @@ +package agent + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" +) + +// TestResolveVerifyCadences pins the per-destination-vs-agent-default +// resolution and the verifiable-layout filter that the scheduler and the +// CLI's rclone wiring both rely on. +func TestResolveVerifyCadences(t *testing.T) { + dests := map[string]*config.Destination{ + "ca": {Name: "ca", Layout: config.LayoutContentAddressed, VerifyEvery: time.Hour}, + "packed": {Name: "packed", Layout: config.LayoutPacked}, // no own cadence → default + "mirror": {Name: "mirror", Layout: config.LayoutMirror}, // never verifiable + } + got := resolveVerifyCadences(dests, 6*time.Hour) + if got["ca"] != time.Hour { + t.Errorf("ca cadence = %s; want its own 1h", got["ca"]) + } + if got["packed"] != 6*time.Hour { + t.Errorf("packed cadence = %s; want the 6h agent default", got["packed"]) + } + if _, ok := got["mirror"]; ok { + t.Errorf("mirror layout must never carry a verify cadence, got %s", got["mirror"]) + } + + // With no agent default, a verifiable destination without its own + // cadence is omitted entirely. + noDefault := resolveVerifyCadences(dests, 0) + if _, ok := noDefault["packed"]; ok { + t.Errorf("packed should be omitted when neither its own nor the agent default is set") + } + if noDefault["ca"] != time.Hour { + t.Errorf("ca should still carry its own 1h with no agent default") + } +} + +// TestResolvePullCadences pins that only nodes declaring +// pull_durability_every carry a cadence. +func TestResolvePullCadences(t *testing.T) { + nodes := map[string]*config.Node{ + "nas": {Name: "nas", PullDurabilityEvery: 24 * time.Hour}, + "laptop": {Name: "laptop"}, // no cadence + } + got := resolvePullCadences(nodes) + if got["nas"] != 24*time.Hour { + t.Errorf("nas cadence = %s; want 24h", got["nas"]) + } + if _, ok := got["laptop"]; ok { + t.Errorf("laptop declares no pull cadence; must be omitted") + } +} + +// TestSchedulerAnyScheduledWork covers the broadened start gate: a +// verify-only or pull-only config (a receive-only node has no volume +// cadence at all) must still start the scheduler. +func TestSchedulerAnyScheduledWork(t *testing.T) { + f := newSchedulerFixture(t, &config.Volume{Name: "idle"}) + sch := f.scheduler() + if sch.anyScheduledWork() { + t.Fatalf("idle config should have no scheduled work") + } + sch.verifyEvery = map[string]time.Duration{"s3archive": time.Hour} + if !sch.anyScheduledWork() { + t.Fatalf("a verify cadence alone must start the scheduler") + } + sch.verifyEvery = nil + sch.pullEvery = map[string]time.Duration{"nas": time.Hour} + if !sch.anyScheduledWork() { + t.Fatalf("a durability-pull cadence alone must start the scheduler (receive-only node)") + } +} + +// TestScheduledWorkInConfig is the config-side half of the same gate, which +// the CLI uses to refuse a do-nothing listener-less agent (F35). It must +// agree with anyScheduledWork on every cadence — in particular a +// receive-only machine (the reference htpc: no volume cadence, no scan, no +// listener) whose sole work is a peer pull_durability_every, or a +// verify-only machine, since refusing either would keep exactly the nodes +// F32/F33 exist to serve from ever starting. +func TestScheduledWorkInConfig(t *testing.T) { + idleVol := map[string]*config.Volume{"idle": {Name: "idle"}} + for _, tc := range []struct { + name string + cfg *config.Config + want bool + }{ + {"nil config", nil, false}, + {"no agent block", &config.Config{Volumes: idleVol}, false}, + {"idle", &config.Config{Agent: &config.Agent{}, Volumes: idleVol}, false}, + {"scan interval", &config.Config{Agent: &config.Agent{ScanInterval: time.Hour}}, true}, + {"volume sync cadence", &config.Config{ + Agent: &config.Agent{}, + Volumes: map[string]*config.Volume{"pics": {Name: "pics", SyncEvery: time.Hour}}, + }, true}, + {"per-destination verify cadence", &config.Config{ + Agent: &config.Agent{}, + Volumes: idleVol, + Destinations: map[string]*config.Destination{"s3archive": {Name: "s3archive", Layout: config.LayoutPacked, VerifyEvery: time.Hour}}, + }, true}, + {"agent-default verify cadence", &config.Config{ + Agent: &config.Agent{VerifyEvery: time.Hour}, + Volumes: idleVol, + Destinations: map[string]*config.Destination{"s3archive": {Name: "s3archive", Layout: config.LayoutContentAddressed}}, + }, true}, + {"verify default on an unverifiable layout", &config.Config{ + Agent: &config.Agent{VerifyEvery: time.Hour}, + Volumes: idleVol, + Destinations: map[string]*config.Destination{"cloudbox": {Name: "cloudbox", Layout: config.LayoutMirror}}, + }, false}, + {"receive-only pull cadence", &config.Config{ + Agent: &config.Agent{}, + Volumes: idleVol, + Nodes: map[string]*config.Node{"nas": {Name: "nas", PullDurabilityEvery: 24 * time.Hour}}, + }, true}, + {"peer node without a pull cadence", &config.Config{ + Agent: &config.Agent{}, + Volumes: idleVol, + Nodes: map[string]*config.Node{"nas": {Name: "nas"}}, + }, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := ScheduledWorkInConfig(tc.cfg); got != tc.want { + t.Fatalf("ScheduledWorkInConfig = %t, want %t", got, tc.want) + } + }) + } +} + +// TestSchedulerVerifyCadence walks the verify cadence across ticks: first +// tick kicks (never verified), a within-cadence tick skips, a past-cadence +// tick re-fires. The kicked/finished discipline is asserted from the log. +func TestSchedulerVerifyCadence(t *testing.T) { + f := newSchedulerFixture(t, &config.Volume{Name: "idle"}) + sch := f.scheduler() + var calls atomic.Int32 + sch.verifyRun = func(context.Context, string) VerifyRunReport { + calls.Add(1) + return VerifyRunReport{RunID: 7, Status: store.RunStatusSuccess} + } + sch.verifyEvery = map[string]time.Duration{"s3archive": time.Hour} + ctx := context.Background() + + sch.tick(ctx) + if calls.Load() != 1 { + t.Fatalf("tick 1 verify calls = %d; want 1", calls.Load()) + } + if !f.containsLogLine("scheduler.kicked", "kind=verify", "destination=s3archive") { + t.Fatalf("missing verify kicked log:\n%s", f.logBuf.String()) + } + if !f.containsLogLine("scheduler.finished", "kind=verify", "destination=s3archive", "status=success", "run_id=7") { + t.Fatalf("missing verify finished log:\n%s", f.logBuf.String()) + } + + f.clock.Add(30 * time.Minute) + sch.tick(ctx) + if calls.Load() != 1 { + t.Fatalf("within-cadence tick verify calls = %d; want 1", calls.Load()) + } + + f.clock.Add(time.Hour) + sch.tick(ctx) + if calls.Load() != 2 { + t.Fatalf("past-cadence tick verify calls = %d; want 2", calls.Load()) + } +} + +// TestSchedulerVerifyNoOp: a destination with nothing recorded to verify +// yields an empty status + zero run id, logged distinctly as a no-op (no +// audit run was written) rather than as a run. +func TestSchedulerVerifyNoOp(t *testing.T) { + f := newSchedulerFixture(t, &config.Volume{Name: "idle"}) + sch := f.scheduler() + sch.verifyRun = func(context.Context, string) VerifyRunReport { + return VerifyRunReport{RunID: 0, Status: ""} + } + sch.verifyEvery = map[string]time.Duration{"s3archive": time.Hour} + + sch.tick(context.Background()) + if !f.containsLogLine("scheduler.finished", "kind=verify", "status=no-op", "run_id=0") { + t.Fatalf("missing no-op verify finished log:\n%s", f.logBuf.String()) + } +} + +// TestSchedulerVerifyErrorConsumesCadence: a failed verify logs an error but +// still advances the watermark, so the next within-cadence tick does not +// re-fire (no special retry — same failure policy as the volume cadences). +func TestSchedulerVerifyErrorConsumesCadence(t *testing.T) { + f := newSchedulerFixture(t, &config.Volume{Name: "idle"}) + sch := f.scheduler() + var calls atomic.Int32 + sch.verifyRun = func(context.Context, string) VerifyRunReport { + calls.Add(1) + return VerifyRunReport{RunID: 9, Status: store.RunStatusFailed, Err: errors.New("boom")} + } + sch.verifyEvery = map[string]time.Duration{"s3archive": time.Hour} + ctx := context.Background() + + sch.tick(ctx) + if calls.Load() != 1 { + t.Fatalf("tick 1 verify calls = %d; want 1", calls.Load()) + } + if !f.containsLogLine("scheduler.error", "kind=verify", "destination=s3archive", "boom") { + t.Fatalf("missing verify error log:\n%s", f.logBuf.String()) + } + f.clock.Add(10 * time.Minute) + sch.tick(ctx) + if calls.Load() != 1 { + t.Fatalf("within-cadence tick after failure verify calls = %d; want 1 (no retry)", calls.Load()) + } +} + +// TestSchedulerDurabilityPullCadence walks the pull cadence for a volume +// with a durability stake: first tick pulls, within-cadence skips, +// past-cadence re-fires. Counts ride into the finished log line. +func TestSchedulerDurabilityPullCadence(t *testing.T) { + f := newSchedulerFixture(t, &config.Volume{ + Name: "media", + OffloadRequires: []string{"s3archive"}, + }) + sch := f.scheduler() + var calls []string + sch.durabilityPull = func(_ context.Context, vol *config.Volume, peer string) DurabilityPullReport { + calls = append(calls, vol.Name+"/"+peer) + return DurabilityPullReport{RunID: 5, Status: store.RunStatusSuccess, Fetched: 3, Applied: 2} + } + sch.pullEvery = map[string]time.Duration{"nas": 24 * time.Hour} + ctx := context.Background() + + sch.tick(ctx) + if len(calls) != 1 || calls[0] != "media/nas" { + t.Fatalf("tick 1 pull calls = %v; want [media/nas]", calls) + } + if !f.containsLogLine("scheduler.kicked", "kind=pull-durability", "volume=media", "peer=nas") { + t.Fatalf("missing pull kicked log:\n%s", f.logBuf.String()) + } + if !f.containsLogLine("scheduler.finished", "kind=pull-durability", "status=success", "run_id=5", "fetched=3", "applied=2") { + t.Fatalf("missing pull finished log:\n%s", f.logBuf.String()) + } + + f.clock.Add(6 * time.Hour) + sch.tick(ctx) + if len(calls) != 1 { + t.Fatalf("within-cadence tick pull calls = %d; want 1", len(calls)) + } + + f.clock.Add(24 * time.Hour) + sch.tick(ctx) + if len(calls) != 2 { + t.Fatalf("past-cadence tick pull calls = %d; want 2", len(calls)) + } +} + +// TestSchedulerDurabilitySkipsVolumeWithoutStake: a volume that names +// neither offload_requires nor sync_to has no evidence to gain from a pull +// (every component would be dropped), so it is skipped — the reference +// htpc's playback-only photos volume, versus its offloadable media volume. +func TestSchedulerDurabilitySkipsVolumeWithoutStake(t *testing.T) { + f := newSchedulerFixture(t, &config.Volume{Name: "photos"}) + sch := f.scheduler() + var calls int + sch.durabilityPull = func(context.Context, *config.Volume, string) DurabilityPullReport { + calls++ + return DurabilityPullReport{Status: store.RunStatusSuccess} + } + sch.pullEvery = map[string]time.Duration{"nas": time.Hour} + + sch.tick(context.Background()) + if calls != 0 { + t.Fatalf("pull invoked %d times for a stakeless volume; want 0", calls) + } +} + +// TestSchedulerDurabilityPullPartialOnRewind: a puller reporting refused +// rewinds finishes 'partial' (surfaced, never applied — the agent does not +// escalate) and still advances the watermark. +func TestSchedulerDurabilityPullPartialOnRewind(t *testing.T) { + f := newSchedulerFixture(t, &config.Volume{ + Name: "media", + SyncTo: []string{"nas"}, + }) + sch := f.scheduler() + sch.durabilityPull = func(context.Context, *config.Volume, string) DurabilityPullReport { + return DurabilityPullReport{RunID: 8, Status: store.RunStatusPartial, Fetched: 2, Rewinds: 1} + } + sch.pullEvery = map[string]time.Duration{"nas": time.Hour} + + sch.tick(context.Background()) + if !f.containsLogLine("scheduler.finished", "kind=pull-durability", "status=partial", "run_id=8", "rewinds=1") { + t.Fatalf("missing partial pull finished log (with refused-rewind count):\n%s", f.logBuf.String()) + } +} + +// TestSchedulerNilCadenceRunners: when the injected runners are nil (no +// rclone wired, no pull configured) the phases are inert and touch no +// watermark — a config that declares neither cadence never trips them. +func TestSchedulerNilCadenceRunners(t *testing.T) { + f := newSchedulerFixture(t, &config.Volume{Name: "idle"}) + sch := f.scheduler() + // Cadences present but runners nil: must not panic and must do nothing. + sch.verifyEvery = map[string]time.Duration{"s3archive": time.Hour} + sch.pullEvery = map[string]time.Duration{"nas": time.Hour} + sch.verifyRun = nil + sch.durabilityPull = nil + + sch.tick(context.Background()) + if f.containsLogLine("kind=verify") || f.containsLogLine("kind=pull-durability") { + t.Fatalf("nil runners produced cadence activity:\n%s", f.logBuf.String()) + } +} diff --git a/agent/scheduler_test.go b/agent/scheduler_test.go index 9a4a8d6..0bb0f09 100644 --- a/agent/scheduler_test.go +++ b/agent/scheduler_test.go @@ -172,14 +172,23 @@ func newSchedulerFixture(t *testing.T, volumeCfg *config.Volume) *schedulerFixtu // uses. func (f *schedulerFixture) scheduler() *scheduler { return &scheduler{ - store: f.srv.store, - volumes: f.srv.cfg.Volumes, - syncRun: f.srv.cfg.SyncRunner, - logger: f.srv.cfg.Logger, - locks: f.srv.router, - tickEvery: time.Second, - now: f.clock.Now, - hooks: newHookRunner(f.srv.store, f.srv.cfg.Logger), + store: f.srv.store, + volumes: f.srv.cfg.Volumes, + syncRun: f.srv.cfg.SyncRunner, + logger: f.srv.cfg.Logger, + locks: f.srv.router, + tickEvery: time.Second, + now: f.clock.Now, + hooks: newHookRunner(f.srv.store, f.srv.cfg.Logger), + verifyRun: f.srv.cfg.VerifyRunner, + durabilityPull: f.srv.cfg.DurabilityPuller, + verifyEvery: resolveVerifyCadences(f.srv.cfg.Destinations, f.srv.cfg.VerifyEvery), + pullEvery: resolvePullCadences(f.srv.cfg.Nodes), + // Watermark maps are always initialised (unlike the other cadence + // fields, they are written) so tests that set verifyEvery/pullEvery + // directly on the returned scheduler don't have to remember to. + lastVerify: map[string]time.Time{}, + lastPull: map[string]time.Time{}, } } diff --git a/agent/serve.go b/agent/serve.go index 544336c..287568d 100644 --- a/agent/serve.go +++ b/agent/serve.go @@ -129,12 +129,13 @@ func (s *Server) startScanLoop(ctx context.Context, wg *sync.WaitGroup, logger i } // startSchedulerLoop spins up the cadence scheduler (#39) in a sibling -// goroutine, but only when at least one volume declares a sync_every -// or index_every cadence. The WaitGroup parallel to startScanLoop's +// goroutine, but only when there is scheduled work: a volume sync_every / +// index_every cadence, a destination verify cadence (F32), or a peer +// durability-pull cadence (F33). The WaitGroup parallel to startScanLoop's // usage lets Serve block on a clean exit during shutdown. func (s *Server) startSchedulerLoop(ctx context.Context, wg *sync.WaitGroup) { sched := newScheduler(s, s.cfg.SchedulerTick, s.cfg.Now) - if !sched.anyScheduledVolume() { + if !sched.anyScheduledWork() { return } wg.Add(1) diff --git a/cmd/squirrel/agent.go b/cmd/squirrel/agent.go index 02f4d8a..ebda9ee 100644 --- a/cmd/squirrel/agent.go +++ b/cmd/squirrel/agent.go @@ -54,19 +54,23 @@ func runAgent(cmd *cobra.Command) error { return err } srv, err := agent.New(agent.Config{ - Listen: cfg.Agent.Listen, - Token: cfg.Agent.Token, - PeerTokens: cfg.Agent.PeerTokens, - TLSCert: cfg.Agent.TLSCert, - TLSKey: cfg.Agent.TLSKey, - Version: version, - Volumes: cfg.Volumes, - Destinations: cfg.Destinations, - SyncRunner: buildSchedulerSyncRunner(cfg, s, rcl), - ScanInterval: cfg.Agent.ScanInterval, - ScanStrategy: cfg.Agent.ScanStrategy, - ScanLogger: cmd.ErrOrStderr(), - Logger: logger, + Listen: cfg.Agent.Listen, + Token: cfg.Agent.Token, + PeerTokens: cfg.Agent.PeerTokens, + TLSCert: cfg.Agent.TLSCert, + TLSKey: cfg.Agent.TLSKey, + Version: version, + Volumes: cfg.Volumes, + Destinations: cfg.Destinations, + Nodes: cfg.Nodes, + VerifyEvery: cfg.Agent.VerifyEvery, + SyncRunner: buildSchedulerSyncRunner(cfg, s, rcl), + VerifyRunner: buildSchedulerVerifyRunner(cfg, s, rcl), + DurabilityPuller: buildSchedulerDurabilityPuller(cfg, s), + ScanInterval: cfg.Agent.ScanInterval, + ScanStrategy: cfg.Agent.ScanStrategy, + ScanLogger: cmd.ErrOrStderr(), + Logger: logger, }, s) if err != nil { return err @@ -83,8 +87,8 @@ func serveAgent(cmd *cobra.Command, cfg *config.Config, srv *agent.Server, logge // scheduler-only agent with no cadences and no scan is silent // degradation, not a valid config. if cfg.Agent.Listen == "" { - if !agentHasWork(cfg) { - return fmt.Errorf("listener-less agent has nothing to run: set [agent] listen to receive peer syncs, or configure a cadence (a volume's sync_every, index_every, or hook.interval, or [agent] scan_interval) in %s", cfg.Path) + if !agent.ScheduledWorkInConfig(cfg) { + return fmt.Errorf("listener-less agent has nothing to run: set [agent] listen to receive peer syncs, or configure a cadence (a volume's sync_every, index_every, or hook.interval, a destination's verify_every, a node's pull_durability_every, or [agent] scan_interval or verify_every) in %s", cfg.Path) } logSchedulerOnlyStartup(logger) return srv.RunSchedulers(cmd.Context()) @@ -102,26 +106,6 @@ func serveAgent(cmd *cobra.Command, cfg *config.Config, srv *agent.Server, logge return srv.Serve(cmd.Context(), ln) } -// agentHasWork reports whether a listener-less agent has any background -// work: a drift-scan interval, or at least one volume with a scheduler -// cadence (sync, standalone index, or interval hook). It mirrors the -// agent scheduler's own "is anything scheduled" gate so the CLI refuses a -// do-nothing agent rather than letting it idle silently. -func agentHasWork(cfg *config.Config) bool { - if cfg.Agent.ScanInterval > 0 { - return true - } - for _, v := range cfg.Volumes { - if v.SyncEvery > 0 || v.IndexEvery > 0 { - return true - } - if v.Hook != nil && v.Hook.Interval > 0 { - return true - } - } - return false -} - // logSchedulerOnlyStartup emits the listener-less counterpart of the // "agent listening" banner so a journal shows the agent came up in // scheduler-only mode with no bound port. @@ -171,11 +155,13 @@ func resolveAgentDBPath(cmd *cobra.Command, cfg *config.Config) (string, error) } // resolveSchedulerRclone locates the rclone binary and writes the -// squirrel-managed rclone.conf when at least one volume declares a -// sync_every cadence with at least one destination on sync_to. Pure -// index-only schedules (or no scheduled volumes at all) skip the -// lookup so a host without rclone installed can still run the agent -// for its peer-sync surface or its index cadences. +// squirrel-managed rclone.conf when the schedule needs rclone: at least one +// volume declares a sync_every cadence with a destination on sync_to, or at +// least one verifiable destination carries a verify cadence (F32, which +// reads provider checksums through rclone). Pure index-only schedules (or a +// receive-only node whose only cadence is a durability pull) skip the lookup +// so a host without rclone installed can still run the agent for its +// peer-sync surface, its index cadences, or its durability pulls. // // The version preflight mirrors what scheduled syncs will invoke: they // run with the default sync.Options{} (Shallow=false), so `--hash blake3` @@ -184,19 +170,25 @@ func resolveAgentDBPath(cmd *cobra.Command, cfg *config.Config) (string, error) // operator gets a clear startup error rather than a midnight pager when // the first scheduled sync fires and rclone rejects the flag. func resolveSchedulerRclone(cmd *cobra.Command, cfg *config.Config) (*sync.Rclone, error) { - if !anyVolumeNeedsScheduledSync(cfg) { + needsSync := anyVolumeNeedsScheduledSync(cfg) + needsVerify := anyDestinationNeedsScheduledVerify(cfg) + if !needsSync && !needsVerify { return nil, nil } rcl, err := sync.Find() if err != nil { - return nil, fmt.Errorf("scheduler needs rclone for scheduled syncs: %w", err) + return nil, fmt.Errorf("scheduler needs rclone for scheduled syncs/verifies: %w", err) } - pairs, err := sync.PairsFor(cfg, "", "") - if err != nil { - return nil, fmt.Errorf("scheduler rclone preflight: %w", err) - } - if err := sync.EnsureMinVersion(cmd.Context(), rcl, cmd.ErrOrStderr(), sync.ShallowForPairs(pairs, false)); err != nil { - return nil, fmt.Errorf("scheduler rclone preflight: %w", err) + // The version preflight (`--hash blake3`) is a sync concern; a + // verify-only schedule reads provider checksums and doesn't need it. + if needsSync { + pairs, err := sync.PairsFor(cfg, "", "") + if err != nil { + return nil, fmt.Errorf("scheduler rclone preflight: %w", err) + } + if err := sync.EnsureMinVersion(cmd.Context(), rcl, cmd.ErrOrStderr(), sync.ShallowForPairs(pairs, false)); err != nil { + return nil, fmt.Errorf("scheduler rclone preflight: %w", err) + } } if _, err := rcl.WriteRcloneConfig(rcloneConfigPathFor(cfg), cfg.Destinations); err != nil { return nil, fmt.Errorf("write rclone config: %w", err) @@ -213,6 +205,33 @@ func anyVolumeNeedsScheduledSync(cfg *config.Config) bool { return false } +// anyDestinationNeedsScheduledVerify reports whether any verifiable +// destination has an effective verify cadence — its own verify_every, or the +// [agent] verify_every default. Mirrors the scheduler's own resolution so +// the rclone/runner wiring lines up with what the scheduler will actually +// fire. +func anyDestinationNeedsScheduledVerify(cfg *config.Config) bool { + agentDefault := cfg.Agent != nil && cfg.Agent.VerifyEvery > 0 + for _, d := range cfg.Destinations { + if d.Layout != config.LayoutContentAddressed && d.Layout != config.LayoutPacked { + continue + } + if d.VerifyEvery > 0 || agentDefault { + return true + } + } + return false +} + +func anyNodeNeedsScheduledPull(cfg *config.Config) bool { + for _, n := range cfg.Nodes { + if n.PullDurabilityEvery > 0 { + return true + } + } + return false +} + // buildSchedulerSyncRunner returns the closure the agent's cadence // scheduler invokes when it kicks a (volume, destination) sync. It // resolves the destination/node by name against the same config the @@ -258,6 +277,103 @@ func schedulerPairFor(cfg *config.Config, vol *config.Volume, destName string) ( return sync.Pair{}, fmt.Errorf("destination %q is not declared in config", destName) } +// buildSchedulerVerifyRunner returns the closure the scheduler invokes to +// verify one destination (F32). It runs sync.VerifyRemote — the same pass as +// `squirrel verify `, recording its own kind='audit' run — so +// the two surfaces share one code path. A nil rcl (no destination needs a +// verify cadence) returns a nil runner, which the scheduler treats as +// "verify-kicking disabled". +func buildSchedulerVerifyRunner(cfg *config.Config, s *store.Store, rcl *sync.Rclone) agent.VerifyRunner { + if rcl == nil || !anyDestinationNeedsScheduledVerify(cfg) { + return nil + } + return func(ctx context.Context, destName string) agent.VerifyRunReport { + dest, ok := cfg.Destinations[destName] + if !ok { + return agent.VerifyRunReport{Status: store.RunStatusFailed, Err: fmt.Errorf("destination %q is not declared in config", destName)} + } + rep, err := sync.VerifyRemote(ctx, s, rcl, dest) + return agent.VerifyRunReport{RunID: rep.RunID, Status: verifyRunStatus(rep, err), Err: err} + } +} + +// verifyRunStatus maps a VerifyRemote outcome to the status the scheduler +// logs, matching sync.recordVerifyOutcome. An empty string means "nothing +// recorded to verify" (no audit run written); the scheduler renders it as a +// no-op rather than a run. +func verifyRunStatus(rep sync.RemoteVerifyReport, err error) string { + switch { + case err != nil: + return store.RunStatusFailed + case rep.RunID == 0: + return "" + case !rep.Clean(): + return store.RunStatusPartial + default: + return store.RunStatusSuccess + } +} + +// buildSchedulerDurabilityPuller returns the closure the scheduler invokes to +// pull one peer's durability vectors for a volume (F33). It runs +// sync.PullDurability — the same metadata-only merge the CLI and the +// post-sync pull share — with allowRewind always false (the agent never +// escalates), and wraps it in a kind='audit' run so the refresh appears in +// `squirrel runs`. A config with no pull cadence returns a nil puller. +func buildSchedulerDurabilityPuller(cfg *config.Config, s *store.Store) agent.DurabilityPuller { + if !anyNodeNeedsScheduledPull(cfg) { + return nil + } + return func(ctx context.Context, vol *config.Volume, peerName string) agent.DurabilityPullReport { + node, ok := cfg.Nodes[peerName] + if !ok { + return agent.DurabilityPullReport{Status: store.RunStatusFailed, Err: fmt.Errorf("node %q is not declared in config", peerName)} + } + runID, err := s.BeginDurabilityPullRun(ctx) + if err != nil { + return agent.DurabilityPullReport{Status: store.RunStatusFailed, Err: err} + } + rep, pullErr := sync.PullDurability(ctx, s, vol, node, false) + return finishDurabilityPullRun(ctx, s, runID, vol.Name, peerName, rep, pullErr) + } +} + +// finishDurabilityPullRun records the pull's 'pull-durability' audit note and +// finishes its run, then returns the scheduler-facing report. A refused +// rewind lands as 'partial' (surfaced, but never applied — the agent does not +// escalate); a transport/merge failure as 'failed'. The pull error and any +// bookkeeping errors (audit-note append, run finish) are joined so a failure +// to record or finish the run — which would otherwise strand a 'running' row +// — always reaches the scheduler's error log rather than hiding behind an +// already-set pull error. +func finishDurabilityPullRun(ctx context.Context, s *store.Store, runID int64, volume, peer string, rep sync.DurabilityPullReport, pullErr error) agent.DurabilityPullReport { + status, errMsg := durabilityPullStatus(rep, pullErr) + note := fmt.Sprintf("volume=%s peer=%s fetched=%d applied=%d dropped=%d rewinds=%d", + volume, peer, rep.Fetched, rep.Applied, rep.Dropped, len(rep.Rewinds)) + auditErr := s.AppendRunAudit(ctx, store.RunAuditEntry{RunID: runID, Transition: store.TransitionPullDurability, Note: note}) + finErr := s.FinishRun(ctx, runID, status, errMsg, int64(rep.Applied)) + return agent.DurabilityPullReport{ + RunID: runID, + Status: status, + Err: errors.Join(pullErr, auditErr, finErr), + Fetched: rep.Fetched, Applied: rep.Applied, Dropped: rep.Dropped, Rewinds: len(rep.Rewinds), + } +} + +// durabilityPullStatus maps a pull outcome to (run status, run error +// message). Refused rewinds are the designed safe behaviour, not a failure, +// but are surfaced as 'partial' so they don't hide behind a green 'success'. +func durabilityPullStatus(rep sync.DurabilityPullReport, pullErr error) (string, string) { + switch { + case pullErr != nil: + return store.RunStatusFailed, pullErr.Error() + case len(rep.Rewinds) > 0: + return store.RunStatusPartial, fmt.Sprintf("%d component(s) refused as rewinds (not applied — the agent does not escalate)", len(rep.Rewinds)) + default: + return store.RunStatusSuccess, "" + } +} + // logAgentStartup emits a single structured startup line via slog so a // systemd unit's journal (or a developer running in the foreground) can // see what's listening where. addr is the resolved listener address (so diff --git a/cmd/squirrel/agent_scheduler_test.go b/cmd/squirrel/agent_scheduler_test.go new file mode 100644 index 0000000..f872da0 --- /dev/null +++ b/cmd/squirrel/agent_scheduler_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "errors" + "testing" + "time" + + "github.com/mbertschler/squirrel/config" + "github.com/mbertschler/squirrel/store" + "github.com/mbertschler/squirrel/sync" +) + +// TestVerifyRunStatus maps VerifyRemote outcomes to the status the scheduler +// logs, including the empty-string "nothing recorded" signal. +func TestVerifyRunStatus(t *testing.T) { + cases := []struct { + name string + rep sync.RemoteVerifyReport + err error + want string + }{ + {"error", sync.RemoteVerifyReport{RunID: 1}, errors.New("boom"), store.RunStatusFailed}, + {"no-op", sync.RemoteVerifyReport{RunID: 0}, nil, ""}, + {"clean", sync.RemoteVerifyReport{RunID: 2, Objects: 3, Verified: 3}, nil, store.RunStatusSuccess}, + {"mismatch", sync.RemoteVerifyReport{RunID: 3, Objects: 3, Missing: []string{"abcd"}}, nil, store.RunStatusPartial}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := verifyRunStatus(c.rep, c.err); got != c.want { + t.Fatalf("verifyRunStatus = %q, want %q", got, c.want) + } + }) + } +} + +// TestDurabilityPullStatus: a clean pull is success, a refused rewind is +// partial (surfaced, never applied), a transport error is failed. +func TestDurabilityPullStatus(t *testing.T) { + if got, _ := durabilityPullStatus(sync.DurabilityPullReport{Applied: 2}, nil); got != store.RunStatusSuccess { + t.Fatalf("clean pull status = %q, want success", got) + } + if got, msg := durabilityPullStatus(sync.DurabilityPullReport{Rewinds: []sync.DurabilityRewind{{}}}, nil); got != store.RunStatusPartial || msg == "" { + t.Fatalf("rewind pull = (%q,%q), want partial with a message", got, msg) + } + if got, msg := durabilityPullStatus(sync.DurabilityPullReport{}, errors.New("timeout")); got != store.RunStatusFailed || msg != "timeout" { + t.Fatalf("errored pull = (%q,%q), want (failed,timeout)", got, msg) + } +} + +// TestAnyDestinationNeedsScheduledVerify covers the wiring predicate: a +// per-destination cadence or an [agent] default on a verifiable layout +// trips it; a mirror layout never does. +func TestAnyDestinationNeedsScheduledVerify(t *testing.T) { + base := func() *config.Config { + return &config.Config{ + Agent: &config.Agent{}, + Destinations: map[string]*config.Destination{}, + } + } + + cfg := base() + cfg.Destinations["m"] = &config.Destination{Layout: config.LayoutMirror, VerifyEvery: time.Hour} + if anyDestinationNeedsScheduledVerify(cfg) { + t.Fatalf("mirror layout must never need scheduled verify") + } + + cfg = base() + cfg.Destinations["p"] = &config.Destination{Layout: config.LayoutPacked, VerifyEvery: time.Hour} + if !anyDestinationNeedsScheduledVerify(cfg) { + t.Fatalf("packed destination with own cadence should need verify") + } + + cfg = base() + cfg.Agent.VerifyEvery = time.Hour + cfg.Destinations["c"] = &config.Destination{Layout: config.LayoutContentAddressed} + if !anyDestinationNeedsScheduledVerify(cfg) { + t.Fatalf("agent default should cover a verifiable destination with no own cadence") + } +} + +// TestAnyNodeNeedsScheduledPull: only a node with pull_durability_every trips. +func TestAnyNodeNeedsScheduledPull(t *testing.T) { + cfg := &config.Config{Nodes: map[string]*config.Node{ + "a": {}, + }} + if anyNodeNeedsScheduledPull(cfg) { + t.Fatalf("no node has a pull cadence") + } + cfg.Nodes["b"] = &config.Node{PullDurabilityEvery: time.Hour} + if !anyNodeNeedsScheduledPull(cfg) { + t.Fatalf("node b has a pull cadence") + } +} diff --git a/cmd/squirrel/offload.go b/cmd/squirrel/offload.go index 2f62c3b..56b31e3 100644 --- a/cmd/squirrel/offload.go +++ b/cmd/squirrel/offload.go @@ -76,6 +76,7 @@ func runOffload(cmd *cobra.Command, volumeName string, paths []string, olderThan RequireDests: requiredDestinations(cfg, vol.OffloadRequires), RelayedCaps: relayedCaps, MaxEvidenceAge: vol.OffloadMaxEvidenceAge, + VerifyCadenced: verifyCadencedTargets(cfg, vol.OffloadRequires), DryRun: dryRun, }) printOffloadReport(cmd.OutOrStdout(), cmd.ErrOrStderr(), rep, dryRun) @@ -104,6 +105,40 @@ func requiredDestinations(cfg *config.Config, require []string) map[string]*conf return out } +// verifyCadencedTargets marks the required targets that carry an effective +// local verify cadence — a per-destination verify_every, or the [agent] +// verify_every default applied to a content-addressed/packed destination. +// The offload gate accepts a locally-advanced fingerprint-verified +// component as content-verified only for a target in this set, so the +// provider-fingerprint evidence keeps being re-confirmed for as long as +// deletions are permitted (issue #155). A required target with no local +// destination — a peer-relayed offsite this node cannot see — is omitted; +// the responder's cadence governs it, relayed and enforced at pull time. +func verifyCadencedTargets(cfg *config.Config, require []string) map[string]bool { + var agentDefault time.Duration + if cfg.Agent != nil { + agentDefault = cfg.Agent.VerifyEvery + } + out := make(map[string]bool, len(require)) + for _, name := range require { + d, ok := cfg.Destinations[name] + if !ok { + continue + } + if d.Layout != config.LayoutContentAddressed && d.Layout != config.LayoutPacked { + continue + } + eff := d.VerifyEvery + if eff == 0 { + eff = agentDefault + } + if eff > 0 { + out[name] = true + } + } + return out +} + // gatherRelayedOffloadCaps does the best-effort peer half of the offload // capability pre-check (#145): for each peer-relayed required target (an // offload_requires name that is neither a local destination nor a local diff --git a/config/agent.go b/config/agent.go index 559faa0..d98d32b 100644 --- a/config/agent.go +++ b/config/agent.go @@ -52,6 +52,14 @@ type Agent struct { // match the stored row; ScanStrategyDeep re-hashes everything // unconditionally — the bit-rot-detection schedule. ScanStrategy string + // VerifyEvery is the default cadence the agent re-verifies every + // content-addressed/packed destination on when the destination itself + // declares no `verify_every` — the offsite fingerprint re-check that + // otherwise happens only when `squirrel verify` is typed. Zero (the + // default) means no fleet-wide verify cadence; a per-destination + // verify_every still applies. Verify is read-only, so the agent never + // escalates or bootstraps a destination on this cadence. + VerifyEvery time.Duration } // rawAgent mirrors the `[agent]` TOML block. We use typed sub-structs @@ -64,6 +72,7 @@ type rawAgent struct { Auth *rawAuth `toml:"auth"` ScanInterval string `toml:"scan_interval"` ScanStrategy string `toml:"scan_strategy"` + VerifyEvery string `toml:"verify_every"` } type rawTLS struct { @@ -106,6 +115,13 @@ func resolveAgent(r *rawAgent) (*Agent, error) { if err := resolveAgentScan(r, a); err != nil { return nil, err } + if r.VerifyEvery != "" { + dur, err := parseVolumeCadence("verify_every", r.VerifyEvery) + if err != nil { + return nil, err + } + a.VerifyEvery = dur + } return a, nil } diff --git a/config/config.go b/config/config.go index 7ae0c1e..e7dd7e5 100644 --- a/config/config.go +++ b/config/config.go @@ -200,6 +200,15 @@ type Destination struct { // LayoutPacked destinations, where it defaults to 3; zero on every // other layout. ZstdLevel int + // VerifyEvery is the agent-scheduler cadence for re-checking this + // destination's recorded objects and packs against their upload + // fingerprints — the same pass as `squirrel verify`, recorded as an + // `audit` run. Meaningful only on the content-addressed and packed + // layouts that keep per-artifact fingerprints; rejected at load on any + // other layout. Zero means "no per-destination verify cadence"; an + // [agent] verify_every default may still apply. Verify is read-only — + // the agent never writes to or bootstraps the destination. + VerifyEvery time.Duration } // Destination layout values. The layout shapes what sync writes under diff --git a/config/config_test.go b/config/config_test.go index 8ca751a..a46b56e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1609,3 +1609,147 @@ password = "obscured" t.Fatalf("crypt remote must include the bucket, got:\n%s", section) } } + +// TestLoadDestinationVerifyEvery parses a per-destination verify_every on a +// content-addressed / packed destination. +func TestLoadDestinationVerifyEvery(t *testing.T) { + t.Setenv("K", "v") + p := writeConfig(t, ` +[destinations.arch] +type = "s3" +provider = "AWS" +bucket = "bkt" +root = "/" +layout = "packed" +verify_every = "168h" +access_key_id = { env = "K" } +secret_access_key = { env = "K" } +`) + cfg, err := Load(p) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Destinations["arch"].VerifyEvery != 168*time.Hour { + t.Fatalf("VerifyEvery = %s, want 168h", cfg.Destinations["arch"].VerifyEvery) + } +} + +// TestLoadRejectsVerifyEveryOnMirror rejects verify_every on a layout that +// keeps no per-object fingerprints — verify has nothing to re-check there. +func TestLoadRejectsVerifyEveryOnMirror(t *testing.T) { + p := writeConfig(t, ` +[destinations.usb] +type = "local" +root = "/media/usb" +verify_every = "168h" +`) + _, err := Load(p) + if err == nil || !strings.Contains(err.Error(), "verify_every requires") { + t.Fatalf("expected verify_every-layout error, got %v", err) + } +} + +// TestLoadRejectsBadVerifyEvery: an unparseable duration is caught at load, +// like the other cadence knobs. +func TestLoadRejectsBadVerifyEvery(t *testing.T) { + t.Setenv("K", "v") + p := writeConfig(t, ` +[destinations.arch] +type = "s3" +provider = "AWS" +bucket = "bkt" +root = "/" +layout = "content-addressed" +verify_every = "nope" +access_key_id = { env = "K" } +secret_access_key = { env = "K" } +`) + _, err := Load(p) + if err == nil || !strings.Contains(err.Error(), "verify_every") { + t.Fatalf("expected verify_every parse error, got %v", err) + } +} + +// TestLoadAgentVerifyEvery parses the fleet-wide default verify cadence. +func TestLoadAgentVerifyEvery(t *testing.T) { + p := writeConfig(t, ` +[agent] +listen = "127.0.0.1:9000" +auth = { token = "t" } +verify_every = "336h" +`) + cfg, err := Load(p) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Agent.VerifyEvery != 336*time.Hour { + t.Fatalf("Agent.VerifyEvery = %s, want 336h", cfg.Agent.VerifyEvery) + } +} + +// TestLoadAgentVerifyEveryDefaultsZero: absent key leaves the default off. +func TestLoadAgentVerifyEveryDefaultsZero(t *testing.T) { + p := writeConfig(t, ` +[agent] +listen = "127.0.0.1:9000" +auth = { token = "t" } +`) + cfg, err := Load(p) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Agent.VerifyEvery != 0 { + t.Fatalf("default Agent.VerifyEvery = %s, want 0", cfg.Agent.VerifyEvery) + } +} + +// TestLoadNodePullDurabilityEvery parses the per-node pull cadence — the +// receive-only htpc's clock for keeping its gate evidence fresh. +func TestLoadNodePullDurabilityEvery(t *testing.T) { + p := writeConfig(t, ` +[nodes.nas] +endpoint = "https://nas.local:8443" +path = "/mnt/nas-export" +pull_durability_every = "24h" +auth = { bearer = "b" } +`) + cfg, err := Load(p) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Nodes["nas"].PullDurabilityEvery != 24*time.Hour { + t.Fatalf("PullDurabilityEvery = %s, want 24h", cfg.Nodes["nas"].PullDurabilityEvery) + } +} + +// TestLoadNodePullDurabilityEveryDefaultsZero: absent key leaves it off. +func TestLoadNodePullDurabilityEveryDefaultsZero(t *testing.T) { + p := writeConfig(t, ` +[nodes.nas] +endpoint = "https://nas.local:8443" +path = "/mnt/nas-export" +auth = { bearer = "b" } +`) + cfg, err := Load(p) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.Nodes["nas"].PullDurabilityEvery != 0 { + t.Fatalf("default PullDurabilityEvery = %s, want 0", cfg.Nodes["nas"].PullDurabilityEvery) + } +} + +// TestLoadRejectsBadPullDurabilityEvery: an unparseable duration is caught. +func TestLoadRejectsBadPullDurabilityEvery(t *testing.T) { + p := writeConfig(t, ` +[nodes.nas] +endpoint = "https://nas.local:8443" +path = "/mnt/nas-export" +pull_durability_every = "soon" +auth = { bearer = "b" } +`) + _, err := Load(p) + if err == nil || !strings.Contains(err.Error(), "pull_durability_every") { + t.Fatalf("expected pull_durability_every error, got %v", err) + } +} diff --git a/config/destinations.go b/config/destinations.go index d772593..573abbb 100644 --- a/config/destinations.go +++ b/config/destinations.go @@ -8,6 +8,7 @@ import ( "path" "sort" "strings" + "time" "github.com/dustin/go-humanize" ) @@ -173,6 +174,10 @@ func resolveDestination(name string, raw map[string]any) (*Destination, error) { if err != nil { return nil, err } + verifyEvery, err := resolveVerifyEvery(raw, layout) + if err != nil { + return nil, err + } params, err := validateAndResolveParams(schema, raw) if err != nil { return nil, err @@ -181,9 +186,32 @@ func resolveDestination(name string, raw map[string]any) (*Destination, error) { Name: name, Type: typ, Root: root, Layout: layout, Params: params, Crypt: crypt, HashAlgo: hashAlgo, Checkers: checkers, PathStyle: pathStyle, PackThreshold: pack.threshold, PackSize: pack.size, ZstdLevel: pack.zstdLevel, + VerifyEvery: verifyEvery, }, nil } +// resolveVerifyEvery validates the optional per-destination `verify_every` +// cadence that drives the agent's scheduled re-check of this destination's +// recorded objects and packs (the same pass as `squirrel verify`). It is +// meaningful only on the content-addressed and packed layouts that keep +// per-artifact fingerprints, so a present key on any other layout is +// rejected rather than silently ignored — a mirror destination has nothing +// for verify to re-check. Empty stays zero: no per-destination cadence, an +// [agent] verify_every default may still apply. +func resolveVerifyEvery(raw map[string]any, layout string) (time.Duration, error) { + v, err := optionalString(raw, "verify_every") + if err != nil { + return 0, err + } + if v == "" { + return 0, nil + } + if layout != LayoutContentAddressed && layout != LayoutPacked { + return 0, fmt.Errorf("verify_every requires the %q or %q layout; layout %q keeps no per-object fingerprints to re-check", LayoutContentAddressed, LayoutPacked, layout) + } + return parseVolumeCadence("verify_every", v) +} + // sftpHashAlgos are the checksum types rclone's sftp backend can read // via a server-side sum command, the valid values for `hash_algo`. var sftpHashAlgos = map[string]bool{ @@ -505,6 +533,7 @@ func validateAndResolveParams(schema destSchema, raw map[string]any) (map[string "type": true, "root": true, "crypt": true, "layout": true, "hash_algo": true, "checkers": true, "force_path_style": true, "pack_threshold": true, "pack_size": true, "zstd_level": true, + "verify_every": true, } for _, key := range schema.requiredString { v, err := requireString(raw, key) diff --git a/config/nodes.go b/config/nodes.go index 9452ed4..63e9f87 100644 --- a/config/nodes.go +++ b/config/nodes.go @@ -6,6 +6,7 @@ import ( "net/url" "regexp" "strings" + "time" "github.com/mbertschler/squirrel/syncproto" ) @@ -38,6 +39,15 @@ type Node struct { // volume) or "off" (always Transfer). The literal travels in the // /v1/sync/begin payload; the receiver validates and applies it. DedupStrategy string + // PullDurabilityEvery is the agent-scheduler cadence for pulling this + // peer's destination durability vectors — the same metadata-only merge + // that `squirrel peer-sync pull-durability` runs and that rides along + // after a successful node sync. Giving evidence freshness its own clock + // lets a receive-only node (one this machine never initiates a sync to) + // keep its offload-gate evidence fresh with zero typed commands. Zero + // means "no scheduled pull". The pull never rewinds a watermark — the + // agent does not escalate. + PullDurabilityEvery time.Duration } // rawNode mirrors the `[nodes.X]` TOML block. Token is `any` so the @@ -45,11 +55,12 @@ type Node struct { // it transparently — accepting either a literal string or // `{ env = "VAR" }`. type rawNode struct { - Endpoint string `toml:"endpoint"` - Path string `toml:"path"` - DedupStrategy string `toml:"dedup_strategy"` - Auth *rawNodeAuth `toml:"auth"` - TLS *rawNodeTLS `toml:"tls"` + Endpoint string `toml:"endpoint"` + Path string `toml:"path"` + DedupStrategy string `toml:"dedup_strategy"` + PullDurabilityEvery string `toml:"pull_durability_every"` + Auth *rawNodeAuth `toml:"auth"` + TLS *rawNodeTLS `toml:"tls"` } type rawNodeAuth struct { @@ -116,12 +127,20 @@ func resolveNode(name string, r *rawNode) (*Node, error) { if err != nil { return nil, err } + var pullEvery time.Duration + if r.PullDurabilityEvery != "" { + pullEvery, err = parseVolumeCadence("pull_durability_every", r.PullDurabilityEvery) + if err != nil { + return nil, err + } + } node := &Node{ - Name: name, - Endpoint: u, - Token: tok, - Path: r.Path, - DedupStrategy: strategy, + Name: name, + Endpoint: u, + Token: tok, + Path: r.Path, + DedupStrategy: strategy, + PullDurabilityEvery: pullEvery, } if r.TLS != nil && r.TLS.CertFingerprint != "" { fp := strings.ToLower(r.TLS.CertFingerprint) diff --git a/design/reference-setup.md b/design/reference-setup.md index 85f5ebf..60f8112 100644 --- a/design/reference-setup.md +++ b/design/reference-setup.md @@ -124,6 +124,9 @@ node_name = "nas" listen = "0.0.0.0:8443" scan_interval = "168h" # weekly drift scan over all volumes scan_strategy = "shallow" +verify_every = "168h" # weekly offsite fingerprint re-check (F32), + # the agent-level default for every packed/CA + # destination (here: s3archive) [agent.tls] cert = "/volume1/squirrel/agent.crt" # self-signed; peers pin the fingerprint key = "/volume1/squirrel/agent.key" @@ -262,8 +265,12 @@ offload_max_evidence_age = "720h" path = "/data/photos" [nodes.nas] # not in any sync_to: exists so the -endpoint = "https://nas.home:8443" # htpc can *pull durability* from nas -path = "/mnt/nas-export" +endpoint = "https://nas.home:8443" # htpc can *pull durability* +path = "/mnt/nas-export" # from nas … +pull_durability_every = "24h" # … on its own clock (F33), so the + # media offload gate's relayed + # s3archive evidence stays fresh with + # zero typed commands [nodes.nas.auth] bearer = { env = "SQUIRREL_PEER_HTPC" } [nodes.nas.tls] @@ -281,8 +288,8 @@ bootstrap the household should run itself. Today's coverage: | Drift detection (`scan_interval`) | ✅ agent, hub | | Index snapshots local + ride-along | ✅ on every successful sync | | Durability pull after node sync | ✅ initiator side only | -| Offsite fingerprint re-check (`squirrel verify`) | ❌ manual only ([F32](friction-log.md)) | -| Durability refresh on a *receive-only* node (htpc) | ❌ never fires ([F33](friction-log.md)) | +| Offsite fingerprint re-check (`verify_every`) | ✅ agent, per-destination or an `[agent]` default (F32) | +| Durability refresh on a *receive-only* node (htpc) | ✅ agent, per-node `pull_durability_every` (F33) | | Offload | ❌ manual by design; the *readiness signal* should still be automatic ([F17](friction-log.md)) | ## Lifecycle checkpoints diff --git a/docs/src/content/docs/guides/agent.md b/docs/src/content/docs/guides/agent.md index 594cf78..794c558 100644 --- a/docs/src/content/docs/guides/agent.md +++ b/docs/src/content/docs/guides/agent.md @@ -20,6 +20,15 @@ The agent requires an `[agent]` block in config. (`index_every` / `sync_every`). - **Scheduled audits** — runs [`audit`](/squirrel/guides/auditing/) passes on a schedule. +- **Scheduled verify** — re-checks each content-addressed/packed destination's + recorded objects and packs against their upload fingerprints on its + `verify_every` cadence (per-destination, or an `[agent]` default) — the same + pass as [`squirrel verify`](/squirrel/guides/verification/), recorded as an + `audit` run. Offsite bitrot detection stops depending on anyone typing it. +- **Scheduled durability pull** — refreshes a peer's relayed durability + evidence on its `pull_durability_every` cadence, independent of any sync. A + receive-only node keeps its [offload](/squirrel/guides/offloading/) gate + evidence fresh unattended. - **Hook firing** — fires per-volume [hooks](/squirrel/guides/hooks/) on change (after each successful index run) and on their `interval`. - **HTTP server** — exposes a health endpoint, serves peer syncs, and drives diff --git a/docs/src/content/docs/reference/configuration.md b/docs/src/content/docs/reference/configuration.md index 3fec920..30860cb 100644 --- a/docs/src/content/docs/reference/configuration.md +++ b/docs/src/content/docs/reference/configuration.md @@ -116,6 +116,7 @@ For `layout = "content-addressed"` or `layout = "packed"` destinations. See | `pack_threshold` | packed | — | Files smaller than this are packed; at/above land as objects (e.g. `1MiB`). | | `pack_size` | packed | — | Target size of one pack before it is closed (e.g. `512MiB`). | | `zstd_level` | packed | — | zstd compression level, `1` fastest … `4` best. | +| `verify_every` | content-addressed / packed | off | Cadence for the agent to re-check this destination's recorded objects and packs against their upload fingerprints — the same pass as [`squirrel verify`](/squirrel/guides/verification/), recorded as an `audit` run. Falls back to `[agent] verify_every`. Read-only; the agent never writes to the destination. | ## `[backups]` @@ -142,3 +143,25 @@ server for peer syncs and the health endpoint, and a bearer token is required. When omitted, the agent runs its schedulers only — the *listener-less* mode for cadence-only machines that never receive peer syncs; `[agent.auth]` is then optional. See [The agent](/squirrel/guides/agent/#listener-less-cadence-only-machines). + +| Key | Required | Meaning | +|---|---|---| +| `listen` | no | Bind address, e.g. `0.0.0.0:8443`. Omit for listener-less, scheduler-only mode (above). | +| `db` | no | Agent-specific index path; wins over the top-level `db`. | +| `auth.token` | with `listen` | Shared bearer token. Required only when the agent binds an HTTP surface. | +| `scan_interval` | no | Drift-scan cadence over every hosted volume. Off when absent. | +| `scan_strategy` | no | `shallow` (default) or `deep` (re-hash everything — bit-rot detection). | +| `verify_every` | no | Fleet-wide default verify cadence applied to every content-addressed/packed destination that declares no `verify_every` of its own. Off when absent. | + +## `[nodes.]` + +A peer that runs its own agent. See [Peer sync](/squirrel/guides/peer-sync/). + +| Key | Required | Meaning | +|---|---|---| +| `endpoint` | yes | Peer's sync-API URL, e.g. `https://nas.home:8443`. | +| `path` | yes | rclone target prefix bytes are copied into. | +| `auth.bearer` | yes | Bearer token this node presents to the peer. | +| `tls.cert_fingerprint` | no | `sha256:` pin for the peer's self-signed cert. | +| `dedup_strategy` | no | `copy` (default) or `off`. | +| `pull_durability_every` | no | Cadence for the agent to pull this peer's durability vectors (the same merge as [`squirrel peer-sync pull-durability`](/squirrel/guides/peer-sync/)), giving evidence freshness its own clock. Lets a **receive-only** node (one this machine never syncs *to*) keep its offload-gate evidence fresh unattended. The agent never rewinds a watermark. Off when absent. | diff --git a/offload/durability_soundness_test.go b/offload/durability_soundness_test.go index edb302c..74444cc 100644 --- a/offload/durability_soundness_test.go +++ b/offload/durability_soundness_test.go @@ -514,3 +514,142 @@ func TestOffloadDurableFileStillPasses(t *testing.T) { oneResult(t, rep, "a.txt", OutcomeOffloaded) oneResult(t, rep, "sub/b.txt", OutcomeOffloaded) } + +// TestOffloadFingerprintVerifiedRelayedGates is the crown-jewel relayed +// path (issue #155): a peer-asserted fingerprint-verified component gates. +// It exists only because the responder relayed a positive verify cadence +// (the pull bakes it to presence+size otherwise), so the puller — which +// holds no local fingerprint of the object — may treat the relayed +// evidence as content-verified. +func TestOffloadFingerprintVerifiedRelayedGates(t *testing.T) { + const target = "s3archive" + root := t.TempDir() + writeFile(t, filepath.Join(root, "a.txt"), "alpha") + s := setupStore(t) + ctx := context.Background() + idx := indexVolume(t, s, root) + v := testVolume(t, s) + self := selfNode(t, s) + + peer, err := s.GetOrCreateOriginNode(ctx, "nas") + if err != nil { + t.Fatalf("GetOrCreateOriginNode: %v", err) + } + if err := s.UpsertDestinationRunIDPulled(ctx, v.ID, target, self.ID, idx.RunID, store.VerifyMethodFingerprint, peer.ID, store.NowNs(), false); err != nil { + t.Fatalf("UpsertDestinationRunIDPulled: %v", err) + } + seedRelayedFreshness(t, s, v.ID, target, self.ID, idx.RunID) + + rep, err := Offload(ctx, s, root, Options{Name: volName, Paths: []string{"."}, Require: []string{target}}) + if err != nil { + t.Fatalf("Offload: %v", err) + } + oneResult(t, rep, "a.txt", OutcomeOffloaded) + mustBeGone(t, filepath.Join(root, "a.txt")) +} + +// TestOffloadFingerprintVerifiedRelayedBakedDownRefused is the fail-closed +// direction: when the responder relays no verify cadence the pull stores +// the component as presence+size (see relayedMethod), and the puller has no +// local fingerprint to back it — so the gate refuses (issue #155). +func TestOffloadFingerprintVerifiedRelayedBakedDownRefused(t *testing.T) { + const target = "s3archive" + root := t.TempDir() + writeFile(t, filepath.Join(root, "a.txt"), "alpha") + s := setupStore(t) + ctx := context.Background() + idx := indexVolume(t, s, root) + v := testVolume(t, s) + self := selfNode(t, s) + + peer, err := s.GetOrCreateOriginNode(ctx, "nas") + if err != nil { + t.Fatalf("GetOrCreateOriginNode: %v", err) + } + if err := s.UpsertDestinationRunIDPulled(ctx, v.ID, target, self.ID, idx.RunID, store.VerifyMethodPresenceSize, peer.ID, store.NowNs(), false); err != nil { + t.Fatalf("UpsertDestinationRunIDPulled: %v", err) + } + seedRelayedFreshness(t, s, v.ID, target, self.ID, idx.RunID) + + rep, err := Offload(ctx, s, root, Options{Name: volName, Paths: []string{"."}, Require: []string{target}}) + if err != nil { + t.Fatalf("Offload: %v", err) + } + res := oneResult(t, rep, "a.txt", OutcomeNotDurable) + if len(res.Reasons) != 1 || !strings.Contains(res.Reasons[0], "not content-verified") { + t.Fatalf("reasons = %v, want a not-content-verified refusal", res.Reasons) + } + mustExist(t, filepath.Join(root, "a.txt")) +} + +// TestOffloadFingerprintVerifiedLocalCadenceCoupling covers a locally- +// advanced fingerprint-verified component: it gates when the destination +// carries a verify cadence, falls back to a locally-held object fingerprint +// when it does not (the node re-reads its own objects), and refuses when +// neither holds (issue #155). +func TestOffloadFingerprintVerifiedLocalCadenceCoupling(t *testing.T) { + const target = "arch" + seed := func(t *testing.T) (*store.Store, string, store.FileRow, int64) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "a.txt"), "alpha") + s := setupStore(t) + ctx := context.Background() + idx := indexVolume(t, s, root) + v := testVolume(t, s) + self := selfNode(t, s) + if err := s.UpsertDestinationRunIDVerified(ctx, v.ID, target, self.ID, idx.RunID, store.VerifyMethodFingerprint, false); err != nil { + t.Fatalf("UpsertDestinationRunIDVerified: %v", err) + } + pushRun := recordPush(t, s, v.ID, target) + return s, root, rowAt(t, s, v.ID, "a.txt"), pushRun + } + + t.Run("cadence accepts", func(t *testing.T) { + s, root, _, _ := seed(t) + rep, err := Offload(context.Background(), s, root, Options{ + Name: volName, Paths: []string{"."}, Require: []string{target}, + VerifyCadenced: map[string]bool{target: true}, + }) + if err != nil { + t.Fatalf("Offload: %v", err) + } + oneResult(t, rep, "a.txt", OutcomeOffloaded) + mustBeGone(t, filepath.Join(root, "a.txt")) + }) + + t.Run("no cadence, no local fingerprint refuses", func(t *testing.T) { + s, root, _, _ := seed(t) + rep, err := Offload(context.Background(), s, root, Options{ + Name: volName, Paths: []string{"."}, Require: []string{target}, + }) + if err != nil { + t.Fatalf("Offload: %v", err) + } + res := oneResult(t, rep, "a.txt", OutcomeNotDurable) + if len(res.Reasons) != 1 || !strings.Contains(res.Reasons[0], "not content-verified") { + t.Fatalf("reasons = %v, want a not-content-verified refusal", res.Reasons) + } + mustExist(t, filepath.Join(root, "a.txt")) + }) + + t.Run("no cadence, local fingerprint falls back", func(t *testing.T) { + s, root, row, pushRun := seed(t) + ctx := context.Background() + if err := s.InsertRemoteObject(ctx, store.RemoteObject{ + ContentID: row.ContentID, Destination: target, UploadedRunID: pushRun, + }); err != nil { + t.Fatalf("InsertRemoteObject: %v", err) + } + if err := s.SetRemoteObjectFingerprint(ctx, row.ContentID, target, "sftp-sha256", "abc", store.NowNs()); err != nil { + t.Fatalf("SetRemoteObjectFingerprint: %v", err) + } + rep, err := Offload(ctx, s, root, Options{ + Name: volName, Paths: []string{"."}, Require: []string{target}, + }) + if err != nil { + t.Fatalf("Offload: %v", err) + } + oneResult(t, rep, "a.txt", OutcomeOffloaded) + mustBeGone(t, filepath.Join(root, "a.txt")) + }) +} diff --git a/offload/gate.go b/offload/gate.go index fd71101..c4026b6 100644 --- a/offload/gate.go +++ b/offload/gate.go @@ -55,9 +55,17 @@ type gate struct { // Zero disables the time-based policy — the opt-in default, so the // version-vector gate behaves exactly as before. maxEvidenceAge time.Duration + // cadenced marks the required targets that carry an effective local + // verify cadence (a per-destination verify_every, or the [agent] + // default). A locally-verified fingerprint-verified component gates + // only while its destination is re-confirmed on a cadence; a target + // this node cannot see (a relayed offsite) is absent from the map, and + // the relayed cadence is enforced at pull time instead. Nil is a valid + // empty set — no target is treated as cadenced. + cadenced map[string]bool } -func loadGate(ctx context.Context, s *store.Store, volumeID int64, require []string, nowNs int64, maxEvidenceAge time.Duration) (*gate, error) { +func loadGate(ctx context.Context, s *store.Store, volumeID int64, require []string, nowNs int64, maxEvidenceAge time.Duration, cadenced map[string]bool) (*gate, error) { self, err := s.GetSelfNode(ctx) if err != nil { return nil, fmt.Errorf("lookup self node: %w", err) @@ -73,6 +81,7 @@ func loadGate(ctx context.Context, s *store.Store, volumeID int64, require []str nodeNames: map[int64]string{self.ID: self.Name}, nowNs: nowNs, maxEvidenceAge: maxEvidenceAge, + cadenced: cadenced, } for _, target := range require { components, err := s.ListDestinationRunIDs(ctx, volumeID, target) @@ -249,15 +258,45 @@ func (g *gate) freshnessFailure(ctx context.Context, target string, row store.Fi // passes only once a verified scan-back fingerprint backs the gated content // — via either layout: a remote_objects row (per-hash object) or a // remote_packs row for the content's pack, each carrying a checksum and a -// verified_at_ns for this destination. Any other method (including a -// size+mtime push or an unknown/pre-v19 component) does not gate. +// verified_at_ns for this destination. +// +// A fingerprint-verified component (the upgrade of a presence+size vector +// once the whole pair carries verified fingerprints) is content-verified +// only while a scheduled verify cadence keeps that evidence fresh: +// +// - relayed (a durability pull tagged it): acceptance was decided at pull +// time, where a fingerprint-verified method survives only when the +// responder relayed a positive verify cadence for the destination +// (else it is baked down to presence+size, which then needs a local +// fingerprint this node does not hold). So a relayed fingerprint- +// verified component passes here directly. +// - local (this node advanced it): it passes while the destination has a +// live verify cadence; without one it falls back to the per-content +// scan-back check, since the node holds the fingerprints itself and +// re-reads them on every verify pass — the cadence coupling binds the +// relayed path, where the evidence cannot be re-read locally. +// +// Any other method (a size+mtime push or an unknown/pre-v19 component) +// does not gate. func (g *gate) methodVerified(ctx context.Context, target string, comp component, row store.FileRow) (bool, error) { if store.ContentVerifiedMethod(comp.method) { return true, nil } + if comp.method == store.VerifyMethodFingerprint { + if comp.source.Valid || g.cadenced[target] { + return true, nil + } + return g.contentFingerprintVerified(ctx, target, row) + } if comp.method != store.VerifyMethodPresenceSize { return false, nil } + return g.contentFingerprintVerified(ctx, target, row) +} + +// contentFingerprintVerified reports whether a verified provider +// fingerprint backs the row's content on the target (either layout). +func (g *gate) contentFingerprintVerified(ctx context.Context, target string, row store.FileRow) (bool, error) { verified, err := g.store.ContentFingerprintVerified(ctx, row.ContentID, target) if err != nil { return false, fmt.Errorf("load fingerprint for content %d on %q: %w", row.ContentID, target, err) diff --git a/offload/offload.go b/offload/offload.go index 7f4ac4f..d3a8e04 100644 --- a/offload/offload.go +++ b/offload/offload.go @@ -69,6 +69,15 @@ type Options struct { // staleness policy — the opt-in default, so the version-vector gate // behaves exactly as before. MaxEvidenceAge time.Duration + // VerifyCadenced marks the required targets that carry an effective + // local verify cadence (a per-destination verify_every, or the [agent] + // default). The gate accepts a locally-advanced fingerprint-verified + // component as content-verified only while its destination is on such a + // cadence, so the provider-fingerprint evidence keeps being re-confirmed + // for as long as deletions are permitted (issue #155). A relayed + // target this node cannot see is absent — the responder's cadence is + // enforced at pull time. Nil is a valid empty set. + VerifyCadenced map[string]bool // DryRun evaluates and reports the per-file gate decisions from the // index alone: no runs row, no file reads, no deletions, no status // flips. Disk-drift checks only happen on a real run, immediately @@ -161,7 +170,7 @@ func Offload(ctx context.Context, s *store.Store, root string, opts Options) (re return Report{}, err } now := time.Now() - g, err := loadGate(ctx, s, vol.ID, opts.Require, now.UnixNano(), opts.MaxEvidenceAge) + g, err := loadGate(ctx, s, vol.ID, opts.Require, now.UnixNano(), opts.MaxEvidenceAge, opts.VerifyCadenced) if err != nil { return Report{}, err } diff --git a/store/content_presence.go b/store/content_presence.go index 9d6c5a0..e4e7035 100644 --- a/store/content_presence.go +++ b/store/content_presence.go @@ -31,6 +31,49 @@ func (s *Store) ContentPresentOnDestination(ctx context.Context, contentID int64 return present != 0, nil } +// CountVolumeContentsPendingFingerprint counts the distinct present +// contents of the volume that do NOT yet carry a verified provider +// fingerprint on the destination — the whole-state "pending artifact" +// tally the fingerprint-verified upgrade gates on. A content counts as +// pending unless it has either a remote_objects row or a member pack's +// remote_packs row on this destination with a non-NULL checksum and +// verified_at_ns, so content not yet uploaded, uploaded but not yet +// fingerprinted, or fingerprinted but not yet re-confirmed all keep the +// tally above zero. Zero means every present content is fingerprint- +// verified on the destination, so the vector may advance to +// VerifyMethodFingerprint over the whole present set. The reserved sync +// subtrees are excluded, matching PresentOriginMaxima — they never travel +// to a destination. +func (s *Store) CountVolumeContentsPendingFingerprint(ctx context.Context, volumeID int64, destination string) (int, error) { + var pending int + err := s.db.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM ( + SELECT DISTINCT f.content_id + FROM files f + JOIN folders fo ON fo.id = f.folder_id + WHERE fo.volume_id = ? AND f.status = 'present' + AND `+reservedSubtreeFilter+` + AND NOT ( + EXISTS ( + SELECT 1 FROM remote_objects ro + WHERE ro.content_id = f.content_id AND ro.destination = ? + AND ro.checksum IS NOT NULL AND ro.verified_at_ns IS NOT NULL + ) + OR EXISTS ( + SELECT 1 FROM pack_members pm + JOIN remote_packs rp ON rp.pack_id = pm.pack_id + WHERE pm.content_id = f.content_id AND rp.destination = ? + AND rp.checksum IS NOT NULL AND rp.verified_at_ns IS NOT NULL + ) + ) + ) + `, volumeID, destination, destination).Scan(&pending) + if err != nil { + return 0, fmt.Errorf("count pending fingerprints for volume %d on %q: %w", volumeID, destination, err) + } + return pending, nil +} + // ContentFingerprintVerified reports whether a verified provider // fingerprint backs the content on the destination via either layout: a // remote_objects row carrying a checksum and a verified_at_ns, or a diff --git a/store/destination_run_ids.go b/store/destination_run_ids.go index 99e9f1e..521fa22 100644 --- a/store/destination_run_ids.go +++ b/store/destination_run_ids.go @@ -31,14 +31,36 @@ const ( // content check on its own — a verified scan-back fingerprint must // back the object before such a component gates offload. VerifyMethodPresenceSize = "presence+size" + // VerifyMethodFingerprint is the upgrade of a presence+size component + // once every object and pack backing the (volume, destination) pair + // carries a verified provider fingerprint (a local BLAKE3 confirmed at + // upload, re-confirmed against the provider's checksum of the stored + // bytes). Capture and `squirrel verify` both mint it (see + // UpgradeDestinationVectorToFingerprintVerified). It relays over the + // durability pull verbatim like any other method, letting a node that + // cannot re-read the object itself (a relayed offsite) treat the + // evidence as content-verified. Because that relayed evidence is only + // as trustworthy as the responder's re-confirmation cadence, the + // offload gate accepts it as content-verified only while a scheduled + // verify cadence keeps it fresh — hence it is deliberately absent from + // ContentVerifiedMethod, which the gate treats as unconditionally + // content-verified. + VerifyMethodFingerprint = "fingerprint-verified" ) // ContentVerifiedMethod reports whether a durability component advanced -// by method carries genuine content verification — the precondition the -// offload gate applies before deleting a local copy. A presence-only or -// size+mtime method is not content-verified; an empty method (a pre-v19 -// component, or one whose provenance is unknown) is treated as -// unverified so the gate refuses rather than over-claims. +// by method carries genuine, cadence-independent content verification — +// the precondition the offload gate applies before deleting a local copy. +// A presence-only or size+mtime method is not content-verified; an empty +// method (a pre-v19 component, or one whose provenance is unknown) is +// treated as unverified so the gate refuses rather than over-claims. +// +// VerifyMethodFingerprint is deliberately excluded: its evidence counts as +// content-verified only while a scheduled verify cadence keeps it +// re-confirmed, a condition the offload gate applies itself (local: the +// configured cadence; relayed: the responder's cadence, baked in at pull +// time). Callers wanting the fingerprint-verified method to count must +// apply that cadence check themselves rather than reading it here. func ContentVerifiedMethod(method string) bool { switch method { case VerifyMethodBlake3, VerifyMethodPeer, VerifyMethodKopia: @@ -59,7 +81,7 @@ func ContentVerifiedMethod(method string) bool { // callers that accept it test for "" explicitly. func KnownVerifyMethod(method string) bool { switch method { - case VerifyMethodBlake3, VerifyMethodSizeMtime, VerifyMethodPeer, VerifyMethodKopia, VerifyMethodPresenceSize: + case VerifyMethodBlake3, VerifyMethodSizeMtime, VerifyMethodPeer, VerifyMethodKopia, VerifyMethodPresenceSize, VerifyMethodFingerprint: return true default: return false @@ -223,6 +245,52 @@ func (s *Store) AdvanceDestinationVectorTo(ctx context.Context, volumeID int64, return s.recordPushFreshness(ctx, volumeID, destination, components) } +// UpgradeDestinationVectorToFingerprintVerified re-stamps the destination's +// whole durability vector for the volume as VerifyMethodFingerprint when — +// and only when — every present content of the volume already carries a +// verified provider fingerprint on the destination (zero pending +// artifacts). It is the per-state gate that fixes the two friction-log F13 +// defects: capture and `squirrel verify` both call it, so the advance +// happens exactly when the outstanding fingerprints are complete, never +// vacuously past a still-pending pack (the advance covers the whole present +// set, so one pending artifact holds all of it) and never left stranded +// after verify fills the last fingerprint. +// +// selfNodeID is the coordinate NULL-origin content counts under, the same +// PresentOriginMaxima the push captured. It returns whether the vector was +// upgraded; a pending artifact, an empty present set, or a destination this +// volume has never successfully pushed to all return (false, nil) without +// touching the vector — the last guard keeps content that is durable on the +// destination only through *another* volume's push (objects are +// destination-global) from minting a vector component for a volume whose +// path→hash manifest never landed there. +func (s *Store) UpgradeDestinationVectorToFingerprintVerified(ctx context.Context, volumeID int64, destination string, selfNodeID int64) (bool, error) { + if _, err := s.LatestSuccessfulSyncRun(ctx, volumeID, destination); err != nil { + if IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("lookup last successful sync of %q: %w", destination, err) + } + pending, err := s.CountVolumeContentsPendingFingerprint(ctx, volumeID, destination) + if err != nil { + return false, err + } + if pending != 0 { + return false, nil + } + components, err := s.PresentOriginMaxima(ctx, volumeID, selfNodeID) + if err != nil { + return false, err + } + if len(components) == 0 { + return false, nil + } + if err := s.AdvanceDestinationVectorTo(ctx, volumeID, destination, VerifyMethodFingerprint, components); err != nil { + return false, fmt.Errorf("upgrade destination vector for %q: %w", destination, err) + } + return true, nil +} + // recordPushFreshness overwrites the destination's push-freshness maxima // to exactly the supplied snapshot — the per-origin-node maxima of the // present set this push enumerated. Distinct from the monotonic vector diff --git a/store/destination_run_ids_test.go b/store/destination_run_ids_test.go index 4046a64..78fed54 100644 --- a/store/destination_run_ids_test.go +++ b/store/destination_run_ids_test.go @@ -801,7 +801,9 @@ func TestContentVerifiedMethod(t *testing.T) { t.Fatalf("method %q should be content-verified", m) } } - for _, m := range []string{VerifyMethodPresenceSize, VerifyMethodSizeMtime, "", "bogus"} { + // fingerprint-verified is intentionally NOT unconditionally content- + // verified: its acceptance is cadence-coupled and applied by the gate. + for _, m := range []string{VerifyMethodPresenceSize, VerifyMethodSizeMtime, VerifyMethodFingerprint, "", "bogus"} { if ContentVerifiedMethod(m) { t.Fatalf("method %q must not be content-verified", m) } diff --git a/store/remote_objects_test.go b/store/remote_objects_test.go index d7c0e4c..31cc65c 100644 --- a/store/remote_objects_test.go +++ b/store/remote_objects_test.go @@ -207,6 +207,45 @@ func TestBeginRemoteVerifyRun(t *testing.T) { } } +// TestBeginDurabilityPullRun: an agent-scheduled durability pull rides on a +// kind='audit' run with no volume and no destination, so it stays out of the +// per-volume drift-audit reads even though it concerns a specific volume — +// the pulled volume/peer live in the run's runs_audit note instead. +func TestBeginDurabilityPullRun(t *testing.T) { + s := openTestStore(t) + ctx := context.Background() + v, err := s.CreateVolume(ctx, "media", "/data/media") + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + id, err := s.BeginDurabilityPullRun(ctx) + if err != nil { + t.Fatalf("BeginDurabilityPullRun: %v", err) + } + run, err := s.GetRun(ctx, id) + if err != nil { + t.Fatalf("GetRun: %v", err) + } + if run.Kind != RunKindAudit || run.VolumeID.Valid || run.Destination.Valid || run.Status != RunStatusRunning { + t.Fatalf("run = %+v, want a running audit run with NULL volume and destination", run) + } + if err := s.AppendRunAudit(ctx, RunAuditEntry{RunID: id, Transition: TransitionPullDurability, Note: "volume=media peer=nas fetched=3 applied=2 dropped=0 rewinds=0"}); err != nil { + t.Fatalf("AppendRunAudit: %v", err) + } + if err := s.FinishRun(ctx, id, RunStatusSuccess, "", 2); err != nil { + t.Fatalf("FinishRun: %v", err) + } + // The NULL volume_id keeps it out of the drift-since-last-sync handshake + // read, which is scoped to a volume. + audits, err := s.ListAuditRunsSince(ctx, v.ID, 0) + if err != nil { + t.Fatalf("ListAuditRunsSince: %v", err) + } + if len(audits) != 0 { + t.Fatalf("durability-pull run leaked into volume drift audits: %+v", audits) + } +} + // TestRemoteObjectInsertRefusesDuplicate: the fingerprint recorded at // upload time is what later verifications compare against, so a second // insert for the same (content, destination) must fail loudly instead diff --git a/store/runs.go b/store/runs.go index ed48fa0..49ac6a2 100644 --- a/store/runs.go +++ b/store/runs.go @@ -143,28 +143,49 @@ func (s *Store) BeginIndexRun(ctx context.Context, kind string, volumeID int64, return id, nil } -// BeginRemoteVerifyRun records the start of a remote-object verification -// pass as a kind='audit' run. The pass is destination-scoped rather than -// volume-scoped — the content-addressed objects/ space is shared by -// every volume — so volume_id is NULL; and the runs CHECK keeps -// destination NULL on audit rows, so the verified destination is -// recorded in the run's 'verify-destination' runs_audit note instead. -// Callers must pair it with FinishRun. -func (s *Store) BeginRemoteVerifyRun(ctx context.Context) (int64, error) { +// beginVolumelessAuditRun inserts a 'running' kind='audit' row with NULL +// volume_id and NULL destination — the shape shared by the out-of-band +// checks that are scoped to a destination or a (volume, peer) pair rather +// than to a volume's on-disk drift history (remote-object verification and +// durability pulls). Keeping volume_id NULL deliberately holds these rows +// out of the per-volume "last audit" reads (ListAuditRunsSince, which feeds +// the drift-since-last-sync handshake, and LatestSuccessfulRunsByVolumeAndKind, +// which feeds the TUI); the runs CHECK also keeps destination NULL on audit +// rows, so the subject is recorded in the run's runs_audit note instead. +// label names the operation for the insert's error messages. Callers must +// pair it with FinishRun. +func (s *Store) beginVolumelessAuditRun(ctx context.Context, label string) (int64, error) { res, err := s.db.ExecContext(ctx, ` INSERT INTO runs (kind, volume_id, destination, started_at_ns, status, file_count) VALUES ('audit', NULL, NULL, ?, 'running', 0) `, NowNs()) if err != nil { - return 0, fmt.Errorf("insert remote-verify run: %w", err) + return 0, fmt.Errorf("insert %s run: %w", label, err) } id, err := res.LastInsertId() if err != nil { - return 0, fmt.Errorf("remote-verify run last insert id: %w", err) + return 0, fmt.Errorf("%s run last insert id: %w", label, err) } return id, nil } +// BeginRemoteVerifyRun records the start of a remote-object verification +// pass as a kind='audit' run; the verified destination lands in the run's +// 'verify-destination' runs_audit note. See beginVolumelessAuditRun. +func (s *Store) BeginRemoteVerifyRun(ctx context.Context) (int64, error) { + return s.beginVolumelessAuditRun(ctx, "remote-verify") +} + +// BeginDurabilityPullRun records the start of an agent-scheduled durability +// pull as a kind='audit' run; the pulled volume and peer land in the run's +// 'pull-durability' runs_audit note. See beginVolumelessAuditRun. +func (s *Store) BeginDurabilityPullRun(ctx context.Context) (int64, error) { + // "pull-durability" matches the runs_audit transition + // (TransitionPullDurability) and the scheduler's log kind, so the run, + // its note, and the scheduler line all correlate under one name. + return s.beginVolumelessAuditRun(ctx, "pull-durability") +} + // BeginPeerSyncRun is BeginRun's sibling for kind='sync' rows tied to a // peer node. It records the (peer_node_id, correlated_run_id) pair // alongside the regular destination name (the peer's name from the diff --git a/store/runs_audit.go b/store/runs_audit.go index 14be1d4..35b86d8 100644 --- a/store/runs_audit.go +++ b/store/runs_audit.go @@ -33,6 +33,12 @@ const ( // CHECK keeps destination NULL on audit rows, so this entry is where // the audit trail names the verified destination. TransitionVerifyDestination = "verify-destination" + // TransitionPullDurability records an agent-scheduled durability pull + // against its kind='audit' run (see BeginDurabilityPullRun). The note + // carries the pulled volume, the peer, and the fetched/applied/dropped/ + // rewind counters — the runs CHECK keeps volume_id and destination NULL + // on audit rows, so this entry is where the audit trail names them. + TransitionPullDurability = "pull-durability" // TransitionResetDestination records a `squirrel destination reset`: // the operator forgetting a destination's recorded upload and // durability state (ResetDestination). It shares the destination-scoped diff --git a/sync/content_addressed.go b/sync/content_addressed.go index 74b7d87..b5c6637 100644 --- a/sync/content_addressed.go +++ b/sync/content_addressed.go @@ -222,11 +222,19 @@ func (h *contentAddressedHandler) push(ctx context.Context, rep *Report, volID, if err := h.uploadSegment(ctx, delta, runID); err != nil { return err } - // presence+size is not a content-verified method (crypt remotes - // expose no hashes): the component advances so a later scan-back - // fingerprint can upgrade it, but the offload gate holds this target - // out until a verified fingerprint backs the gated object. - if err := h.store.AdvanceDestinationVectorTo(ctx, volID, h.dest.Name, store.VerifyMethodPresenceSize, advance); err != nil { + // presence+size is not a content-verified method (crypt remotes expose + // no hashes): the component advances so the offload gate's per-object + // scan-back can back it locally. When this run's capture leaves the + // whole (volume, destination) pair with a verified fingerprint behind + // every present content, the advance is instead stamped + // fingerprint-verified — the content-verified method that also relays + // over the durability pull (see the durability-vector upgrade in + // store). A single still-pending object holds it at presence+size. + method, err := h.advanceMethod(ctx, volID, len(advance)) + if err != nil { + return err + } + if err := h.store.AdvanceDestinationVectorTo(ctx, volID, h.dest.Name, method, advance); err != nil { return fmt.Errorf("advance destination vector for %s: %w", h.dest.Name, err) } rep.Status = store.RunStatusSuccess @@ -234,6 +242,27 @@ func (h *contentAddressedHandler) push(ctx context.Context, rep *Report, volID, return nil } +// advanceMethod chooses the method the content-addressed push advances its +// durability vector with: fingerprint-verified when every present content +// of the volume already carries a verified provider fingerprint on this +// destination (the whole-state check, not just this run's uploads), and +// presence+size otherwise. advanceLen is the size of the advance snapshot; +// an empty snapshot advances nothing, so the method is immaterial and the +// pending query is skipped. +func (h *contentPusher) advanceMethod(ctx context.Context, volID int64, advanceLen int) (string, error) { + if advanceLen == 0 { + return store.VerifyMethodPresenceSize, nil + } + pending, err := h.store.CountVolumeContentsPendingFingerprint(ctx, volID, h.dest.Name) + if err != nil { + return "", err + } + if pending == 0 { + return store.VerifyMethodFingerprint, nil + } + return store.VerifyMethodPresenceSize, nil +} + // watermark resolves the run id the delta starts after: the last // successful sync of this (volume, destination), or 0 for a fresh // destination. The last success must still have its manifest segment at diff --git a/sync/content_addressed_test.go b/sync/content_addressed_test.go index 1777a24..d6013df 100644 --- a/sync/content_addressed_test.go +++ b/sync/content_addressed_test.go @@ -465,14 +465,16 @@ func TestContentAddressedPushHappyPath(t *testing.T) { if vector[0].OriginNodeID != self.ID || vector[0].OriginRunID == 0 { t.Fatalf("vector component = %+v, want self node at the introduction run", vector[0]) } - // The content-addressed advance records presence+size, not a - // content-verified method: the offload gate holds this target out - // until a verified fingerprint backs the object (#109). - if vector[0].VerifyMethod != VerifyMethodPresenceSize { - t.Fatalf("verify method = %q, want %q", vector[0].VerifyMethod, VerifyMethodPresenceSize) + // Both objects were fingerprinted this run, so the whole (volume, + // destination) pair is fingerprint-verified and the advance is stamped + // fingerprint-verified — the #155 upgrade at capture. It is not + // unconditionally content-verified (ContentVerifiedMethod excludes it): + // the offload gate additionally requires a verify cadence (#155). + if vector[0].VerifyMethod != VerifyMethodFingerprint { + t.Fatalf("verify method = %q, want %q", vector[0].VerifyMethod, VerifyMethodFingerprint) } if store.ContentVerifiedMethod(vector[0].VerifyMethod) { - t.Fatalf("presence+size must not count as content-verified") + t.Fatalf("fingerprint-verified must not count as unconditionally content-verified") } // Transfers and confirmations address the crypt overlay remote. @@ -485,6 +487,37 @@ func TestContentAddressedPushHappyPath(t *testing.T) { } } +// TestContentAddressedVerifyUpgradesVector is the friction-log F13(b) side +// for the content-addressed layout: a push whose object fingerprint capture +// failed advances presence+size, and a later verify that fills the +// fingerprint upgrades the component to fingerprint-verified (issue #155). +func TestContentAddressedVerifyUpgradesVector(t *testing.T) { + f := setupContentAddressedFixture(t) + ctx := context.Background() + t.Setenv("RCLONE_FAKE_NO_HASHES", "1") + f.write(t, "a.txt", "alpha") + f.index(t) + if _, err := f.sync(t); err != nil { + t.Fatalf("sync: %v", err) + } + vec, _ := f.store.ListDestinationRunIDs(ctx, f.volumeID(t), "offsite") + if len(vec) != 1 || vec[0].VerifyMethod != VerifyMethodPresenceSize { + t.Fatalf("vector = %+v, want one presence+size component while the fingerprint is pending", vec) + } + + t.Setenv("RCLONE_FAKE_NO_HASHES", "") + if _, err := VerifyRemote(ctx, f.store, f.rcl, f.pair.Destination); err != nil { + t.Fatalf("VerifyRemote: %v", err) + } + vec, err := f.store.ListDestinationRunIDs(ctx, f.volumeID(t), "offsite") + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + if len(vec) != 1 || vec[0].VerifyMethod != VerifyMethodFingerprint { + t.Fatalf("vector = %+v, want fingerprint-verified after verify", vec) + } +} + // TestContentAddressedUploadOnce: a second run uploads only hashes the // destination has no record of — a new path carrying already-recorded // content transfers nothing and still lands in the manifest. diff --git a/sync/durability.go b/sync/durability.go index 013f3fa..da23392 100644 --- a/sync/durability.go +++ b/sync/durability.go @@ -174,7 +174,7 @@ func pullDurability(ctx context.Context, s *store.Store, client *nodeClient, vol if err != nil { return rep, err } - err = s.UpsertDestinationRunIDPulled(ctx, volumeID, c.Destination, nodeID, c.OriginRun, c.VerifyMethod, sourceNode.ID, c.VerifiedAtNs, allowRewind) + err = s.UpsertDestinationRunIDPulled(ctx, volumeID, c.Destination, nodeID, c.OriginRun, relayedMethod(c), sourceNode.ID, c.VerifiedAtNs, allowRewind) var rewind *store.DestinationRewindError if errors.As(err, &rewind) { rep.Rewinds = append(rep.Rewinds, DurabilityRewind{ @@ -210,6 +210,23 @@ func pullDurability(ctx context.Context, s *store.Store, client *nodeClient, vol return rep, nil } +// relayedMethod is the verify method a pulled component is stored under. +// It is the responder's method verbatim, except a fingerprint-verified +// component is baked down to presence+size unless the responder relayed a +// positive verify cadence for the destination: that method is content- +// verified for the puller's offload gate (the fingerprint cannot be +// re-read locally), so it is only honoured while the responder keeps +// re-confirming it. Absence of the cadence (an older responder that never +// sends it, or a destination the responder runs on no cadence) fails the +// coupling closed — the component then needs a local fingerprint the +// relaying node does not hold, so the gate refuses (issue #155). +func relayedMethod(c syncproto.DurabilityComponent) string { + if c.VerifyMethod == store.VerifyMethodFingerprint && c.VerifyEveryNs <= 0 { + return store.VerifyMethodPresenceSize + } + return c.VerifyMethod +} + // validateComponent guards the wire-supplied component before it // touches the local vector: destination and origin names are // identities, the run id must be a positive origin-space id, and the diff --git a/sync/durability_test.go b/sync/durability_test.go index 04aa5d6..54ef682 100644 --- a/sync/durability_test.go +++ b/sync/durability_test.go @@ -5,11 +5,39 @@ import ( "fmt" "strings" "testing" + "time" "github.com/mbertschler/squirrel/store" "github.com/mbertschler/squirrel/syncproto" ) +// TestRelayedMethodCadenceCoupling: a relayed fingerprint-verified +// component is stored verbatim only when the responder relayed a positive +// verify cadence; without it the method is baked down to presence+size so +// the puller (which holds no local fingerprint) fails the offload gate +// closed. Other methods pass through regardless (issue #155). +func TestRelayedMethodCadenceCoupling(t *testing.T) { + base := syncproto.DurabilityComponent{Destination: "s3", OriginNode: "laptop", OriginRun: 5} + + c := base + c.VerifyMethod = store.VerifyMethodFingerprint + c.VerifyEveryNs = int64(168 * time.Hour) + if got := relayedMethod(c); got != store.VerifyMethodFingerprint { + t.Fatalf("relayedMethod(cadenced) = %q, want fingerprint-verified", got) + } + + c.VerifyEveryNs = 0 + if got := relayedMethod(c); got != store.VerifyMethodPresenceSize { + t.Fatalf("relayedMethod(no cadence) = %q, want presence+size (fail closed)", got) + } + + c = base + c.VerifyMethod = store.VerifyMethodBlake3 + if got := relayedMethod(c); got != store.VerifyMethodBlake3 { + t.Fatalf("relayedMethod(blake3) = %q, want blake3 unchanged", got) + } +} + // seedReceiverDurability records vector components on the receiver so // the pull tests have something to fetch. Returns the receiver's self // name (the origin-node identity its components travel under). @@ -437,6 +465,7 @@ func TestValidateComponentVerifyMethod(t *testing.T) { store.VerifyMethodPeer, store.VerifyMethodKopia, store.VerifyMethodPresenceSize, + store.VerifyMethodFingerprint, } { c := base c.VerifyMethod = method diff --git a/sync/handler.go b/sync/handler.go index ab7e21a..5ee15f3 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -35,6 +35,10 @@ const ( // (crypt remotes expose no hashes), so results carrying it stay // unverified until the provider-checksum fingerprint pass lands. VerifyMethodPresenceSize = store.VerifyMethodPresenceSize + // VerifyMethodFingerprint upgrades a presence+size component once every + // object and pack backing the (volume, destination) pair is + // fingerprint-verified. Minted by capture and by `squirrel verify`. + VerifyMethodFingerprint = store.VerifyMethodFingerprint ) // VerifyResult is the typed durability report of one handler push: how diff --git a/sync/packed.go b/sync/packed.go index 6e73c7a..1a117a5 100644 --- a/sync/packed.go +++ b/sync/packed.go @@ -267,21 +267,45 @@ func (h *packedHandler) push(ctx context.Context, rep *Report, volID, runID int6 // certifyPacked closes the durability seam: it records a remote_packs row // per landed pack, reads each pack's scan-back fingerprint, and advances -// the destination vector only once every pack this run assembled is -// fingerprint-verified — the third leg of the three-artifact gate (packs + -// placement map + manifest segment all landed and confirmed). A pending -// capture holds the vector so the offload gate never counts unverified -// packed content as durable; `squirrel verify` fills the fingerprint later -// and a subsequent sync advances the vector. rep.Verification stays -// unverified (presence+size is not a content-verified method), so RunPair -// does not advance the vector a second time. +// the destination vector to fingerprint-verified only once the whole +// (volume, destination) pair carries a verified fingerprint behind every +// present file content. That whole-state check is +// CountVolumeContentsPendingFingerprint, which counts present contents +// lacking a verified remote_objects (per-hash object) or remote_packs (pack +// member) fingerprint on this destination — so it covers packs and per-hash +// objects, the two artifacts that carry content bytes. +// +// The placement map and manifest segment are deliberately NOT in that +// pending set: they are re-derivable squirrel-written metadata that carry +// no scan-back fingerprint. Their durability before this advance is handled +// upstream in push, which uploads and confirms both landed at their +// expected size before promoting the run to success — a run whose map or +// segment failed to land returns failed and never reaches certifyPacked, so +// the vector is untouched. +// +// Gating on the whole pending set rather than this run's writes is the +// friction-log F13 fix: a run that packs nothing no longer advances +// vacuously past an earlier still-pending pack, and a still-pending artifact +// anywhere in the pair holds the whole advance. `squirrel verify` fills a +// pending fingerprint later and re-attempts this advance itself (see the +// store upgrade), so the vector no longer stalls until the next +// content-writing sync. rep.Verification stays unverified (presence+size is +// not content-verified), so RunPair does not advance the vector a second +// time. func (h *packedHandler) certifyPacked(ctx context.Context, rep *Report, volID, runID int64, writes []store.PackWrite, advance []store.OriginComponent) error { - verified := h.capturePackFingerprints(ctx, rep, runID, writes) - if verified < len(writes) { - rep.Warnings = append(rep.Warnings, fmt.Sprintf("destination %q: %d of %d pack(s) this run are not yet fingerprint-verified; the durability vector was not advanced — run `squirrel verify` to certify them", h.dest.Name, len(writes)-verified, len(writes))) + h.capturePackFingerprints(ctx, rep, runID, writes) + pending, err := h.store.CountVolumeContentsPendingFingerprint(ctx, volID, h.dest.Name) + if err != nil { + return fmt.Errorf("count pending fingerprints for %s: %w", h.dest.Name, err) + } + if pending > 0 { + rep.Warnings = append(rep.Warnings, fmt.Sprintf("destination %q: %d content(s) are not yet fingerprint-verified; the durability vector was not advanced — run `squirrel verify` to certify them", h.dest.Name, pending)) + return nil + } + if len(advance) == 0 { return nil } - if err := h.store.AdvanceDestinationVectorTo(ctx, volID, h.dest.Name, store.VerifyMethodPresenceSize, advance); err != nil { + if err := h.store.AdvanceDestinationVectorTo(ctx, volID, h.dest.Name, store.VerifyMethodFingerprint, advance); err != nil { return fmt.Errorf("advance destination vector for %s: %w", h.dest.Name, err) } return nil @@ -294,9 +318,9 @@ func (h *packedHandler) certifyPacked(ctx context.Context, rep *Report, volID, r // (never rclone's md5 slot, which is blank for a multipart object); other // backends read `rclone lsjson --hash`. A pack whose fingerprint could not // be read stays pending (checksum NULL) with a warning — never a fabricated -// value. It returns how many packs were fingerprint-verified this run. -func (h *packedHandler) capturePackFingerprints(ctx context.Context, rep *Report, runID int64, writes []store.PackWrite) int { - before := rep.Fingerprints +// value. The whole-pair pending tally (CountVolumeContentsPendingFingerprint) +// is what certifyPacked gates the vector advance on, so this returns nothing. +func (h *packedHandler) capturePackFingerprints(ctx context.Context, rep *Report, runID int64, writes []store.PackWrite) { targets := make([]captureTarget, 0, len(writes)) for _, w := range writes { pack, err := h.store.GetPackByKey(ctx, w.Pack.PackKey) @@ -320,7 +344,6 @@ func (h *packedHandler) capturePackFingerprints(ctx context.Context, rep *Report }) } h.captureScanBackFingerprints(ctx, rep, PacksDirName, targets) - return int(rep.Fingerprints - before) } // watermark resolves the run id the delta starts after: the last diff --git a/sync/packed_test.go b/sync/packed_test.go index 0e197ce..c586284 100644 --- a/sync/packed_test.go +++ b/sync/packed_test.go @@ -183,8 +183,8 @@ func TestPackedDurabilityNotCertified(t *testing.T) { if rep.Status != store.RunStatusSuccess { t.Fatalf("Status = %q, want success", rep.Status) } - // presence+size is not itself a content-verified method; the gate - // upgrades it through the pack's verified fingerprint. + // The run report stays unverified (presence+size); the durability + // vector is upgraded separately once the pack's fingerprint lands. if rep.Verification.Verified() { t.Fatalf("packed push reported verified; presence+size is not content-verified") } @@ -199,8 +199,10 @@ func TestPackedDurabilityNotCertified(t *testing.T) { if err != nil { t.Fatalf("ListDestinationRunIDs: %v", err) } - if len(vector) != 1 || vector[0].VerifyMethod != store.VerifyMethodPresenceSize { - t.Fatalf("vector = %+v, want one presence+size component (certified after fingerprint)", vector) + // With the pack fingerprinted the whole pair is fingerprint-verified, + // so certify advances the vector to fingerprint-verified (#155). + if len(vector) != 1 || vector[0].VerifyMethod != store.VerifyMethodFingerprint { + t.Fatalf("vector = %+v, want one fingerprint-verified component (certified after fingerprint)", vector) } }) @@ -236,6 +238,75 @@ func TestPackedDurabilityNotCertified(t *testing.T) { }) } +// TestPackedVerifyThenAdvance is the friction-log F13(b) fix for packed: a +// pack whose fingerprint capture failed holds the whole vector (empty), and +// a later `squirrel verify` that fills the fingerprint re-attempts the +// advance itself — upgrading the vector to fingerprint-verified rather than +// stalling until the next content-writing sync. +func TestPackedVerifyThenAdvance(t *testing.T) { + f := setupPackedFixture(t, "1MiB") + ctx := context.Background() + t.Setenv("RCLONE_FAKE_NO_HASHES", "1") + f.write(t, "small.txt", "tiny") + f.index(t) + if _, err := f.sync(t); err != nil { + t.Fatalf("sync: %v", err) + } + if vec, _ := f.store.ListDestinationRunIDs(ctx, f.volumeID(t), "offsite"); len(vec) != 0 { + t.Fatalf("vector = %+v, want empty while the pack fingerprint is pending", vec) + } + + t.Setenv("RCLONE_FAKE_NO_HASHES", "") + rep, err := VerifyRemote(ctx, f.store, f.rcl, f.pair.Destination) + if err != nil { + t.Fatalf("VerifyRemote: %v", err) + } + if rep.PacksPopulated != 1 || !rep.Clean() { + t.Fatalf("verify rep = %+v, want one pack fingerprint populated on a clean pass", rep) + } + vec, err := f.store.ListDestinationRunIDs(ctx, f.volumeID(t), "offsite") + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + if len(vec) != 1 || vec[0].VerifyMethod != store.VerifyMethodFingerprint { + t.Fatalf("vector = %+v, want one fingerprint-verified component after verify", vec) + } +} + +// TestPackedLaterRunHeldByPendingPack is the friction-log F13(a) fix: a +// later run whose own pack is fingerprint-verified must NOT advance the +// vector while an earlier run's pack is still pending. The per-state gate +// holds the whole advance — the old per-run gate advanced vacuously past +// the earlier pending pack. +func TestPackedLaterRunHeldByPendingPack(t *testing.T) { + f := setupPackedFixture(t, "1MiB") + ctx := context.Background() + + // Run A: the pack lands but the backend exposes no checksum -> pending. + t.Setenv("RCLONE_FAKE_NO_HASHES", "1") + f.write(t, "one.txt", "first") + f.index(t) + if _, err := f.sync(t); err != nil { + t.Fatalf("run A sync: %v", err) + } + + // Run B: its own pack IS fingerprinted, but run A's pack stays pending. + t.Setenv("RCLONE_FAKE_NO_HASHES", "") + f.write(t, "two.txt", "second") + f.index(t) + if _, err := f.sync(t); err != nil { + t.Fatalf("run B sync: %v", err) + } + + vec, err := f.store.ListDestinationRunIDs(ctx, f.volumeID(t), "offsite") + if err != nil { + t.Fatalf("ListDestinationRunIDs: %v", err) + } + if len(vec) != 0 { + t.Fatalf("vector = %+v, want empty: run B must not advance past run A's still-pending pack", vec) + } +} + // TestPackedAssemblyDeterministic: the same input set produces byte-for-byte // identical pack bytes and the same pack key. func TestPackedAssemblyDeterministic(t *testing.T) { diff --git a/sync/verify_remote.go b/sync/verify_remote.go index 117d4b5..3eca4b5 100644 --- a/sync/verify_remote.go +++ b/sync/verify_remote.go @@ -123,9 +123,44 @@ func VerifyRemote(ctx context.Context, s *store.Store, rcl *Rclone, dest *config if err := recordVerifyOutcome(ctx, s, &rep, verifyErr); err != nil { return rep, err } + // A clean pass may have filled the last pending fingerprint of a + // (volume, destination) pair: re-attempt the durability-vector upgrade + // now so the vector no longer stalls until the next content-writing + // sync (friction-log F13). A dirty pass (mismatch or missing artifact) + // leaves the recorded fingerprints as found, so it must not advance + // evidence. Like RunPair's advance, an upgrade failure surfaces as the + // command's error even though the audit run already closed. + if verifyErr == nil && rep.Clean() { + if err := upgradeFingerprintVectors(ctx, s, dest.Name); err != nil { + return rep, err + } + } return rep, verifyErr } +// upgradeFingerprintVectors re-stamps the durability vector of every +// volume as fingerprint-verified where the destination now carries a +// verified fingerprint behind all its present content +// (UpgradeDestinationVectorToFingerprintVerified is a no-op for a volume +// that was never pushed to the destination or still has a pending +// artifact, so iterating every volume is safe and self-limiting). +func upgradeFingerprintVectors(ctx context.Context, s *store.Store, destination string) error { + self, err := s.GetSelfNode(ctx) + if err != nil { + return fmt.Errorf("verify %q: self node: %w", destination, err) + } + volumes, err := s.ListVolumes(ctx) + if err != nil { + return fmt.Errorf("verify %q: list volumes: %w", destination, err) + } + for _, v := range volumes { + if _, err := s.UpgradeDestinationVectorToFingerprintVerified(ctx, v.ID, destination, self.ID); err != nil { + return fmt.Errorf("verify %q: upgrade vector for volume %q: %w", destination, v.Name, err) + } + } + return nil +} + // verifyRecorded sweeps a destination's recorded content objects (the // large-file per-object sweep, shared with content-addressed) and, for a // packed destination, its recorded packs — one fingerprint check per pack diff --git a/syncproto/syncproto.go b/syncproto/syncproto.go index a7ac0ef..7a006cd 100644 --- a/syncproto/syncproto.go +++ b/syncproto/syncproto.go @@ -411,13 +411,25 @@ type DurabilityResponse struct { // verification time is unknown (a pre-v23 responder, or evidence never // re-verified); the puller treats that as fail-closed. The responder's // own verification instant, never fresher than this hop's pull. +// +// VerifyEveryNs is the responder's effective verify cadence (in +// nanoseconds) for Destination — a per-destination verify_every or the +// [agent] default — relayed so the puller can hold the cadence coupling +// the offload gate requires of a fingerprint-verified component: that +// evidence counts as content-verified only while it keeps being +// re-confirmed. The puller keeps a relayed VerifyMethodFingerprint +// component content-verified only when this is positive; a zero (an older +// responder that never sends it, or a destination the responder configures +// with no cadence) fails the coupling closed. Mirrors VerifiedAtNs: relayed +// verbatim, absence is the safe direction. Immaterial for any other method. type DurabilityComponent struct { - Destination string `json:"destination"` - OriginNode string `json:"origin_node"` - OriginRun int64 `json:"origin_run"` - UpdatedAtNs int64 `json:"updated_at_ns"` - VerifyMethod string `json:"verify_method,omitempty"` - VerifiedAtNs int64 `json:"verified_at_ns,omitempty"` + Destination string `json:"destination"` + OriginNode string `json:"origin_node"` + OriginRun int64 `json:"origin_run"` + UpdatedAtNs int64 `json:"updated_at_ns"` + VerifyMethod string `json:"verify_method,omitempty"` + VerifiedAtNs int64 `json:"verified_at_ns,omitempty"` + VerifyEveryNs int64 `json:"verify_every_ns,omitempty"` } // DurabilityFreshness is one origin-space freshness coordinate: the