diff --git a/cmd/api/api/health_check.go b/cmd/api/api/health_check.go new file mode 100644 index 00000000..30b7bc0e --- /dev/null +++ b/cmd/api/api/health_check.go @@ -0,0 +1,138 @@ +package api + +import ( + "fmt" + + "github.com/kernel/hypeman/lib/healthcheck" + "github.com/kernel/hypeman/lib/oapi" + "github.com/samber/lo" +) + +func toDomainHealthCheck(policy *oapi.HealthCheck) (*healthcheck.Policy, error) { + if policy == nil { + return nil, nil + } + + out := &healthcheck.Policy{} + if policy.Type != nil { + out.Type = healthcheck.Type(*policy.Type) + } + if policy.Interval != nil { + out.Interval = *policy.Interval + } + if policy.Timeout != nil { + out.Timeout = *policy.Timeout + } + if policy.StartPeriod != nil { + out.StartPeriod = *policy.StartPeriod + } + if policy.FailureThreshold != nil { + out.FailureThreshold = *policy.FailureThreshold + } + if policy.SuccessThreshold != nil { + out.SuccessThreshold = *policy.SuccessThreshold + } + if policy.Http != nil { + if policy.Http.Port < 1 || policy.Http.Port > 65535 { + return nil, fmt.Errorf("health_check.http.port must be between 1 and 65535") + } + out.HTTP = &healthcheck.HTTPCheck{ + Port: uint16(policy.Http.Port), + } + if policy.Http.Path != nil { + out.HTTP.Path = *policy.Http.Path + } + if policy.Http.Scheme != nil { + out.HTTP.Scheme = string(*policy.Http.Scheme) + } + if policy.Http.ExpectedStatus != nil { + out.HTTP.ExpectedStatus = *policy.Http.ExpectedStatus + } + } + if policy.Tcp != nil { + if policy.Tcp.Port < 1 || policy.Tcp.Port > 65535 { + return nil, fmt.Errorf("health_check.tcp.port must be between 1 and 65535") + } + out.TCP = &healthcheck.TCPCheck{Port: uint16(policy.Tcp.Port)} + } + if policy.Exec != nil { + out.Exec = &healthcheck.ExecCheck{ + Command: append([]string(nil), policy.Exec.Command...), + } + if policy.Exec.WorkingDir != nil { + out.Exec.WorkingDir = *policy.Exec.WorkingDir + } + } + + return healthcheck.NormalizePolicy(out) +} + +func toOAPIHealthCheck(policy *healthcheck.Policy) *oapi.HealthCheck { + if policy == nil { + return nil + } + + typ := oapi.HealthCheckType(policy.Type) + out := &oapi.HealthCheck{ + Type: &typ, + } + if policy.Interval != "" { + out.Interval = lo.ToPtr(policy.Interval) + } + if policy.Timeout != "" { + out.Timeout = lo.ToPtr(policy.Timeout) + } + if policy.StartPeriod != "" { + out.StartPeriod = lo.ToPtr(policy.StartPeriod) + } + if policy.FailureThreshold != 0 { + out.FailureThreshold = lo.ToPtr(policy.FailureThreshold) + } + if policy.SuccessThreshold != 0 { + out.SuccessThreshold = lo.ToPtr(policy.SuccessThreshold) + } + if policy.HTTP != nil { + out.Http = &oapi.HealthCheckHTTP{ + Port: int(policy.HTTP.Port), + } + if policy.HTTP.Path != "" { + out.Http.Path = lo.ToPtr(policy.HTTP.Path) + } + if policy.HTTP.Scheme != "" { + scheme := oapi.HealthCheckHTTPScheme(policy.HTTP.Scheme) + out.Http.Scheme = &scheme + } + if policy.HTTP.ExpectedStatus != 0 { + out.Http.ExpectedStatus = lo.ToPtr(policy.HTTP.ExpectedStatus) + } + } + if policy.TCP != nil { + out.Tcp = &oapi.HealthCheckTCP{ + Port: int(policy.TCP.Port), + } + } + if policy.Exec != nil { + out.Exec = &oapi.HealthCheckExec{ + Command: append([]string(nil), policy.Exec.Command...), + } + if policy.Exec.WorkingDir != "" { + out.Exec.WorkingDir = lo.ToPtr(policy.Exec.WorkingDir) + } + } + return out +} + +func toOAPIHealthStatus(snapshot healthcheck.StatusSnapshot) *oapi.InstanceHealthStatus { + out := &oapi.InstanceHealthStatus{ + Status: oapi.InstanceHealthStatusStatus(snapshot.Status), + ConsecutiveSuccesses: snapshot.ConsecutiveSuccesses, + ConsecutiveFailures: snapshot.ConsecutiveFailures, + LastCheckedAt: snapshot.LastCheckedAt, + LastSuccessAt: snapshot.LastSuccessAt, + LastFailureAt: snapshot.LastFailureAt, + } + if snapshot.LastError != "" { + out.LastError = lo.ToPtr(snapshot.LastError) + } + return out +} diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 7bcde181..0a5a9c0f 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -11,6 +11,7 @@ import ( "github.com/c2h5oh/datasize" "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/logger" @@ -285,6 +286,13 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Message: err.Error(), }, nil } + healthCheck, err := toDomainHealthCheck(request.Body.HealthCheck) + if err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_health_check", + Message: err.Error(), + }, nil + } domainReq := instances.CreateInstanceRequest{ Name: request.Body.Name, @@ -310,6 +318,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst SkipKernelHeaders: request.Body.SkipKernelHeaders != nil && *request.Body.SkipKernelHeaders, SkipGuestAgent: request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent, AutoStandby: autoStandby, + HealthCheck: healthCheck, } if request.Body.SnapshotPolicy != nil { snapshotPolicy, err := toInstanceSnapshotPolicy(*request.Body.SnapshotPolicy) @@ -1028,10 +1037,18 @@ func (s *ApiService) UpdateInstance(ctx context.Context, request oapi.UpdateInst Message: err.Error(), }, nil } + healthCheck, err := toDomainHealthCheck(request.Body.HealthCheck) + if err != nil { + return oapi.UpdateInstance400JSONResponse{ + Code: "invalid_health_check", + Message: err.Error(), + }, nil + } result, err := s.InstanceManager.UpdateInstance(ctx, inst.Id, instances.UpdateInstanceRequest{ Env: env, AutoStandby: autoStandby, + HealthCheck: healthCheck, }) if err != nil { switch { @@ -1163,6 +1180,8 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { oapiInst.SnapshotPolicy = &oapiPolicy } oapiInst.AutoStandby = toOAPIAutoStandbyPolicy(inst.AutoStandby) + oapiInst.HealthCheck = toOAPIHealthCheck(inst.HealthCheck) + oapiInst.HealthStatus = toOAPIHealthStatus(healthcheck.Snapshot(inst.HealthCheck, string(inst.State), inst.HealthCheckRuntime)) // Convert volume attachments if len(inst.Volumes) > 0 { diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 039aa917..b28fbc58 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -9,6 +9,7 @@ import ( "github.com/c2h5oh/datasize" "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/instances/phasetracking" @@ -279,6 +280,7 @@ func (m *captureUpdateManager) UpdateInstance(ctx context.Context, id string, re Image: "docker.io/library/alpine:latest", Env: req.Env, AutoStandby: req.AutoStandby, + HealthCheck: req.HealthCheck, CreatedAt: now, HypervisorType: hypervisor.TypeCloudHypervisor, }, @@ -301,6 +303,7 @@ func (m *captureCreateManager) CreateInstance(ctx context.Context, req instances OverlaySize: req.OverlaySize, Vcpus: req.Vcpus, AutoStandby: req.AutoStandby, + HealthCheck: req.HealthCheck, CreatedAt: now, HypervisorType: hypervisor.TypeCloudHypervisor, }, @@ -657,6 +660,51 @@ func TestCreateInstance_MapsAutoStandbyPolicy(t *testing.T) { assert.Equal(t, idleTimeout, *instance.AutoStandby.IdleTimeout) } +func TestCreateInstance_MapsHealthCheckPolicy(t *testing.T) { + t.Parallel() + + svc := newTestService(t) + origMgr := svc.InstanceManager + mockMgr := &captureCreateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + typ := oapi.HealthCheckTypeExec + interval := "5s" + timeout := "1s" + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-health-check", + Image: "docker.io/library/alpine:latest", + HealthCheck: &oapi.HealthCheck{ + Type: &typ, + Interval: &interval, + Timeout: &timeout, + Exec: &oapi.HealthCheckExec{ + Command: []string{"true"}, + }, + }, + }, + }) + require.NoError(t, err) + + created, ok := resp.(oapi.CreateInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.HealthCheck) + assert.Equal(t, healthcheck.TypeExec, mockMgr.lastReq.HealthCheck.Type) + assert.Equal(t, []string{"true"}, mockMgr.lastReq.HealthCheck.Exec.Command) + assert.Equal(t, "5s", mockMgr.lastReq.HealthCheck.Interval) + assert.Equal(t, "1s", mockMgr.lastReq.HealthCheck.Timeout) + + instance := oapi.Instance(created) + require.NotNil(t, instance.HealthCheck) + require.NotNil(t, instance.HealthCheck.Type) + assert.Equal(t, oapi.HealthCheckTypeExec, *instance.HealthCheck.Type) + require.NotNil(t, instance.HealthStatus) + assert.Equal(t, oapi.InstanceHealthStatusStatusStarting, instance.HealthStatus.Status) +} + func TestUpdateInstance_MapsEnvPatch(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -770,6 +818,71 @@ func TestUpdateInstance_MapsAutoStandbyPatch(t *testing.T) { assert.True(t, *instance.AutoStandby.Enabled) } +func TestUpdateInstance_MapsHealthCheckPatch(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + now := time.Now() + mockMgr := &captureUpdateManager{ + Manager: origMgr, + result: &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-health-check", + Name: "inst-update-health-check", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + HealthCheck: &healthcheck.Policy{ + Type: healthcheck.TypeTCP, + TCP: &healthcheck.TCPCheck{Port: 8080}, + }, + }, + State: instances.StateStopped, + }, + } + svc.InstanceManager = mockMgr + + typ := oapi.HealthCheckTypeTcp + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-update-health-check", + Name: "inst-update-health-check", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + NetworkEnabled: true, + }, + State: instances.StateStopped, + } + + resp, err := svc.UpdateInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.UpdateInstanceRequestObject{ + Id: resolved.Id, + Body: &oapi.UpdateInstanceRequest{ + HealthCheck: &oapi.HealthCheck{ + Type: &typ, + Tcp: &oapi.HealthCheckTCP{Port: 8080}, + }, + }, + }) + require.NoError(t, err) + updated, ok := resp.(oapi.UpdateInstance200JSONResponse) + require.True(t, ok, "expected 200 response") + + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.HealthCheck) + assert.Equal(t, resolved.Id, mockMgr.lastID) + assert.Equal(t, healthcheck.TypeTCP, mockMgr.lastReq.HealthCheck.Type) + assert.Equal(t, uint16(8080), mockMgr.lastReq.HealthCheck.TCP.Port) + + instance := oapi.Instance(updated) + require.NotNil(t, instance.HealthCheck) + require.NotNil(t, instance.HealthCheck.Type) + assert.Equal(t, oapi.HealthCheckTypeTcp, *instance.HealthCheck.Type) + require.NotNil(t, instance.HealthStatus) + assert.Equal(t, oapi.InstanceHealthStatusStatusUnknown, instance.HealthStatus.Status) +} + func TestUpdateInstance_RejectsZeroAutoStandbyIgnoreDestinationPort(t *testing.T) { t.Parallel() svc := newTestService(t) diff --git a/cmd/api/main.go b/cmd/api/main.go index ddd925a1..f7e24c4e 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -565,6 +565,12 @@ func run() error { return app.AutoStandbyController.Run(gctx) }) } + if app.HealthCheckController != nil { + grp.Go(func() error { + logger.Info("starting health check controller") + return app.HealthCheckController.Run(gctx) + }) + } // Run the server grp.Go(func() error { diff --git a/cmd/api/wire.go b/cmd/api/wire.go index 0888ec4a..2d9a7a1a 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -13,6 +13,7 @@ import ( "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" + "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/ingress" "github.com/kernel/hypeman/lib/instances" @@ -41,6 +42,7 @@ type application struct { ResourceManager *resources.Manager GuestMemoryController guestmemory.Controller AutoStandbyController *autostandby.Controller + HealthCheckController *healthcheck.Controller VMMetricsManager *vm_metrics.Manager Registry *registry.Registry ApiService *api.ApiService @@ -64,6 +66,7 @@ func initializeApp() (*application, func(), error) { providers.ProvideResourceManager, providers.ProvideGuestMemoryController, providers.ProvideAutoStandbyController, + providers.ProvideHealthCheckController, providers.ProvideVMMetricsManager, providers.ProvideRegistry, api.New, diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index 57f55c6d..b4ccc8fc 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -14,6 +14,7 @@ import ( "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" + "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/ingress" "github.com/kernel/hypeman/lib/instances" @@ -71,6 +72,7 @@ func initializeApp() (*application, func(), error) { return nil, nil, err } autostandbyController := providers.ProvideAutoStandbyController(instancesManager, logger) + healthcheckController := providers.ProvideHealthCheckController(instancesManager, logger) vm_metricsManager, err := providers.ProvideVMMetricsManager(instancesManager, config, logger) if err != nil { return nil, nil, err @@ -95,6 +97,7 @@ func initializeApp() (*application, func(), error) { ResourceManager: resourcesManager, GuestMemoryController: controller, AutoStandbyController: autostandbyController, + HealthCheckController: healthcheckController, VMMetricsManager: vm_metricsManager, Registry: registry, ApiService: apiService, @@ -121,6 +124,7 @@ type application struct { ResourceManager *resources.Manager GuestMemoryController guestmemory.Controller AutoStandbyController *autostandby.Controller + HealthCheckController *healthcheck.Controller VMMetricsManager *vm_metrics.Manager Registry *registry.Registry ApiService *api.ApiService diff --git a/lib/healthcheck/README.md b/lib/healthcheck/README.md new file mode 100644 index 00000000..dba2ba0d --- /dev/null +++ b/lib/healthcheck/README.md @@ -0,0 +1,80 @@ +# Health Checks + +Health checks track whether a running workload is healthy without changing the instance lifecycle state. + +An instance can be `Running` and `unhealthy` at the same time. `Initializing` still means Hypeman is bringing up the guest and waiting for the existing boot readiness signals. Health checks begin only after the instance reaches `Running`. + +## Check Types + +`none` disables health checks. This is the default. + +`http` sends a GET request to the instance network address on the configured port and path. The check succeeds when the response status equals `expected_status`, which defaults to `200`. + +`tcp` opens a TCP connection to the instance network address and configured port. The check succeeds when the connection opens. + +`exec` runs a command inside the guest after the guest agent is ready. Exit code `0` is healthy. A nonzero exit code, timeout, missing command, or launch error is unhealthy. + +## Timing + +Each enabled health check has: + +- `interval`: how often to run the check after the previous attempt +- `timeout`: the maximum time one check may run +- `start_period`: startup grace period before failures can mark the workload unhealthy +- `failure_threshold`: consecutive failures required to mark unhealthy +- `success_threshold`: consecutive successes required to mark healthy + +The defaults are: + +```yaml +interval: 10s +timeout: 2s +start_period: 30s +failure_threshold: 3 +success_threshold: 1 +``` + +Failures during `start_period` are recorded, but the status remains `starting`. A success during `start_period` can mark the workload `healthy`. + +Once healthy, isolated failures do not immediately flip the status. The workload becomes `unhealthy` only after `failure_threshold` consecutive failures. + +## Status + +Instance responses include `health_check` and `health_status`. + +`disabled` means no check is configured. + +`unknown` means a check is configured but the instance is not currently running. + +`starting` means the instance is running and Hypeman has not yet observed enough successful or failed checks to declare the workload healthy or unhealthy. + +`healthy` means the configured check has reached its success threshold. + +`unhealthy` means the configured check has reached its failure threshold outside the start period. + +Health status includes the last check time, last success time, last failure time, consecutive success and failure counts, and a truncated last error. + +## Lifecycle + +Health checks do not affect lifecycle state. + +Startup remains: + +```text +Initializing -> Running +``` + +With health checks enabled, the health dimension evolves separately: + +```text +unknown -> starting -> healthy +unknown -> starting -> unhealthy +``` + +Stopping, deleting, standing by, or restoring an instance stops active checks. Starting or restoring an instance begins a fresh health window. + +## Restart Policy + +Health checks only report health. They do not restart instances. + +If Hypeman later adds restart-on-unhealthy behavior, it should consume `health_status=unhealthy` explicitly rather than making health checks mutate lifecycle state. diff --git a/lib/healthcheck/controller.go b/lib/healthcheck/controller.go new file mode 100644 index 00000000..2a01e67f --- /dev/null +++ b/lib/healthcheck/controller.go @@ -0,0 +1,243 @@ +package healthcheck + +import ( + "context" + "log/slog" + "sync" + "time" +) + +const defaultTimerBufferSize = 256 + +type ControllerOptions struct { + Log *slog.Logger + Now func() time.Time +} + +type controllerState struct { + instance Instance + policy *Policy + timer *time.Timer +} + +type Controller struct { + store InstanceStore + runner ProbeRunner + log *slog.Logger + now func() time.Time + + timerFired chan string + + mu sync.Mutex + states map[string]*controllerState +} + +func NewController(store InstanceStore, runner ProbeRunner, opts ControllerOptions) *Controller { + log := opts.Log + if log == nil { + log = slog.Default() + } + now := opts.Now + if now == nil { + now = time.Now + } + if runner == nil { + runner = DefaultProbeRunner{} + } + + return &Controller{ + store: store, + runner: runner, + log: log, + now: now, + timerFired: make(chan string, defaultTimerBufferSize), + states: make(map[string]*controllerState), + } +} + +func (c *Controller) Run(ctx context.Context) error { + c.log.Info("health check controller started") + + if err := c.startupResync(ctx); err != nil { + c.log.Warn("health check startup resync failed", "error", err) + } + + events, unsubscribe, err := c.store.SubscribeInstanceEvents() + if err != nil { + return err + } + defer unsubscribe() + defer c.stopAllTimers() + + for { + select { + case <-ctx.Done(): + return nil + case event, ok := <-events: + if !ok { + return nil + } + c.handleInstanceEvent(ctx, event) + case id := <-c.timerFired: + c.runCheck(ctx, id) + } + } +} + +func (c *Controller) startupResync(ctx context.Context) error { + insts, err := c.store.ListInstances(ctx) + if err != nil { + return err + } + for _, inst := range insts { + c.syncInstance(ctx, &inst, false, false) + } + return nil +} + +func (c *Controller) handleInstanceEvent(ctx context.Context, event InstanceEvent) { + if event.Action == InstanceEventDelete || event.Instance == nil { + c.removeInstance(event.InstanceID) + return + } + resetRuntime := event.Action == InstanceEventStart || event.Action == InstanceEventRestore + c.syncInstance(ctx, event.Instance, true, resetRuntime) +} + +func (c *Controller) syncInstance(ctx context.Context, inst *Instance, immediate bool, resetRuntime bool) { + policy, err := NormalizePolicy(inst.HealthCheck) + if err != nil { + c.log.Warn("invalid health check policy", "instance_id", inst.ID, "error", err) + c.removeInstance(inst.ID) + return + } + if !Enabled(policy) || inst.State != StateRunning { + c.removeInstance(inst.ID) + return + } + + interval, _, _, err := DurationConfig(policy) + if err != nil { + c.log.Warn("invalid health check duration", "instance_id", inst.ID, "error", err) + c.removeInstance(inst.ID) + return + } + + runtime := CloneRuntime(inst.Runtime) + if resetRuntime || runtime == nil { + runtime = initialRuntime(inst, c.now()) + if err := c.store.SetRuntime(ctx, inst.ID, runtime); err != nil { + c.log.Warn("failed to persist initial health check status", "instance_id", inst.ID, "error", err) + } + } + + delay := interval + if immediate || runtime.LastCheckedAt == nil { + delay = 0 + } + + instCopy := *inst + instCopy.Runtime = runtime + + c.mu.Lock() + state, ok := c.states[inst.ID] + if !ok { + state = &controllerState{} + c.states[inst.ID] = state + } + state.instance = instCopy + state.policy = policy + c.scheduleLocked(inst.ID, state, delay) + c.mu.Unlock() +} + +func (c *Controller) removeInstance(id string) { + c.mu.Lock() + defer c.mu.Unlock() + + if state := c.states[id]; state != nil && state.timer != nil { + state.timer.Stop() + } + delete(c.states, id) +} + +func (c *Controller) runCheck(ctx context.Context, id string) { + c.mu.Lock() + state := c.states[id] + if state == nil { + c.mu.Unlock() + return + } + inst := state.instance + policy := ClonePolicy(state.policy) + previous := CloneRuntime(inst.Runtime) + c.mu.Unlock() + + _, timeout, _, err := DurationConfig(policy) + if err != nil { + c.log.Warn("invalid health check duration", "instance_id", id, "error", err) + c.removeInstance(id) + return + } + + checkCtx, cancel := context.WithTimeout(ctx, timeout) + result := c.runner.Check(checkCtx, inst, policy) + cancel() + + runtime := ApplyProbeResult(policy, inst, previous, c.now(), result) + if err := c.store.SetRuntime(ctx, id, runtime); err != nil { + c.log.Warn("failed to persist health check status", "instance_id", id, "error", err) + } + + interval, _, _, err := DurationConfig(policy) + if err != nil { + c.log.Warn("invalid health check interval", "instance_id", id, "error", err) + c.removeInstance(id) + return + } + + c.mu.Lock() + state = c.states[id] + if state != nil { + state.instance.Runtime = runtime + c.scheduleLocked(id, state, interval) + } + c.mu.Unlock() +} + +func (c *Controller) scheduleLocked(id string, state *controllerState, delay time.Duration) { + if state.timer != nil { + state.timer.Stop() + } + state.timer = time.AfterFunc(delay, func() { + select { + case c.timerFired <- id: + default: + c.log.Warn("dropped health check timer event", "instance_id", id) + } + }) +} + +func (c *Controller) stopAllTimers() { + c.mu.Lock() + defer c.mu.Unlock() + + for _, state := range c.states { + if state.timer != nil { + state.timer.Stop() + } + } + c.states = make(map[string]*controllerState) +} + +func initialRuntime(inst *Instance, now time.Time) *Runtime { + now = now.UTC() + startedAt := now + if inst.StartedAt != nil { + startedAt = inst.StartedAt.UTC() + } + return &Runtime{ + Status: StatusStarting, + StartedAt: &startedAt, + } +} diff --git a/lib/healthcheck/controller_test.go b/lib/healthcheck/controller_test.go new file mode 100644 index 00000000..74d98cd1 --- /dev/null +++ b/lib/healthcheck/controller_test.go @@ -0,0 +1,150 @@ +package healthcheck + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type controllerTestStore struct { + instances []Instance + events chan InstanceEvent + runtimes chan *Runtime +} + +func (s *controllerTestStore) ListInstances(context.Context) ([]Instance, error) { + return append([]Instance(nil), s.instances...), nil +} + +func (s *controllerTestStore) SetRuntime(_ context.Context, _ string, runtime *Runtime) error { + s.runtimes <- CloneRuntime(runtime) + return nil +} + +func (s *controllerTestStore) SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) { + return s.events, func() {}, nil +} + +type controllerTestRunner struct { + result ProbeResult +} + +func (r controllerTestRunner) Check(context.Context, Instance, *Policy) ProbeResult { + return r.result +} + +func TestControllerPersistsHealthyStatus(t *testing.T) { + policy, err := NormalizePolicy(&Policy{ + Type: TypeExec, + Interval: "1h", + Exec: &ExecCheck{Command: []string{"true"}}, + }) + require.NoError(t, err) + + now := time.Date(2026, 5, 16, 1, 0, 0, 0, time.UTC) + store := &controllerTestStore{ + instances: []Instance{{ + ID: "inst-1", + State: StateRunning, + GuestAgentReady: true, + HealthCheck: policy, + }}, + events: make(chan InstanceEvent), + runtimes: make(chan *Runtime, 4), + } + controller := NewController(store, controllerTestRunner{result: ProbeResult{Success: true}}, ControllerOptions{ + Now: func() time.Time { return now }, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + done <- controller.Run(ctx) + }() + + var healthy *Runtime + deadline := time.After(time.Second) + for healthy == nil { + select { + case runtime := <-store.runtimes: + if runtime.Status == StatusHealthy { + healthy = runtime + } + case <-deadline: + t.Fatal("timed out waiting for healthy status") + } + } + + assert.Equal(t, 1, healthy.ConsecutiveSuccesses) + + cancel() + require.NoError(t, <-done) +} + +func TestControllerResetsRuntimeOnStartEvent(t *testing.T) { + policy, err := NormalizePolicy(&Policy{ + Type: TypeExec, + Interval: "1h", + Exec: &ExecCheck{Command: []string{"true"}}, + }) + require.NoError(t, err) + + previousCheckedAt := time.Date(2026, 5, 16, 0, 0, 0, 0, time.UTC) + startedAt := time.Date(2026, 5, 16, 1, 0, 0, 0, time.UTC) + store := &controllerTestStore{ + events: make(chan InstanceEvent), + runtimes: make(chan *Runtime, 4), + } + controller := NewController(store, controllerTestRunner{result: ProbeResult{Success: true}}, ControllerOptions{ + Now: func() time.Time { return startedAt }, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + done <- controller.Run(ctx) + }() + + store.events <- InstanceEvent{ + Action: InstanceEventStart, + InstanceID: "inst-1", + Instance: &Instance{ + ID: "inst-1", + State: StateRunning, + StartedAt: &startedAt, + GuestAgentReady: true, + HealthCheck: policy, + Runtime: &Runtime{ + Status: StatusHealthy, + StartedAt: &previousCheckedAt, + LastCheckedAt: &previousCheckedAt, + ConsecutiveSuccesses: 9, + }, + }, + } + + var starting *Runtime + deadline := time.After(time.Second) + for starting == nil { + select { + case runtime := <-store.runtimes: + if runtime.Status == StatusStarting { + starting = runtime + } + case <-deadline: + t.Fatal("timed out waiting for starting status") + } + } + + assert.Equal(t, startedAt, *starting.StartedAt) + assert.Nil(t, starting.LastCheckedAt) + assert.Zero(t, starting.ConsecutiveSuccesses) + + cancel() + require.NoError(t, <-done) +} diff --git a/lib/healthcheck/policy.go b/lib/healthcheck/policy.go new file mode 100644 index 00000000..8298f323 --- /dev/null +++ b/lib/healthcheck/policy.go @@ -0,0 +1,269 @@ +package healthcheck + +import ( + "fmt" + "time" +) + +const ( + defaultInterval = 10 * time.Second + defaultTimeout = 2 * time.Second + defaultStartPeriod = 30 * time.Second + defaultFailureThreshold = 3 + defaultSuccessThreshold = 1 +) + +func ClonePolicy(policy *Policy) *Policy { + if policy == nil { + return nil + } + + cloned := &Policy{ + Type: policy.Type, + Interval: policy.Interval, + Timeout: policy.Timeout, + StartPeriod: policy.StartPeriod, + FailureThreshold: policy.FailureThreshold, + SuccessThreshold: policy.SuccessThreshold, + } + if policy.HTTP != nil { + cloned.HTTP = &HTTPCheck{ + Port: policy.HTTP.Port, + Path: policy.HTTP.Path, + Scheme: policy.HTTP.Scheme, + ExpectedStatus: policy.HTTP.ExpectedStatus, + } + } + if policy.TCP != nil { + cloned.TCP = &TCPCheck{Port: policy.TCP.Port} + } + if policy.Exec != nil { + cloned.Exec = &ExecCheck{ + Command: append([]string(nil), policy.Exec.Command...), + WorkingDir: policy.Exec.WorkingDir, + } + } + return cloned +} + +func CloneRuntime(runtime *Runtime) *Runtime { + if runtime == nil { + return nil + } + + cloned := &Runtime{ + Status: runtime.Status, + ConsecutiveSuccesses: runtime.ConsecutiveSuccesses, + ConsecutiveFailures: runtime.ConsecutiveFailures, + LastError: runtime.LastError, + } + if runtime.StartedAt != nil { + startedAt := runtime.StartedAt.UTC() + cloned.StartedAt = &startedAt + } + if runtime.LastCheckedAt != nil { + lastCheckedAt := runtime.LastCheckedAt.UTC() + cloned.LastCheckedAt = &lastCheckedAt + } + if runtime.LastSuccessAt != nil { + lastSuccessAt := runtime.LastSuccessAt.UTC() + cloned.LastSuccessAt = &lastSuccessAt + } + if runtime.LastFailureAt != nil { + lastFailureAt := runtime.LastFailureAt.UTC() + cloned.LastFailureAt = &lastFailureAt + } + return cloned +} + +func Enabled(policy *Policy) bool { + return policy != nil && policy.Type != "" && policy.Type != TypeNone +} + +func NormalizePolicy(policy *Policy) (*Policy, error) { + if policy == nil { + return nil, nil + } + + normalized := ClonePolicy(policy) + if normalized.Type == "" { + inferred, err := inferType(normalized) + if err != nil { + return nil, err + } + normalized.Type = inferred + } + if normalized.Type == "" { + normalized.Type = TypeNone + } + if normalized.Type == TypeNone { + normalized.HTTP = nil + normalized.TCP = nil + normalized.Exec = nil + return normalized, nil + } + + if normalized.Interval == "" { + normalized.Interval = defaultInterval.String() + } + if normalized.Timeout == "" { + normalized.Timeout = defaultTimeout.String() + } + if normalized.StartPeriod == "" { + normalized.StartPeriod = defaultStartPeriod.String() + } + if normalized.FailureThreshold == 0 { + normalized.FailureThreshold = defaultFailureThreshold + } + if normalized.SuccessThreshold == 0 { + normalized.SuccessThreshold = defaultSuccessThreshold + } + + interval, err := parsePositiveDuration("health_check.interval", normalized.Interval) + if err != nil { + return nil, err + } + timeout, err := parsePositiveDuration("health_check.timeout", normalized.Timeout) + if err != nil { + return nil, err + } + if timeout > interval { + return nil, fmt.Errorf("health_check.timeout must be less than or equal to health_check.interval") + } + if _, err := parseNonNegativeDuration("health_check.start_period", normalized.StartPeriod); err != nil { + return nil, err + } + if normalized.FailureThreshold < 1 { + return nil, fmt.Errorf("health_check.failure_threshold must be at least 1") + } + if normalized.SuccessThreshold < 1 { + return nil, fmt.Errorf("health_check.success_threshold must be at least 1") + } + + switch normalized.Type { + case TypeHTTP: + if normalized.HTTP == nil { + return nil, fmt.Errorf("health_check.http is required when type is http") + } + if normalized.TCP != nil || normalized.Exec != nil { + return nil, fmt.Errorf("health_check type http cannot include tcp or exec checks") + } + if normalized.HTTP.Port == 0 { + return nil, fmt.Errorf("health_check.http.port must be between 1 and 65535") + } + if normalized.HTTP.Path == "" { + normalized.HTTP.Path = "/" + } + if normalized.HTTP.Path[0] != '/' { + return nil, fmt.Errorf("health_check.http.path must start with /") + } + if normalized.HTTP.Scheme == "" { + normalized.HTTP.Scheme = "http" + } + if normalized.HTTP.Scheme != "http" && normalized.HTTP.Scheme != "https" { + return nil, fmt.Errorf("health_check.http.scheme must be http or https") + } + if normalized.HTTP.ExpectedStatus == 0 { + normalized.HTTP.ExpectedStatus = 200 + } + if normalized.HTTP.ExpectedStatus < 100 || normalized.HTTP.ExpectedStatus > 599 { + return nil, fmt.Errorf("health_check.http.expected_status must be between 100 and 599") + } + case TypeTCP: + if normalized.TCP == nil { + return nil, fmt.Errorf("health_check.tcp is required when type is tcp") + } + if normalized.HTTP != nil || normalized.Exec != nil { + return nil, fmt.Errorf("health_check type tcp cannot include http or exec checks") + } + if normalized.TCP.Port == 0 { + return nil, fmt.Errorf("health_check.tcp.port must be between 1 and 65535") + } + case TypeExec: + if normalized.Exec == nil { + return nil, fmt.Errorf("health_check.exec is required when type is exec") + } + if normalized.HTTP != nil || normalized.TCP != nil { + return nil, fmt.Errorf("health_check type exec cannot include http or tcp checks") + } + if len(normalized.Exec.Command) == 0 { + return nil, fmt.Errorf("health_check.exec.command must not be empty") + } + default: + return nil, fmt.Errorf("health_check.type must be one of none, http, tcp, exec") + } + + return normalized, nil +} + +func DurationConfig(policy *Policy) (interval, timeout, startPeriod time.Duration, err error) { + if policy == nil { + return defaultInterval, defaultTimeout, defaultStartPeriod, nil + } + interval, err = parseDurationOrDefault(policy.Interval, defaultInterval) + if err != nil { + return 0, 0, 0, err + } + timeout, err = parseDurationOrDefault(policy.Timeout, defaultTimeout) + if err != nil { + return 0, 0, 0, err + } + startPeriod, err = parseDurationOrDefault(policy.StartPeriod, defaultStartPeriod) + if err != nil { + return 0, 0, 0, err + } + return interval, timeout, startPeriod, nil +} + +func inferType(policy *Policy) (Type, error) { + var inferred Type + count := 0 + if policy.HTTP != nil { + inferred = TypeHTTP + count++ + } + if policy.TCP != nil { + inferred = TypeTCP + count++ + } + if policy.Exec != nil { + inferred = TypeExec + count++ + } + if count == 1 { + return inferred, nil + } + if count > 1 { + return "", fmt.Errorf("health_check must include exactly one of http, tcp, or exec when type is omitted") + } + return TypeNone, nil +} + +func parseDurationOrDefault(value string, fallback time.Duration) (time.Duration, error) { + if value == "" { + return fallback, nil + } + return time.ParseDuration(value) +} + +func parsePositiveDuration(name, value string) (time.Duration, error) { + parsed, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("%s is invalid: %w", name, err) + } + if parsed <= 0 { + return 0, fmt.Errorf("%s must be positive", name) + } + return parsed, nil +} + +func parseNonNegativeDuration(name, value string) (time.Duration, error) { + parsed, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("%s is invalid: %w", name, err) + } + if parsed < 0 { + return 0, fmt.Errorf("%s must not be negative", name) + } + return parsed, nil +} diff --git a/lib/healthcheck/policy_test.go b/lib/healthcheck/policy_test.go new file mode 100644 index 00000000..7f6c0638 --- /dev/null +++ b/lib/healthcheck/policy_test.go @@ -0,0 +1,79 @@ +package healthcheck + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizePolicyInfersHTTPDefaults(t *testing.T) { + policy, err := NormalizePolicy(&Policy{ + HTTP: &HTTPCheck{Port: 8080}, + }) + require.NoError(t, err) + + assert.Equal(t, TypeHTTP, policy.Type) + assert.Equal(t, "10s", policy.Interval) + assert.Equal(t, "2s", policy.Timeout) + assert.Equal(t, "30s", policy.StartPeriod) + assert.Equal(t, 3, policy.FailureThreshold) + assert.Equal(t, 1, policy.SuccessThreshold) + assert.Equal(t, "/", policy.HTTP.Path) + assert.Equal(t, "http", policy.HTTP.Scheme) + assert.Equal(t, 200, policy.HTTP.ExpectedStatus) +} + +func TestNormalizePolicyRejectsExecWithoutCommand(t *testing.T) { + _, err := NormalizePolicy(&Policy{ + Type: TypeExec, + Exec: &ExecCheck{}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "exec.command") +} + +func TestNormalizePolicyRejectsAmbiguousChecks(t *testing.T) { + _, err := NormalizePolicy(&Policy{ + HTTP: &HTTPCheck{Port: 8080}, + TCP: &TCPCheck{Port: 8080}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "exactly one") +} + +func TestNormalizePolicyRejectsMismatchedChecks(t *testing.T) { + _, err := NormalizePolicy(&Policy{ + Type: TypeHTTP, + HTTP: &HTTPCheck{Port: 8080}, + Exec: &ExecCheck{Command: []string{"true"}}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot include") +} + +func TestNormalizePolicyClearsDisabledChecks(t *testing.T) { + policy, err := NormalizePolicy(&Policy{ + Type: TypeNone, + HTTP: &HTTPCheck{Port: 8080}, + }) + require.NoError(t, err) + + assert.Equal(t, TypeNone, policy.Type) + assert.Nil(t, policy.HTTP) +} + +func TestNormalizePolicyRejectsTimeoutLongerThanInterval(t *testing.T) { + _, err := NormalizePolicy(&Policy{ + Type: TypeTCP, + Interval: "1s", + Timeout: "2s", + TCP: &TCPCheck{Port: 443}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "timeout") +} diff --git a/lib/healthcheck/probe.go b/lib/healthcheck/probe.go new file mode 100644 index 00000000..8a6eae13 --- /dev/null +++ b/lib/healthcheck/probe.go @@ -0,0 +1,106 @@ +package healthcheck + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "strconv" + "time" +) + +type ProbeRunner interface { + Check(ctx context.Context, inst Instance, policy *Policy) ProbeResult +} + +type ExecRunner interface { + Run(ctx context.Context, inst Instance, check ExecCheck, timeout time.Duration) error +} + +type DefaultProbeRunner struct { + HTTPClient *http.Client + ExecRunner ExecRunner +} + +func (r DefaultProbeRunner) Check(ctx context.Context, inst Instance, policy *Policy) ProbeResult { + switch policy.Type { + case TypeHTTP: + return r.checkHTTP(ctx, inst, *policy.HTTP) + case TypeTCP: + return r.checkTCP(ctx, inst, *policy.TCP) + case TypeExec: + return r.checkExec(ctx, inst, *policy.Exec, policy) + default: + return ProbeResult{Success: false, Error: "health check is disabled"} + } +} + +func (r DefaultProbeRunner) checkHTTP(ctx context.Context, inst Instance, check HTTPCheck) ProbeResult { + if !inst.NetworkEnabled || inst.IP == "" { + return ProbeResult{Success: false, Error: "instance has no network address"} + } + + u := url.URL{ + Scheme: check.Scheme, + Host: net.JoinHostPort(inst.IP, strconv.Itoa(int(check.Port))), + Path: check.Path, + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return ProbeResult{Success: false, Error: err.Error()} + } + + client := r.HTTPClient + if client == nil { + client = http.DefaultClient + } + resp, err := client.Do(req) + if err != nil { + return ProbeResult{Success: false, Error: err.Error()} + } + defer resp.Body.Close() + + if resp.StatusCode != check.ExpectedStatus { + return ProbeResult{ + Success: false, + Error: fmt.Sprintf("expected HTTP status %d, got %d", check.ExpectedStatus, resp.StatusCode), + } + } + return ProbeResult{Success: true} +} + +func (r DefaultProbeRunner) checkTCP(ctx context.Context, inst Instance, check TCPCheck) ProbeResult { + if !inst.NetworkEnabled || inst.IP == "" { + return ProbeResult{Success: false, Error: "instance has no network address"} + } + + var dialer net.Dialer + conn, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(inst.IP, strconv.Itoa(int(check.Port)))) + if err != nil { + return ProbeResult{Success: false, Error: err.Error()} + } + _ = conn.Close() + return ProbeResult{Success: true} +} + +func (r DefaultProbeRunner) checkExec(ctx context.Context, inst Instance, check ExecCheck, policy *Policy) ProbeResult { + if r.ExecRunner == nil { + return ProbeResult{Success: false, Error: "exec health checks are unavailable"} + } + if inst.SkipGuestAgent { + return ProbeResult{Success: false, Error: "exec health checks require guest-agent"} + } + if !inst.GuestAgentReady { + return ProbeResult{Success: false, Error: "guest-agent is not ready"} + } + + _, timeout, _, err := DurationConfig(policy) + if err != nil { + return ProbeResult{Success: false, Error: err.Error()} + } + if err := r.ExecRunner.Run(ctx, inst, check, timeout); err != nil { + return ProbeResult{Success: false, Error: err.Error()} + } + return ProbeResult{Success: true} +} diff --git a/lib/healthcheck/status.go b/lib/healthcheck/status.go new file mode 100644 index 00000000..f79682ed --- /dev/null +++ b/lib/healthcheck/status.go @@ -0,0 +1,109 @@ +package healthcheck + +import "time" + +const maxLastErrorLength = 512 + +type ProbeResult struct { + Success bool + Error string +} + +func Snapshot(policy *Policy, state string, runtime *Runtime) StatusSnapshot { + if !Enabled(policy) { + return StatusSnapshot{Status: StatusDisabled} + } + if state != StateRunning { + return StatusSnapshot{Status: StatusUnknown} + } + if runtime == nil || runtime.Status == "" { + return StatusSnapshot{Status: StatusStarting} + } + return StatusSnapshot{ + Status: runtime.Status, + ConsecutiveSuccesses: runtime.ConsecutiveSuccesses, + ConsecutiveFailures: runtime.ConsecutiveFailures, + LastCheckedAt: cloneTime(runtime.LastCheckedAt), + LastSuccessAt: cloneTime(runtime.LastSuccessAt), + LastFailureAt: cloneTime(runtime.LastFailureAt), + LastError: runtime.LastError, + } +} + +func ApplyProbeResult(policy *Policy, inst Instance, previous *Runtime, now time.Time, result ProbeResult) *Runtime { + runtime := CloneRuntime(previous) + if runtime == nil { + runtime = &Runtime{} + } + + now = now.UTC() + if runtime.StartedAt == nil { + startedAt := now + if inst.StartedAt != nil { + startedAt = inst.StartedAt.UTC() + } + runtime.StartedAt = &startedAt + } + runtime.LastCheckedAt = &now + + _, _, startPeriod, err := DurationConfig(policy) + if err != nil { + startPeriod = defaultStartPeriod + } + inStartPeriod := runtime.StartedAt != nil && now.Sub(*runtime.StartedAt) < startPeriod + + if result.Success { + runtime.ConsecutiveSuccesses++ + runtime.ConsecutiveFailures = 0 + runtime.LastSuccessAt = &now + runtime.LastError = "" + successThreshold := defaultSuccessThreshold + if policy != nil { + successThreshold = policy.SuccessThreshold + } + if successThreshold == 0 { + successThreshold = defaultSuccessThreshold + } + if runtime.ConsecutiveSuccesses >= successThreshold { + runtime.Status = StatusHealthy + } else if runtime.Status == "" { + runtime.Status = StatusStarting + } + return runtime + } + + runtime.ConsecutiveFailures++ + runtime.ConsecutiveSuccesses = 0 + runtime.LastFailureAt = &now + runtime.LastError = truncateLastError(result.Error) + + failureThreshold := defaultFailureThreshold + if policy != nil { + failureThreshold = policy.FailureThreshold + } + if failureThreshold == 0 { + failureThreshold = defaultFailureThreshold + } + if !inStartPeriod && runtime.ConsecutiveFailures >= failureThreshold { + runtime.Status = StatusUnhealthy + } else if runtime.Status == "" { + runtime.Status = StatusStarting + } + + return runtime +} + +func cloneTime(t *time.Time) *time.Time { + if t == nil { + return nil + } + cloned := t.UTC() + return &cloned +} + +func truncateLastError(value string) string { + if len(value) <= maxLastErrorLength { + return value + } + return value[:maxLastErrorLength] +} diff --git a/lib/healthcheck/status_test.go b/lib/healthcheck/status_test.go new file mode 100644 index 00000000..87f5e3f4 --- /dev/null +++ b/lib/healthcheck/status_test.go @@ -0,0 +1,65 @@ +package healthcheck + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestApplyProbeResultKeepsFailuresInStartPeriodStarting(t *testing.T) { + startedAt := time.Date(2026, 5, 16, 1, 0, 0, 0, time.UTC) + now := startedAt.Add(5 * time.Second) + policy, err := NormalizePolicy(&Policy{ + Type: TypeTCP, + StartPeriod: "30s", + FailureThreshold: 1, + TCP: &TCPCheck{Port: 8080}, + }) + require.NoError(t, err) + + runtime := ApplyProbeResult(policy, Instance{ + State: StateRunning, + StartedAt: &startedAt, + }, nil, now, ProbeResult{Success: false, Error: "connection refused"}) + + assert.Equal(t, StatusStarting, runtime.Status) + assert.Equal(t, 1, runtime.ConsecutiveFailures) + assert.Equal(t, "connection refused", runtime.LastError) +} + +func TestApplyProbeResultMarksUnhealthyAfterThreshold(t *testing.T) { + startedAt := time.Date(2026, 5, 16, 1, 0, 0, 0, time.UTC) + now := startedAt.Add(time.Minute) + policy, err := NormalizePolicy(&Policy{ + Type: TypeTCP, + StartPeriod: "10s", + FailureThreshold: 2, + TCP: &TCPCheck{Port: 8080}, + }) + require.NoError(t, err) + + runtime := ApplyProbeResult(policy, Instance{State: StateRunning, StartedAt: &startedAt}, nil, now, ProbeResult{Success: false, Error: "first"}) + assert.Equal(t, StatusStarting, runtime.Status) + + runtime = ApplyProbeResult(policy, Instance{State: StateRunning, StartedAt: &startedAt}, runtime, now.Add(10*time.Second), ProbeResult{Success: false, Error: "second"}) + assert.Equal(t, StatusUnhealthy, runtime.Status) + assert.Equal(t, 2, runtime.ConsecutiveFailures) +} + +func TestApplyProbeResultMarksHealthyOnSuccess(t *testing.T) { + now := time.Date(2026, 5, 16, 1, 0, 0, 0, time.UTC) + policy, err := NormalizePolicy(&Policy{ + Type: TypeExec, + Exec: &ExecCheck{Command: []string{"true"}}, + }) + require.NoError(t, err) + + runtime := ApplyProbeResult(policy, Instance{State: StateRunning}, nil, now, ProbeResult{Success: true}) + + assert.Equal(t, StatusHealthy, runtime.Status) + assert.Equal(t, 1, runtime.ConsecutiveSuccesses) + assert.Zero(t, runtime.ConsecutiveFailures) + assert.Empty(t, runtime.LastError) +} diff --git a/lib/healthcheck/types.go b/lib/healthcheck/types.go new file mode 100644 index 00000000..9ebc62e8 --- /dev/null +++ b/lib/healthcheck/types.go @@ -0,0 +1,114 @@ +package healthcheck + +import ( + "context" + "time" +) + +const StateRunning = "Running" + +type Type string + +const ( + TypeNone Type = "none" + TypeHTTP Type = "http" + TypeTCP Type = "tcp" + TypeExec Type = "exec" +) + +type Status string + +const ( + StatusDisabled Status = "disabled" + StatusStarting Status = "starting" + StatusHealthy Status = "healthy" + StatusUnhealthy Status = "unhealthy" + StatusUnknown Status = "unknown" +) + +type Policy struct { + Type Type `json:"type,omitempty"` + Interval string `json:"interval,omitempty"` + Timeout string `json:"timeout,omitempty"` + StartPeriod string `json:"start_period,omitempty"` + FailureThreshold int `json:"failure_threshold,omitempty"` + SuccessThreshold int `json:"success_threshold,omitempty"` + HTTP *HTTPCheck `json:"http,omitempty"` + TCP *TCPCheck `json:"tcp,omitempty"` + Exec *ExecCheck `json:"exec,omitempty"` +} + +type HTTPCheck struct { + Port uint16 `json:"port"` + Path string `json:"path,omitempty"` + Scheme string `json:"scheme,omitempty"` + ExpectedStatus int `json:"expected_status,omitempty"` +} + +type TCPCheck struct { + Port uint16 `json:"port"` +} + +type ExecCheck struct { + Command []string `json:"command,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` +} + +type Runtime struct { + Status Status `json:"status,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + ConsecutiveSuccesses int `json:"consecutive_successes,omitempty"` + ConsecutiveFailures int `json:"consecutive_failures,omitempty"` + LastCheckedAt *time.Time `json:"last_checked_at,omitempty"` + LastSuccessAt *time.Time `json:"last_success_at,omitempty"` + LastFailureAt *time.Time `json:"last_failure_at,omitempty"` + LastError string `json:"last_error,omitempty"` +} + +type StatusSnapshot struct { + Status Status + ConsecutiveSuccesses int + ConsecutiveFailures int + LastCheckedAt *time.Time + LastSuccessAt *time.Time + LastFailureAt *time.Time + LastError string +} + +type Instance struct { + ID string + Name string + State string + NetworkEnabled bool + IP string + StartedAt *time.Time + GuestAgentReady bool + SkipGuestAgent bool + HealthCheck *Policy + Runtime *Runtime +} + +type InstanceEventAction string + +const ( + InstanceEventCreate InstanceEventAction = "create" + InstanceEventUpdate InstanceEventAction = "update" + InstanceEventStart InstanceEventAction = "start" + InstanceEventStop InstanceEventAction = "stop" + InstanceEventStandby InstanceEventAction = "standby" + InstanceEventRestore InstanceEventAction = "restore" + InstanceEventDelete InstanceEventAction = "delete" + InstanceEventFork InstanceEventAction = "fork" +) + +type InstanceEvent struct { + Action InstanceEventAction + InstanceID string + Instance *Instance +} + +type InstanceStore interface { + ListInstances(ctx context.Context) ([]Instance, error) + SetRuntime(ctx context.Context, id string, runtime *Runtime) error + SubscribeInstanceEvents() (<-chan InstanceEvent, func(), error) +} diff --git a/lib/instances/create.go b/lib/instances/create.go index 23338f19..dc2f57b0 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -374,6 +374,7 @@ func (m *manager) createInstance( SkipGuestAgent: req.SkipGuestAgent, SnapshotPolicy: cloneSnapshotPolicy(req.SnapshotPolicy), AutoStandby: cloneAutoStandbyPolicy(req.AutoStandby), + HealthCheck: cloneHealthCheckPolicy(req.HealthCheck), } // 12. Ensure directories @@ -629,6 +630,14 @@ func validateCreateRequest(req *CreateInstanceRequest) error { return err } req.AutoStandby = normalizedAutoStandby + normalizedHealthCheck, err := normalizeHealthCheckPolicy(req.HealthCheck) + if err != nil { + return err + } + req.HealthCheck = normalizedHealthCheck + if err := validateHealthCheckCompatibility(req.HealthCheck, req.NetworkEnabled, req.SkipGuestAgent); err != nil { + return err + } // Validate volume attachments if err := validateVolumeAttachments(req.Volumes); err != nil { diff --git a/lib/instances/create_health_check_test.go b/lib/instances/create_health_check_test.go new file mode 100644 index 00000000..1a0d4443 --- /dev/null +++ b/lib/instances/create_health_check_test.go @@ -0,0 +1,45 @@ +package instances + +import ( + "testing" + + "github.com/kernel/hypeman/lib/healthcheck" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateCreateRequestHealthCheck(t *testing.T) { + t.Run("normalizes exec health check", func(t *testing.T) { + req := &CreateInstanceRequest{ + Name: "health-exec", + Image: "docker.io/library/alpine:latest", + HealthCheck: &healthcheck.Policy{ + Type: healthcheck.TypeExec, + Exec: &healthcheck.ExecCheck{Command: []string{"true"}}, + }, + } + + err := validateCreateRequest(req) + require.NoError(t, err) + require.NotNil(t, req.HealthCheck) + assert.Equal(t, "10s", req.HealthCheck.Interval) + assert.Equal(t, "2s", req.HealthCheck.Timeout) + }) + + t.Run("rejects network health check without networking", func(t *testing.T) { + req := &CreateInstanceRequest{ + Name: "health-tcp", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: false, + HealthCheck: &healthcheck.Policy{ + Type: healthcheck.TypeTCP, + TCP: &healthcheck.TCPCheck{Port: 8080}, + }, + } + + err := validateCreateRequest(req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "network.enabled") + }) +} diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 1bb4102f..d515b051 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -501,6 +501,9 @@ func cloneStoredMetadata(src StoredMetadata) StoredMetadata { if src.AutoStandby != nil { dst.AutoStandby = cloneAutoStandbyPolicy(src.AutoStandby) } + if src.HealthCheck != nil { + dst.HealthCheck = cloneHealthCheckPolicy(src.HealthCheck) + } if src.SnapshotPolicy != nil { dst.SnapshotPolicy = cloneSnapshotPolicy(src.SnapshotPolicy) } diff --git a/lib/instances/health_check.go b/lib/instances/health_check.go new file mode 100644 index 00000000..437fd5c0 --- /dev/null +++ b/lib/instances/health_check.go @@ -0,0 +1,60 @@ +package instances + +import ( + "context" + "fmt" + + "github.com/kernel/hypeman/lib/healthcheck" +) + +func cloneHealthCheckPolicy(policy *healthcheck.Policy) *healthcheck.Policy { + return healthcheck.ClonePolicy(policy) +} + +func normalizeHealthCheckPolicy(policy *healthcheck.Policy) (*healthcheck.Policy, error) { + normalized, err := healthcheck.NormalizePolicy(policy) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + return normalized, nil +} + +func validateHealthCheckCompatibility(policy *healthcheck.Policy, networkEnabled bool, skipGuestAgent bool) error { + if !healthcheck.Enabled(policy) { + return nil + } + switch policy.Type { + case healthcheck.TypeHTTP, healthcheck.TypeTCP: + if !networkEnabled { + return fmt.Errorf("%w: %s health checks require network.enabled=true", ErrInvalidRequest, policy.Type) + } + case healthcheck.TypeExec: + if skipGuestAgent { + return fmt.Errorf("%w: exec health checks require skip_guest_agent=false", ErrInvalidRequest) + } + } + return nil +} + +// GetHealthCheckRuntime returns persisted health check runtime status. +func (m *manager) GetHealthCheckRuntime(_ context.Context, id string) (*healthcheck.Runtime, error) { + meta, err := m.loadMetadata(id) + if err != nil { + return nil, err + } + return healthcheck.CloneRuntime(meta.HealthCheckRuntime), nil +} + +// SetHealthCheckRuntime persists health check runtime status. +func (m *manager) SetHealthCheckRuntime(_ context.Context, id string, runtime *healthcheck.Runtime) error { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + + meta, err := m.loadMetadata(id) + if err != nil { + return err + } + meta.HealthCheckRuntime = healthcheck.CloneRuntime(runtime) + return m.saveMetadata(meta) +} diff --git a/lib/instances/lifecycle_events.go b/lib/instances/lifecycle_events.go index 5ae42024..4bba4851 100644 --- a/lib/instances/lifecycle_events.go +++ b/lib/instances/lifecycle_events.go @@ -14,11 +14,13 @@ type LifecycleEventConsumer string const ( LifecycleEventConsumerWaitForState LifecycleEventConsumer = "wait_for_state" LifecycleEventConsumerAutoStandby LifecycleEventConsumer = "auto_standby" + LifecycleEventConsumerHealthCheck LifecycleEventConsumer = "health_check" ) var allLifecycleEventConsumers = []LifecycleEventConsumer{ LifecycleEventConsumerWaitForState, LifecycleEventConsumerAutoStandby, + LifecycleEventConsumerHealthCheck, } // LifecycleEventAction identifies which instance lifecycle action occurred. diff --git a/lib/instances/metadata_clone.go b/lib/instances/metadata_clone.go index d6efa4b5..70742edc 100644 --- a/lib/instances/metadata_clone.go +++ b/lib/instances/metadata_clone.go @@ -1,5 +1,7 @@ package instances +import "github.com/kernel/hypeman/lib/healthcheck" + // deepCopyMetadata returns a metadata copy that can be mutated without // affecting the originally loaded instance metadata. func deepCopyMetadata(src *metadata) *metadata { @@ -10,5 +12,6 @@ func deepCopyMetadata(src *metadata) *metadata { return &metadata{ StoredMetadata: cloneStoredMetadata(src.StoredMetadata), AutoStandbyRuntime: cloneAutoStandbyRuntime(src.AutoStandbyRuntime), + HealthCheckRuntime: healthcheck.CloneRuntime(src.HealthCheckRuntime), } } diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index cad949fb..72f4f578 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -219,6 +219,8 @@ func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T assert.Equal(t, int64(1), point.Value) case string(LifecycleEventConsumerAutoStandby): assert.Equal(t, int64(1), point.Value) + case string(LifecycleEventConsumerHealthCheck): + assert.Equal(t, int64(0), point.Value) default: t.Fatalf("unexpected consumer label: %s", metricLabel(t, point.Attributes, "consumer")) } @@ -234,6 +236,8 @@ func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T assert.Equal(t, int64(m.lifecycleEvents.bufferSize), point.Value) case string(LifecycleEventConsumerAutoStandby): assert.Equal(t, int64(m.lifecycleEvents.bufferSize), point.Value) + case string(LifecycleEventConsumerHealthCheck): + assert.Equal(t, int64(0), point.Value) default: t.Fatalf("unexpected consumer label: %s", metricLabel(t, point.Attributes, "consumer")) } diff --git a/lib/instances/query.go b/lib/instances/query.go index 079c5b03..eba80c8b 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/logger" @@ -503,6 +504,7 @@ func (m *manager) toInstanceWithStateDerivation(ctx context.Context, meta *metad StateError: result.Error, HasSnapshot: m.hasSnapshot(meta.StoredMetadata.DataDir), BootMarkersHydrated: result.BootMarkersHydrated, + HealthCheckRuntime: healthcheck.CloneRuntime(meta.HealthCheckRuntime), } refreshHypervisorPID(&inst.StoredMetadata, result.State) diff --git a/lib/instances/storage.go b/lib/instances/storage.go index f290298c..d07baea6 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -10,6 +10,7 @@ import ( "time" "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/images" ) @@ -38,6 +39,7 @@ const ( type metadata struct { StoredMetadata AutoStandbyRuntime *autostandby.Runtime `json:"auto_standby_runtime,omitempty"` + HealthCheckRuntime *healthcheck.Runtime `json:"health_check_runtime,omitempty"` } // ensureDirectories creates the instance directory structure diff --git a/lib/instances/types.go b/lib/instances/types.go index 04a315e3..0fe2fd04 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/snapshot" @@ -164,6 +165,9 @@ type StoredMetadata struct { // Automatic standby policy driven by host-observed inbound TCP activity. AutoStandby *autostandby.Policy + // Workload health check policy. Health is reported separately from lifecycle state. + HealthCheck *healthcheck.Policy + // Shutdown configuration StopTimeout int // Grace period in seconds for graceful stop (0 = use default 5s) @@ -187,6 +191,7 @@ type Instance struct { StateError *string // Error message if state couldn't be determined (non-nil when State=Unknown) HasSnapshot bool // Derived from filesystem check BootMarkersHydrated bool // True when missing boot markers were hydrated from logs in this read + HealthCheckRuntime *healthcheck.Runtime } // GetHypervisorType returns the hypervisor type as a string. @@ -253,6 +258,7 @@ type CreateInstanceRequest struct { SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) SnapshotPolicy *SnapshotPolicy // Optional snapshot policy defaults for this instance AutoStandby *autostandby.Policy // Optional automatic standby policy + HealthCheck *healthcheck.Policy // Optional workload health check policy } // StartInstanceRequest is the domain request for starting a stopped instance @@ -265,6 +271,7 @@ type StartInstanceRequest struct { type UpdateInstanceRequest struct { Env map[string]string // Updated environment variables (merged with existing) AutoStandby *autostandby.Policy // Replaces the persisted auto-standby policy when non-nil + HealthCheck *healthcheck.Policy // Replaces the persisted health check policy when non-nil } // ForkInstanceRequest is the domain request for forking an instance. diff --git a/lib/instances/update.go b/lib/instances/update.go index f8556219..d8f97833 100644 --- a/lib/instances/update.go +++ b/lib/instances/update.go @@ -36,6 +36,11 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta return nil, err } req.AutoStandby = normalizedAutoStandby + normalizedHealthCheck, err := normalizeHealthCheckPolicy(req.HealthCheck) + if err != nil { + return nil, err + } + req.HealthCheck = normalizedHealthCheck if err := validateUpdateInstanceRequest(meta, req); err != nil { return nil, err @@ -47,6 +52,10 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta if req.AutoStandby != nil { nextMeta.AutoStandby = cloneAutoStandbyPolicy(req.AutoStandby) } + if req.HealthCheck != nil { + nextMeta.HealthCheck = cloneHealthCheckPolicy(req.HealthCheck) + nextMeta.HealthCheckRuntime = nil + } if len(req.Env) == 0 { if err := m.saveMetadata(nextMeta); err != nil { return nil, fmt.Errorf("save metadata: %w", err) @@ -94,8 +103,16 @@ func (m *manager) updateInstance(ctx context.Context, id string, req UpdateInsta } func validateUpdateInstanceRequest(meta *metadata, req UpdateInstanceRequest) error { - if len(req.Env) == 0 && req.AutoStandby == nil { - return fmt.Errorf("%w: request must include env and/or auto_standby", ErrInvalidRequest) + if len(req.Env) == 0 && req.AutoStandby == nil && req.HealthCheck == nil { + return fmt.Errorf("%w: request must include env, auto_standby, and/or health_check", ErrInvalidRequest) + } + if req.HealthCheck != nil { + if meta == nil { + return fmt.Errorf("%w: instance metadata is required", ErrInvalidRequest) + } + if err := validateHealthCheckCompatibility(req.HealthCheck, meta.NetworkEnabled, meta.SkipGuestAgent); err != nil { + return err + } } if len(req.Env) == 0 { return nil diff --git a/lib/instances/update_test.go b/lib/instances/update_test.go index 6230f5a6..3ec0e55c 100644 --- a/lib/instances/update_test.go +++ b/lib/instances/update_test.go @@ -8,6 +8,7 @@ import ( "github.com/kernel/hypeman/lib/autostandby" "github.com/kernel/hypeman/lib/egressproxy" + "github.com/kernel/hypeman/lib/healthcheck" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,7 +30,7 @@ func TestValidateUpdateInstanceRequest(t *testing.T) { err := validateUpdateInstanceRequest(baseMeta, UpdateInstanceRequest{}) require.Error(t, err) assert.ErrorIs(t, err, ErrInvalidRequest) - assert.Contains(t, err.Error(), "env and/or auto_standby") + assert.Contains(t, err.Error(), "env, auto_standby, and/or health_check") }) t.Run("rejects instances without credential backed envs", func(t *testing.T) { @@ -67,6 +68,28 @@ func TestValidateUpdateInstanceRequest(t *testing.T) { }) require.NoError(t, err) }) + + t.Run("allows exec health check without env changes", func(t *testing.T) { + err := validateUpdateInstanceRequest(baseMeta, UpdateInstanceRequest{ + HealthCheck: &healthcheck.Policy{ + Type: healthcheck.TypeExec, + Exec: &healthcheck.ExecCheck{Command: []string{"true"}}, + }, + }) + require.NoError(t, err) + }) + + t.Run("rejects network health check without networking", func(t *testing.T) { + err := validateUpdateInstanceRequest(&metadata{}, UpdateInstanceRequest{ + HealthCheck: &healthcheck.Policy{ + Type: healthcheck.TypeTCP, + TCP: &healthcheck.TCPCheck{Port: 8080}, + }, + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "network.enabled") + }) } type fakeUpdateInstanceRulesService struct { @@ -324,3 +347,57 @@ func TestManagerUpdateInstanceAutoStandbyOnlyPublishesLifecycleUpdate(t *testing t.Fatal("timed out waiting for lifecycle update event") } } + +func TestManagerUpdateInstanceHealthCheckOnlyPublishesLifecycleUpdate(t *testing.T) { + t.Parallel() + + manager, _ := setupTestManager(t) + id := "inst-health-check-update" + require.NoError(t, manager.ensureDirectories(id)) + meta := &metadata{ + StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + CreatedAt: time.Now(), + DataDir: manager.paths.InstanceDir(id), + SocketPath: manager.paths.InstanceSocket(id, "cloud-hypervisor.sock"), + NetworkEnabled: true, + HealthCheck: &healthcheck.Policy{ + Type: healthcheck.TypeExec, + Exec: &healthcheck.ExecCheck{Command: []string{"true"}}, + }, + }, + HealthCheckRuntime: &healthcheck.Runtime{ + Status: healthcheck.StatusHealthy, + ConsecutiveSuccesses: 3, + }, + } + require.NoError(t, manager.saveMetadata(meta)) + + events, unsubscribe := manager.SubscribeLifecycleEvents(LifecycleEventConsumerHealthCheck) + defer unsubscribe() + + updated, err := manager.UpdateInstance(context.Background(), id, UpdateInstanceRequest{ + HealthCheck: &healthcheck.Policy{ + Type: healthcheck.TypeTCP, + TCP: &healthcheck.TCPCheck{Port: 8080}, + }, + }) + require.NoError(t, err) + require.NotNil(t, updated) + require.NotNil(t, updated.HealthCheck) + assert.Equal(t, healthcheck.TypeTCP, updated.HealthCheck.Type) + assert.Nil(t, updated.HealthCheckRuntime) + + select { + case event := <-events: + assert.Equal(t, LifecycleEventUpdate, event.Action) + assert.Equal(t, id, event.InstanceID) + require.NotNil(t, event.Instance) + require.NotNil(t, event.Instance.HealthCheck) + assert.Equal(t, healthcheck.TypeTCP, event.Instance.HealthCheck.Type) + assert.Nil(t, event.Instance.HealthCheckRuntime) + case <-time.After(time.Second): + t.Fatal("timed out waiting for lifecycle update event") + } +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 9bb980a6..bd645a3b 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -119,6 +119,20 @@ const ( Ok HealthStatus = "ok" ) +// Defines values for HealthCheckType. +const ( + HealthCheckTypeExec HealthCheckType = "exec" + HealthCheckTypeHttp HealthCheckType = "http" + HealthCheckTypeNone HealthCheckType = "none" + HealthCheckTypeTcp HealthCheckType = "tcp" +) + +// Defines values for HealthCheckHTTPScheme. +const ( + HealthCheckHTTPSchemeHttp HealthCheckHTTPScheme = "http" + HealthCheckHTTPSchemeHttps HealthCheckHTTPScheme = "https" +) + // Defines values for ImageStatus. const ( ImageStatusConverting ImageStatus = "converting" @@ -136,6 +150,15 @@ const ( InstanceHypervisorVz InstanceHypervisor = "vz" ) +// Defines values for InstanceHealthStatusStatus. +const ( + InstanceHealthStatusStatusDisabled InstanceHealthStatusStatus = "disabled" + InstanceHealthStatusStatusHealthy InstanceHealthStatusStatus = "healthy" + InstanceHealthStatusStatusStarting InstanceHealthStatusStatus = "starting" + InstanceHealthStatusStatusUnhealthy InstanceHealthStatusStatus = "unhealthy" + InstanceHealthStatusStatusUnknown InstanceHealthStatusStatus = "unknown" +) + // Defines values for InstanceState. const ( InstanceStateCreated InstanceState = "Created" @@ -159,8 +182,8 @@ const ( // Defines values for MemoryReclaimResponseHostPressureState. const ( - Healthy MemoryReclaimResponseHostPressureState = "healthy" - Pressure MemoryReclaimResponseHostPressureState = "pressure" + MemoryReclaimResponseHostPressureStateHealthy MemoryReclaimResponseHostPressureState = "healthy" + MemoryReclaimResponseHostPressureStatePressure MemoryReclaimResponseHostPressureState = "pressure" ) // Defines values for RestoreSnapshotRequestTargetHypervisor. @@ -449,6 +472,9 @@ type CreateInstanceRequest struct { // Gpu GPU configuration for the instance Gpu *GPUConfig `json:"gpu,omitempty"` + // HealthCheck Workload health check policy. Health is reported separately from instance lifecycle state. + HealthCheck *HealthCheck `json:"health_check,omitempty"` + // HotplugSize Additional memory for hotplug (human-readable format like "3GB", "1G"). Omit to disable hotplug memory. HotplugSize *string `json:"hotplug_size,omitempty"` @@ -755,6 +781,67 @@ type Health struct { // HealthStatus defines model for Health.Status. type HealthStatus string +// HealthCheck Workload health check policy. Health is reported separately from instance lifecycle state. +type HealthCheck struct { + Exec *HealthCheckExec `json:"exec,omitempty"` + + // FailureThreshold Consecutive failed checks required to mark the workload unhealthy. + FailureThreshold *int `json:"failure_threshold,omitempty"` + Http *HealthCheckHTTP `json:"http,omitempty"` + + // Interval Delay between checks as a Go duration. + Interval *string `json:"interval,omitempty"` + + // StartPeriod Startup grace period before failures can mark the workload unhealthy. + StartPeriod *string `json:"start_period,omitempty"` + + // SuccessThreshold Consecutive successful checks required to mark the workload healthy. + SuccessThreshold *int `json:"success_threshold,omitempty"` + Tcp *HealthCheckTCP `json:"tcp,omitempty"` + + // Timeout Per-check timeout as a Go duration. + Timeout *string `json:"timeout,omitempty"` + + // Type Probe type. Omit health_check or set type=none to disable health checks. + Type *HealthCheckType `json:"type,omitempty"` +} + +// HealthCheckType Probe type. Omit health_check or set type=none to disable health checks. +type HealthCheckType string + +// HealthCheckExec defines model for HealthCheckExec. +type HealthCheckExec struct { + // Command Command and arguments to run inside the guest after guest-agent readiness. + Command []string `json:"command"` + + // WorkingDir Optional working directory for the command. + WorkingDir *string `json:"working_dir,omitempty"` +} + +// HealthCheckHTTP defines model for HealthCheckHTTP. +type HealthCheckHTTP struct { + // ExpectedStatus Exact status code required for a successful probe. + ExpectedStatus *int `json:"expected_status,omitempty"` + + // Path HTTP path to request. + Path *string `json:"path,omitempty"` + + // Port Port to probe on the instance network address. + Port int `json:"port"` + + // Scheme HTTP scheme to use for the probe. + Scheme *HealthCheckHTTPScheme `json:"scheme,omitempty"` +} + +// HealthCheckHTTPScheme HTTP scheme to use for the probe. +type HealthCheckHTTPScheme string + +// HealthCheckTCP defines model for HealthCheckTCP. +type HealthCheckTCP struct { + // Port Port to open on the instance network address. + Port int `json:"port"` +} + // Image defines model for Image. type Image struct { // Cmd CMD from container metadata @@ -888,6 +975,10 @@ type Instance struct { // HasSnapshot Whether a snapshot exists for this instance HasSnapshot *bool `json:"has_snapshot,omitempty"` + // HealthCheck Workload health check policy. Health is reported separately from instance lifecycle state. + HealthCheck *HealthCheck `json:"health_check,omitempty"` + HealthStatus *InstanceHealthStatus `json:"health_status,omitempty"` + // HotplugSize Hotplug memory size (human-readable) HotplugSize *string `json:"hotplug_size,omitempty"` @@ -982,6 +1073,33 @@ type InstanceGPU struct { Profile *string `json:"profile,omitempty"` } +// InstanceHealthStatus defines model for InstanceHealthStatus. +type InstanceHealthStatus struct { + // ConsecutiveFailures Consecutive failed checks in the current health window. + ConsecutiveFailures int `json:"consecutive_failures"` + + // ConsecutiveSuccesses Consecutive successful checks in the current health window. + ConsecutiveSuccesses int `json:"consecutive_successes"` + + // LastCheckedAt Most recent check completion time. + LastCheckedAt *time.Time `json:"last_checked_at"` + + // LastError Truncated error from the most recent failed check. + LastError *string `json:"last_error"` + + // LastFailureAt Most recent failed check completion time. + LastFailureAt *time.Time `json:"last_failure_at"` + + // LastSuccessAt Most recent successful check completion time. + LastSuccessAt *time.Time `json:"last_success_at"` + + // Status Current workload health status. + Status InstanceHealthStatusStatus `json:"status"` +} + +// InstanceHealthStatusStatus Current workload health status. +type InstanceHealthStatusStatus string + // InstanceState Instance state: // - Created: VMM created but not started (Cloud Hypervisor native) // - Initializing: VM is running while guest init is still in progress @@ -1354,6 +1472,9 @@ type UpdateInstanceRequest struct { // Only keys referenced by the instance's existing credential `source.env` bindings // are accepted. Use this to rotate real credential values without restarting the VM. Env *map[string]string `json:"env,omitempty"` + + // HealthCheck Workload health check policy. Health is reported separately from instance lifecycle state. + HealthCheck *HealthCheck `json:"health_check,omitempty"` } // Volume defines model for Volume. @@ -16170,281 +16291,295 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y963IbOZI/+iqIOrMx0gxJUVfb3Oj4H7Vku7VttXUs23N2mz4UWAWSaFUB1QCKEu3w", - "132AecR5khNIAHUjiizJlmSNHbsxLbOqcEkkEpmJzF9+CkKepJwRpmQw+BTIcEYSDH8eKoXD2XseZwl5", - "Q/7MiFT651TwlAhFCbyU8IypUYrVTP8rIjIUNFWUs2AQnGE1Q1czIgiaQytIzngWR2hMEHxHoqATkGuc", - "pDEJBsFWwtRWhBUOOoFapPonqQRl0+BzJxAER5zFC9PNBGexCgYTHEvSqXV7qptGWCL9SRe+ydsbcx4T", - "zILP0OKfGRUkCga/l6fxIX+Zj/8godKdH2aKnyvMovHijMc0XCxP9hVl2TX0hnCmeIIVDZE036AUPkJj", - "LEmEOEM4VHROEGVjnrEIvT06QyFnjIS6MTlkfCyJmJMITQRPkJoRNONSwTtK4PASKTyOSW/Igk5tPQjT", - "T6L1VPrHjKgZEZ7BUolsK2jCBVIzKhFl+mlIeuUFUyIjy5TtBDSKyUjRhPBMLRPqF36FYs6mMC3XLkoy", - "qdAMzwn6SARHf2Y4ppMFZdNmIo3JhAuCflmkJMEMpTEOiURUIcoUd7MxNCp4bD/xMRedMi7IKCJSUYZ1", - "+6OUC7MjqqN/DX/gGJXehaHB+0jNsHJczrhCl4Sk1YniK3xZJePvOzudZ/1+/0MnoIokZlvha5pkSTA4", - "2N/f3e8ECWXm39v56ClTZEqEHr79BQuBF6XpSJ6JkIxCGolVMwljSphCRyfHb245gWC734P/23oadILt", - "Zzu97YOn8O/tg6A8rSXCV0f+efXWO1dYZXJZBpndNLKMMioxyfKsf8uSMRGIT1CYCUGYihcIthSJWjBd", - "Zdp931KEnE3oNBNuC/q2XIWcMywRZkZodGvyomis1b4LtRCL+BUbCZJgyjSNlwbxxj1Ceociu4n0kELO", - "lOBxrIWCUiRJlXS7qKPFOEM4TWMaguipbKq9pC+DTsCyONYPayMsVpvEdErhhVakobK0SO5bpDgiTBGR", - "7/A2pKmIxaaOC3J7V6OQi+2loKQs9E+X1WmeaAkvSGimm58AFYqMScgTgnTT1RXY6e8cdPt73f7B2+0n", - "g/7eoL//P0EnmHCRYBUMgggr0tUL3maZVsvvo4JK+kVkXyyOKg/tejUZ3I5dYixVvqthk1O1GGHPmN7S", - "hEiFk1RvbD2GEjGbtrVrsL4OjvIrCbz9RQRm5FqNLIW88/HxB7lOSaiPGO62Z35i6/Y6iE4QRrkM0Oxq", - "BOPKiTz7ookIgqUesNY79On0e5AxmaX6LCTRKI2x0u1qJQXYYJRQKfWn+Q8RlWZjdgLH5CPG1UhkjJkX", - "GVFXXFyW37StjGgadIIZlqP5NM2CzqpzoMrU0AWJcSqhPbviYkSE4CIwuuZiNOHCLZI+xAoSrmhqiUIy", - "P7M8FAo6QYUAuXx0c3HjzlfVOzjoBXhJGDXd6NUwmeWBl9taHm4+tNWS0ohlo5W6ZUb2Y1mVABHFU8al", - "oqFsJTfhNNbLm/DIIzqP8+YQjQhTdEKJsIoqQSJjcKy5RpBuBFGGMlnbB7kuPSJzbfyM5nsjFabLRKlZ", - "CuXFKx32xRFTOuby5c93yhomrc7da4nMMYU9eUzm1BwtVWXILs0oEnROhEd85yeqEYXmPbSh97oWIYwz", - "slmhFJvTiOI24iCCMY2oh3vOjk6QeYxOjtHGjFxXO9l5Mn4aNDfJcOLhhV+yBLOu3hB6WK59eLfc9qs9", - "r87PkyQbTQXP0uWWT16fnr5D8BAxUBnLLT7d8al+aUhHOIoEkdI/f/ewPLZ+v98f4J1Bv9/r+0Y5Jyzi", - "opGk5rGfpNv9iKxoshVJbftLJP3t/cnxySE64iLlAoygtRunTJ7yvMpsU10VH///nNE4Wub6sf6ZiFF+", - "iPgIduLUqJNjpyfY79D7U7ShZUhExtl0Stl0sw2/h1yTQx91vkMchorsO9pMVE5LufV5GwqC13Sn32jV", - "2fJWy8xKjhLZ1Lp7RUvUhMYxlSTkLJLlPihTB3vNkyltGHNCLXX1XP+MEiIlnhK0AS4VMD+MMNWKzQTT", - "mESb7ZTZpsn8wcelI6TC3sAWXTwOt3d2vbIjwVMyiujU+sTqR5T+XbOYbkcheNs/ETjM280DuhRkstzf", - "CxDd0IkgEyKI5vEv7C4VfE4YttbLX6Df4P/aKpyFW9ZTuAXEPCte/9wJ/sxIRkYpl9SMcEly2SeajYDU", - "CL7wjxkerVrrEkdJhcXq/QFvfIWdWOh1a2lj3RZatcHTtZ+81e/UZSeIxlyXKEmBRhH5XCs1Hu2AM2Uf", - "1NyXfIpiyozFoVU7sxagVy1S8lPMQSR+JTrk5F/e/HrctxBe5oeG1vSzTq6Ax3xapuaMYKHGpELMhiPM", - "NlSMrpH8Z5XtUzursCSj1RLkjDJGIvAX241t3tRqrNfMgF10SdVoToT07jkY1q9UIftGY1MxDy8nNCaj", - "GZYz62CLImqchWeVmXi0tYojHoM97hoELQLs1/NfDnf2D5DtwEND67nULyzPpPS1bt68ixQWYxzHXt5o", - "Zrebn9HLHOLngMJZ2XT25BzoGNNIusCuprWTMzkzf4Hs1qOCs0+LAc1esf77g2fSRyAkjJXQeHvj1wFz", - "z/A05pqmC5Qx+mdWUbB76GQCDmJ9UNCIRB2E4QH4HbT9NyWMCC2nCs9QSQlGG6Q37XXQUOuFXa0Fd/FO", - "t9/v9odBVY2N97rGvE+xUkToAf5/v+Pux8Pu//S7zz4Uf4563Q9//4uPAdpq5k4rtPPccHu/g9xgy+p6", - "faDrVPlbS//y8H0Sxyz1iZYTN13po5NlxcHMNeLhJRE9yrdiOhZYLLbYlLLrQYwVkao689XvflVawDxW", - "EIFNNZluSIaa0QNsvBHzKyJCLYFjohlPdrQQpkp2ENZ2MwgvpE/J/0QhZnovGOWCC0RYhK6omiEM71Wp", - "lSy6OKVdaoYadIIEX78ibKpmweBgd4nPNZNv2D+6H/7mftr8P15WF1lMPEz+hmeKsimCx+VrPTeG/Ipm", - "1Yo46mYxqHkJZSfms+3lO6gvW2E3kVUrbYy5xqXWQih3ka0ZyPL9rja2Eo/p8HpOhKCRO5aPTo/RRkwv", - "id0vSGQMDbN+fzeEF+BPYn8JeZJgFpnfNnvodUKVPg6z4pQ3V7a12zUSzjgoKnHMb3KdBpoiGDg4XnmO", - "ryKNl9pHebvLp/4vXKpughmeEjBH7YtoLPgl0QM1dwKUSHRJFlrLWaCpbrQ7pxJueAibozk2XofekL2d", - "cUnMK+6RBN8+nROU8PDSXP3OOFjycxxnRHbQ1UyrHOATJDi2PyNzMTZkMz1IGfKURNoIMa/B1NAFYfML", - "lOAUtjkWBPY4SrAiguKYfjRX+HDLQCKqT7ghI7AxUIr1ng9DLiK4YeOI4HBWosJfJbowCssFNH9BmWbr", - "C7Mxa5fVn4LX797+/Prdb8ej12fPfzs8Gf36/L/1z+ajYPD7p8CEauSays8ECyLQXz7BfD8b9TYiIhgE", - "h5macUE/Gm/N506gaSA1f+GU9nhKGKa9kCdBJ/hb+Z8fPn9wCplxY8/1NvAM7LNXGTJnqUckHTtvoETW", - "w+TuNjTJtIh6efZuS5/OKZZSzQTPprPqxrCqwY22RETl5Yjy0Tj1jYnKS3Sy9RppxQXFVG/QXFHZ7vdP", - "f96Sw0D/Y9/9Y7OHjs2uheFrGcSF1Z/kTLNPHvVxdPYO4TjmofWhTJoueF1XPgFPmBKLlFOfEVcTTsWr", - "yzKq2y2e3kAUbY0p25J6GbrhzegOfHNrU+I5m1PBWaLNuTkWVJ/TsrpXfnt9/Hz0/Lf3wUAfBFEWWq/k", - "2es3b4NBsNvv9wMfg2oOWiMDX569M7eeZtuoNM6mI0k/elSJw3x+KCEJF8aEtt+gjVlV0zD7FsHiDIPd", - "lz8b5tp+CXzlFsXeEeWtmIZr13ovf/Zxy2yREjGn0udn+yV/5lZ+OdynwtvmlixnWuDiXsl+CWOeRd1S", - "l51gQgUJIbxC/+tPkmhFfv6xei3l+c7v/mqlwK7RTHGcUkZWqKbfiIp4xcVlzHHU3f7KGqK9UPWExpgH", - "1fXNb9YcSyxFnI0xi65opGajiF8xPWSPXLVPUP5yLlyv9Uxw/K///ef708LO2n45Tq2k3d7Z/0JJW5Ot", - "ummvDyWfSJb6p/Eu9U/i/em//vefbiYPOwmjiNxKqbPr/9y0UA+asbGExh3acDOcn955wIri1qCGz5Hj", - "vbXXwD5BzedExHhRErx2TMF2H6RfbVSCQpQkst9pMXqJ9MdrxLBuzR3yL+tG/k7fL2g9g/KM6WctK+y5", - "0GYk+UC2d07tnzvLQ2oY0SVNR6A1j/A09/muCgk9v6SpVcXhC7OMcWwEQZSB8j7mXPWGzESo6LWDBSbX", - "JASZJxVW6PDsRKIrGsfgIQKhsny0aMW+FNoEr0ul/1dkrIPGmdLaOlcEWbsJOslgLPDymKCMYXcfXtOd", - "7QSXwwuALJdEMBKPjG4sW1LGfITsR43EgalOsLQhakJlaZVex7+enqON4wXDCQ3Rr6bVUx5lMUHnJrpg", - "s0q9zpClAsIUdCd6P1PbL58gnqkun3SVIMQNMYHGch+bvaydvzx7Z6/75WZvyN4QTVjCIhvo604cGwQa", - "cfZXvWNJVG223H+N6E0hHZLhVM64GqV58PQq6XRuXy9M8fbOhE4wD9OsuqQ7ncYg0DkVKsOxlrUVddJ7", - "wW+C2D1mg4mRL5svVu4VQbOqejPb1uNiWoaIdm+4rMdxYjSl1o6Tkim/5EJxduandoNd0/4JcwNZ6Tgq", - "TM0v6OvcNLIUvGN+7riZ3YJKJzlNau6mr0OeQ1kyzVsFn5sYLKMRSrRxoa15y8fafr/ooIu/VX7Qe9+Z", - "Flq/uEKGGiBPmP6p3H7dKbHWXXCjcO/y4mB5+/U4lI2RTmi+jZTATJoYtRlOSQ/9AkIcKZKkWpKxKaIS", - "5aFdiPGr/0TcKDXu0yHTQ5MmTsSSI3caSTpllE03tZqvDyYcRcazNMlUJvR7cyoLalZZx3lvlqJazeiI", - "kceQIUFZGGcRQRfOw3NR1QuX/T/LJqF1CC1ZOIYkYNmAsae2kkzp7vWEE6zCmaYTz5QJHLNTrwb11bxM", - "6y5U7Vjyq7ZbrP95Li7qiTBzj4mjJ2cvecAtWPJPNrkBraLid1FekgUsuXNH4iWHZNkT6fcXCiJ5PCf2", - "2C37MseQ6sON4lS4MY1D0vog9favJ7n4vHPrlkLTqzX5q6aCJ8VHqq6bbMExVvt3MeFOCunJmf462jCW", - "BIgPpscAgTp20TG2EgEPBGKaWWIUUUFCtdQ8ZdMhgxiSC/tLz7Z2oTe51lG+SuIU5CGA0l5eWlRaWaf2", - "QTN6ajyhSpGoU9UNLglJ5fpJafXaOq493nVBrgR1gswFFbdUzwibcBGSxBoJX2Y4Pi815jXjbtbEckiH", - "oW9pzC4/A7JTSGTih8x6gJu1krZRz16MalabCSGodnmB4/gCbdiXNpEgf0Akvl0rxlnB7G+PzhwL5Nfe", - "7087miO1FLiYKZWO9P/Ikd7FF/XG7LduhxeZZU/7YF/t7e3aVbVONzPgWrNV/5o3LKJ5aZz63XizpvlC", - "j9LGmbRR5Y+KTwpP6iVlUdsGftXvNnrncsXIWRp37aBLBelm6VRgCLH9mu65W9+bAjWbJfiaPF5fmGSR", - "IZhJxZNyvP1GLcSDVoNBqsSa87gbYYXBldnS32qGuxx4nCxMU8YWa/LEjKZjT9wQ/QipAFM6xeOFqt4f", - "bHuz+b70EtuNxbcsTQH8xoIk0Ujx1SHMdILcu20iFk2+geKj+YTy1ekdNv6lkn9njiNr1+omumlIrTsB", - "dJxwZiJMDRFAaXx/Wr676w1ZF47fATrOO8ibzZvEoFviyNycbHBRGoRJ5EDjxSbC6P1pD73NR/tXibTB", - "Micuo2GGJRoTwlAGrmc4DbvmLC4PIJNwaKr659Z3YpIfNuGKkttnvTznGLw0eQY1hEqNaW0+JnMSFsre", - "CWNW9oK18lqtCvx+Q6ZUKlEL+0Ybb14c7e7uPqv7L3f2u/3t7vb+2+3+oK///3/aR4h//fwOX1uHVdli", - "g8/K0ufo3cnxjnWWVvtRH/fws6fX11g9O6BX8tnHZCymf+zie8kA8Yuy4yJqDm1kkoiuE5Oaq3yxcqWQ", - "tIZYuFuHuN1RxFoRgLvqXUOJt/rNu0ht8QVN25Ddmyef1AXm2rDr0uSWLflFCnZnsUtKGpyNbgypN47z", - "mMrLnwXBl5Cyt3xuJ3hK5MicZ/54hkyaIBtybb0bgnM1kebetOr13N57svd092Dvab/vyehYZnge0lGo", - "T6BWA3h9dIJivCACwTdoAy68IjSO+bjK6Pu7B0+f9J9t77Qdh7niaUeH3PByX6ENS5G/O5wS96QyqJ2d", - "Jwe7u7v9g4OdvVajsv7iVoNyvuWKSvJk98ne9tOdvVZU8Cn0z12GTV2B92VWHprsfv2vrkxJSCc0RJCj", - "g/QHaCOBI4zkt1XVPTnGkcs/9Z8dCtNYroyYMJ3ZN42jLcliRdOYmGewIK180TDzY2jJi5DBWJ7ve7OW", - "bF7S2ggBN5f8FVTJL6uQ7tQkNJeUJ0riaGB26Fo5B6tZDOxDEx/YObTkhlfadOrGZE7iMhOYo8tk1gqC", - "cj4xi1aZFWVzHNNoRFmaeVmikZQvMgG6qGkU4THPlLlmtAnaRScQ9Qy2x0SL63Z27gsuLtfGj+qTOM9D", - "X+sVOgRH+sS6auAUx8h+7VIUSkpffh1oLk3tc4nemC+Mh6j4Oc2qqDYd6Ml6khgSRCoOktQ6DG0zbbVL", - "v94CzlIX/mH6K2TnPcW+dCcmXODrWthiSgB/Qa3VWDSnvIX3z+H11uHo+sO1jpQWdGfk6j6IDvH6Xc22", - "XclwejcUXxWMlvsaipfgFBY0Ij0EuwuiYlx+YG2nnSuepiTK/T+9IbPx3PlP0tyg6A8NHdSMUIG4oFNa", - "7bjqYLvLqLabsKLjpluzY/nDZQ0VHkL4RvOmxxNlsBYuXcoUKecv2UUIOsF5jkxhJVGVNG9ydI8lihSh", - "lktDfHn27qaxaangE+rDG4JYCPvUWmYuauvVXv+8u/3/mAhMzW+golFm4icSHtWAJOz77U6el2fvzprG", - "lEM7oPLoluaUR7ysArdyFLGXSvZW0lowjv31wZJ3Uujez3y67ETghIyzyYSIUeJxrr3Qz5F5wYQ2UYZO", - "f67qs1pvbms1n1UWB8zmCQ5tZn476nsccrVpdErU/OBfrjfEHMNN+Xx6qYR9x6b09dBvOZgGenn2TqIi", - "Ssnjqasub2O8/NlsIWmIY9OiSc+lrOxgA+ZsrSGfFR9aV6RHT/ZjsLiNgDbm0zSDbXj+pnvy+v1WEpF5", - "pzImiCya8ZjocW+WpMXcZfUVwf0VITFv8nQYxpBtN1CJVvkObk2k0n71UEdxheORjLkvWOOtfojgIdp4", - "/8JkXekRdFBaWUr9e4kKFf4+8O4YLZGauj2HDusu08oG99qOVTRM414pTa/SqW+r/EJwbEBAq/y8DIDE", - "L6sLzS/Xg+6YRnz9nrjA8JpR40veOjo9NgpDyJnClBGBEqKwhRwthbiAOhR0gq4+oyJMEgi1m/zn6uiW", - "Bhd8ORur0Yl7tITbcScO3IZ88zcmBCFCCWZ0QqSy+eaVnuUM7+wfDAwqRkQme/sHvV7vpjkqz4uklFZL", - "sWVC+EvpKj05+7J1uINUlDZz+RScHb79JRgEW5kUWzEPcbwlx5QNSv/O/1k8gD/MP8eUeVNYWgGp0MkS", - "gEr1SlOfWeb3QQnz0uH7tcK189szENkAeXPefGOFp9o+MRz3pYnFt4YeKfCvVAlypBwQ2gJ+hH5c7Ql1", - "ihG8Y/vMmKJxgcyy7AO9FbaOXAk/sAQ9kBKWAw7Esfkr5Gyud4UPfaAiwN2zL7o/sFEuo4h6OPkf1toz", - "QRKQVbV+vwVbOE3Xs61fUczlX1vUFZsb7TmJHlzq3+aOrdr76+l//fn/yrMnf2z/+er9+/+ev/yv49/o", - "f7+Pz15/UQbV6rT4B81t/2rp7HCxVMlpb8tKp1iFHoVqxqVqoLB9ghQ38Zo9dASG32DIuugVVUTgeICG", - "QS1EeBigDXKNQ2W+Qpwh3ZTNdNjUH58Z94/++JOzLT/X24hsSoOwC5JnMslsHPEEU7Y5ZENm20JuIhLu", - "9PVfEQpxqjJB9OppHTZeoLEAWG9rnhedd9AnnKafN4cMLFxyrYSeQYqFynE8XA/AFHZUJmbAvk4ilxhu", - "LOQhy8+lPC/c+Gh6uRMEfPP1iEs/UbzmCxfVVJynfV8GPUR96YWMqVQEArNzztZslIejoaf9iqh42n/a", - "X6vg5zy0gv1gJyzj/TumbLGXDAND10ZwQ4RaC1+6lk1mj6Bf3r4902TQ/z1HrqGCFvkSGyPPxABK4yNU", - "sSxF/20GXrRRWN2WEzJOMvgsbpE19NyEh759dY4UEYkL2N8INTknNNTzg+t/KmWmWZFidHh0+nyz16Jg", - "AdA2H/+KdXybz7Ce3GGdZk2+wJzjNX076OQYwnPtDi0UOAirecEFio2AKfb1AL2TpBrrCktlbvXNSsaL", - "wvNmToBhsOlaTOuSYoDe5HojzodSKZJQdeYV+xKatRcvJuZnqfXOEvy4cHaRFW0Q4YNVHiSuT9xmUbB6", - "+3soDnvexnWXfJo329tlZ6juzM8axdp/bdSUr6/u7N5M3XEIuOkMSx93zwiK6YSEizAmCF5aATtPa4m3", - "otl1Xel3LfJ73h2A2eujr/y5D7N7v7u9/XZ77+Y2/00BL6o5qaV85hzzoj1YxV2APizbv9dUjRqDGpB+", - "bEMYnJX3/hTNsGR/VfCwZutt7z5pBV+qe20bDlAOBOATM6RcSrkE1/wa26T6XtI4NtEhkk4ZjtEztHF+", - "8vLXk1evNlEXvX59Wl+KVV/41qcF9oUTFS/P3kH2EJYjd6PWHESKi0Bsck2lksv5v60upldjbfxSwcPw", - "JlRvfkWQDHebvzSN+4C/eMgwyW8PemMlWMaXIl5Y4+GOAC8ahasPLKIqZ83PXxe64k6Gs7baSvmsdTHs", - "t8aK6ATUE797KLUIJBE6OSswJwsnn2u+Nidbumi73+9t99u4PBMcruj79PCofef9HXO0D/B4EEYDMvkC", - "l6tlbKMM4/gKLyQaOnNlGBj7qGQYlbatNWlaXWcvQ3LcDoGjrlD45bRRrBwGubQ45U3KRYsAzXq+cJLF", - "JgegjG2+XK5JpgYcw4C75brkkMEAOzZJNy+xhMNQZIU/wdXOMJpnllq+HzJBZMqZNBV1euhXspAooeDD", - "z7uH+AyJ8jigaMg2hIsZy4PDUpxJEukfICCj4y7+9dCoAhA7/cGQyVkGpUA2e+iIM5klRFhXCxpT8ANv", - "IpkZ4wrGC9SAwlSSRkQMmX7NA7/xKVeUBwf9fr+flygJBrv6330fN/m5aB1mSjswlC9FoFiFLH5exRRv", - "bdN8QeGdVhFDTnWzsUL2q9FNbosICnkWR1pRHmvRbPwYJLLuFklUAdcO0vwdu2SaoSpTN5cGmtH/zIhY", - "oPenp5UrJkEmFo26xcSBpRvWgac3WoadNabl2tHcEpfkPrBI6sdqSZ356sgjZR+3i1k2HNrC112YF944", - "E8rM0mg+WTGnmpcyIvNRlvm0Zv3IZSq9e3dyXGEOjA+2n/afPus+HW8fdPei/nYXb+8edHf2cX+yGz7Z", - "bagH0T7O7PahY9Xd3JwZCIQHj79J/IwGer/lsV/jTKE8LlRv5CNtfqCSnWPy4MAJd1I6LnQz4BOzZpBJ", - "aDawrPpYAQQNwG+iTE8ZnI+6ERvtN0Av4V14hBNzktpBaOO36nfD0cLcO2jB4Lo259rqIZ/bIw2+0ecb", - "gmpoetqaDNYcXd2EkTED9BuHb/IDlvG6XWteh5Nt+fW6Dbxhw/BcuDZ05kA+BqgesKoplXBl2B1D2KWp", - "L5tirUDkEb5X+JLY1dDvSNOdbtsK4wF6kQvgXIRbkb0hif3TnAs2SwEyMDYrcbCWmwLNiQVXlEI8O4FZ", - "raATuEWAUNDloFA356AT2CF6U5/KHO+79yM4BkldxN9lisYWewQmRaHsGMwJAw81CQyLs0eikdE0mm7x", - "TVCX1Ubyj5w8en+KNiDL+O/I+ib0vzbzG//y5t/bebb37ODJzrODVrlExQDXnyNHEHK4PLi1h0qYZiNX", - "0adh6kdn74xWGxp9EZxNdu6l0O1U8FAbPZShokRQ0fmz3rNyClXEM1MuzQ7J5lt+LhUFXFnPqeHa+k8a", - "z+lkwv78GF7u/CFosn19IHfGXh9BUX3Qa1CdlC8wlrwPZNw12Kj+LBdgKCEbE8HeEAkzQOdEIeCfLsIh", - "6AJ5pKBlOZcuZinuZay93d3dp0/2d1rxlR1daeOMwLxaHuWpHUFpi8GbaOPN+TnaKjGcadOFTwNsC7N6", - "on+fIQuT3q/W1+1t93d9XNKglhVcY9ueJ40kf291LTspS3QIeMz1sKVd7qX27m7/yd7+0/1229hVsxTX", - "qyWMg8ox5LHoQuWV34BLr7eHZ0i3LiY4rBpC2zu7e/sHT57eaFTqRqMCZCyDaHODgT19crC/t7uz3S6j", - "0XexZXN1Kxu2Krs8m87DFJ7V8JBiWfR2mk4Ln35rGOwNCWNMk8PQBaXVTh+DXDMS5rViEdocDNbXtHRw", - "tfi2lSVYK8dltAQuUKmOaW+9V/12TvJmMW3Og/VifFlVjzHT5LKpNwYg9Ra0SwWZU57Jr9AQVyTUzDSJ", - "ORc3+rYpyu8NkVmsjCebSvT+9K8gRDRzIalIWr1etOy3IkHplpO70Qau8ISfq5uI1Wo12iz9qgl3GrZp", - "Z1V0emX7N+YBRlpUZWx9RMkRjsMMIAFxvp56VpDRwzMF8S8LE3sVx5wzFM4wmxIosWAASNkUYTTjcdQL", - "/DducTSaeG/C+BWKuUEwuSQktWh5ZhD6M1tuGm285EWhSMNKNdTr/cRIFYuHVq8OHqysNN2QN6jpiRUv", - "gWuYTypOg5hPJRibCqLKenVMJ21ZGWuBGfTHeWJs1GpC5I4+7T1DrElv3xFqjk4+sYaz1TEUzymJQ8Gl", - "LMrdvz+tDnNVWHBCGU20nF0fJVIdbAvWNb5pz9llixe3BYn1HYieeMsvORKBhyGselVldeuGTzDLAD+v", - "xMjkOqXCsEe7GIsZl2qUJ3ndcLBSjQAbLROkyAR15+UM0moWRsTBO95z0Ym225ArLxp+i6+XuMrfVNMA", - "m2Wql6J+anVyHvSx8XKa28rMuiJVr56XdZNEzAJMi0polZZyANEG46oilkqAUJtt7jv9Nqrup6lQ86u9", - "/nnbHMnVKZFnWM1O2IR7EFdvcK9gE01cCExKBBT15wxFhFESOeMxv2CwLjRIXYklQVFGLOWMQiqwJTg2", - "2xuQVZnzvVE2rcn6eodtvP1mDKuh06Bf+2Kbm2vpT3h4KzKglYk1kQgXqQ+tAmeoHPkd0ssNCzLNYixQ", - "PQ94xZDlIokpu2zTulwkYx7TEOkP6rdGEx7H/GqkH8mfYC6brWanPxgVkbu1WyAzOBu3bRak1m8xhZ/0", - "LDdrWSPgetky32/p71sFAnjDz17QmNhU2XeMXpcYvYottLfTb0ooami0kkq0nGZ9U8ltWda3410G9GFe", - "i8ST5GICyWqXH1VHZGW+3ut4Ki9Xpk8tu2LQhostcNhNVbqWMJRaeULaBUvWo2jcaLYkCau97z3df3LQ", - "EsTqi3ydK2qVf4Fnc56s8Gg2rNRpG7fZ0/2nz57t7u0/27mRg8oFXDWsT1PQVXl9aiWHak6zfQhO6N9o", - "UCbkyj+khrCr6oAq5YNuPaDPK7ZuAV7QcO3RiJ4fl1fS3bNUPaDtfIwrtKXDispVqpC3QSYTAkblyNCt", - "WwymlvTSagwhTnFI1cLjMMFXppBC/kotCb+NN606WA9JbdsWR0VLLpmNi7jgDdc5+ptxrdd44WlrLDyZ", - "jZvc+K/rvRonfuEDKl8RtbihKcp1LLsL8vlcYVkJHtF/hxCQVFRArIehmTfa13p3vJ6Xey8CbH1AEv7S", - "7uXlry1nye1bUZLrFF91hDZvwRvZ0J4T2Vf8dX1wd00+2APwdl+NxmWUypUwoBVIy+LUvXm/7Wo3Ln9n", - "TrCb91eKJL7Jh3XAPuBHOwZL8qLtToUlGrhJcbEep/0OYLdM6MKtgLds1MO9YG/Zn+8Eb2tpOc6Jcu+e", - "a4s+i1fArDNFxBx7HFOuCeReqfpRjSTuIOviQ9vJZi1HaG/m19VsYnzLKDWtBY5SQSb0egW3mBfMcV3N", - "QpCWAlEVil+ijQRfo70nKJxhIWtjZ3Q6U/Gi6mTd8yThfFlpdKK06ty+aEGxmu7D5RsNu5zl1n1b9ryU", - "MuMvpkCi0Sr4iKP8NeczTvECdMtGQ/DJ7l6/v7vTvxV+xNeq8VBqpynwtPSddeZUrh7LLeRhpstAoFeC", - "mlKBjkxSCYKTAQRtpTgkKCYTyK7MAZjX2vRLXa8evL0ktfkjRRiXXShXD9j6WayIY5yB4uHasdAbbhqB", - "u6WtpgyVny8Pe0UKZi5mwqVcTE/K4W63f/B2e3ewfzDY3r4LwImcSE0hPE8+bl89iXfwZC9+unjy5/bs", - "yXQn2fVmndxBOZFadc5adRE7h5SIOsJrHRlZkpgy0pV5BNz6OOcVssDcJK3d/zfzPpgZrFQWzquTLOsM", - "WBXEqRc6vI/8ODv6lS6U+vBPjlcP+1ZxZPWB+BmsPhTgp3aDAQCk7S9F28lYy3PnXenF1ifPytjGdWeP", - "L7YctrZ3lRso7uPnimCs7LBVJ/byqeYx4aZcUDVLVh8P+Ws5dgdchn+UKqrmY/XQyZQBnnP55/zuo1xy", - "XX8cdIL44151z9jf22fmWayKnAHtUpfVgBZ3AwAXvpoK8EphWggTnoAFAUL8tN3dfgY39PHHvZ/63Wc9", - "9I9SpEDHUKtMvm33duXXfhsa5oIS9E5zc7797EbX6I6eqzjoV3suNR3EFsXC8ngBpuvOChdBXVng4vHS", - "GteSl+6sfJc9zUZlLSkiMV74SqDEeIHGZAKY8fU491IDaEymlMkOItdO6GCJMCqbQrY49m5fDgPEBRoG", - "+8kw6KFDi/0C1mqBlV5pHlCyS3xCE1u5zeJkN4el7CTt0jTqxsPNAMHcVx71rOfXz5697W/fGBDsZsdk", - "7wuiqr/I3G1n4sZYqibb4hWWKrdJkcishdGB4lls0fMVLLCZERDeCpmlA5dIWbCslQGyUPycJ6SDplyh", - "IidiraYHwxcZ8/JDdfxFAXRIEW1kiJ01DNFuTHnOJF0lvk6OUSp4lIVFFHAMg87CkEg5yaCee6+tVr/+", - "nvUuHRoQXj/hAq13aDR5MNZnbJPr5vX+jVyrUpeaYZuXeru/fqnvxAvSCbI0Wi/DzEvtJNiNMH7WxJV6", - "fDJVstc0wdJkPrSQ6G/KFFw2cqEUHAq1SpSlrjap5qllTpKeiqT4euSF2zgmMdHH1HIjyBQLtqEvVBZS", - "dL1I3T546ncb4utRCLmfSwP5lZBU2yoJl7Z4b4LZwjuwOj4/2ui72rQSQfNdgxFoqVUd3JO1mljjUrUv", - "dVDzapsEuHJliRzs5uvWObBfrq1Ccxd+uIdU0l7by4UarIID/chhglz/JqzXlKaPa+f13r5sp5O9tZZx", - "E1REPbS37GY+7P6PcSujUW+w9dPf/+/uh7/9xV/gqmI3SyK6EZnAfeclWXRN5W5to/eqQAmAkaWVaVsW", - "ShGcgNMovCTGSZXg6/J49/u50Fj8hpOlKcBFcUJZ/u+1E/r7X5qvWUtkfAdyci3LfjGE3F3gcyvujqON", - "hIipK4LhogM3e0MGAYeXZCFRCQTUqjSOUf8q80/K1bcvjBrYg5rxYwpYynLItFWLw5Ck2pqwYIgUxiI4", - "SJ96DXoLRuqi+bVcwoDAbLOrllA2fJXeBwEjV13TQ9TVvLe3fwB8RFmZkttLS+xbdIMD0FSmVlPZ4zV6", - "RSVkn7gg69LLaIMkqVo4qG0XBrt5M1yCw7xB7833V8Yg7D/7GpDL71ZiLH+HRZLLsBFuQGsBI5bWvxHY", - "1B9Hd1zH9zJ70hZ+rOJR1axDqbrNYXaJVmdGEC26HBKnn5lIVAsCNM3q1RW2Eqa2LIS5L/slgrruK2OP", - "i13mMBQAL6BFSO1Khbo0s9JImtfm1OmN9Vr7zQQ606S5mhFBSgsBHxS4STckmY0LbZFTZYCGUyK69cqg", - "pniNoBBompv8jgR57PCyH3Q1LtYpvs57AB86lks3TTCPAiFy++XPUJDqjasQSSeuCRhGTav3gyBVuWgV", - "TRxXLS9GmauW523e9248K6tWSL+mvVVjzqKPCmv6+PEfmKoXXIAd0JzBdOdYSmBjRERACncdKakVzBBN", - "SDTimVq9/22lDpu+FDlVvgAxdzYPBia2hePXyAKXY1OM4YNPb5AkzARVC20kW5V0TLAg4jAzGx4ICR3B", - "z0XHgAn++TN4CyeegMWXhBFBQ3R4dgL7McEMVGX0/rSE42sgnZdQXUDJe310Yu1MhxMJdgNVwHqulvvh", - "2QmUhhbG1gr6vZ1eHzZzShhOaTAIdnvbUChbMxxMcQtKiMCfNhcht1dOIqsH/Wxe0V8JnBBFhAwGv3ti", - "+hURpiSJBK0TT0t2Q4qpsIZDGkOmgWEVqr8FUC13lA7MedwxBG/tJpNqYeMuSfraLusHzQlm18AUd/p9", - "Y5IyZQ9eXJQM3vrD5mcW/bbS54A8HoSpJb3e6ZSW5J87wV5/+0bjWVvl19ftO4YzNeOCfiQwzP0bEuFW", - "nZ4wEwyODKyIDXcp7zNgofIO+/2DXi+ZJQkWC0euglYpl03KMNG2OyNXth7OH3zcQ/YSAECT5YxnsZYm", - "yES6O3NfYdGbfkRYhDM6J0Nmz2lTsRkLMOYTpM9nY7ZUt4bp2qx+noP4M48WNermzW3p5rrO9VsQuA7O", - "KskIMMhGTcWuCqcvZQyK5kpikWzzqi/LMTVQ5VyG3FvenTDMVFE025Q3vyQL61f2NtgKxUcLPFgWIgBc", - "w/lJdjb96SuAuevP/DrOnyFL3qo6weAyJoyzqNC5XEQ1FmMcx16Yh2nMxzi2VeAviUdFfQlvWKKU4Ymd", - "csN4RAzUbLpQM87M39k4Yyozf48Fv5JEaBXIQvhbWtsSyJZ1rwB9LAEYfVMgSPe5ZYa49emSLD73huww", - "SlzxJ2k+wbHktjy+QUuiErkQXMO7flDkhuiOo0wqnliWYuVqvmaYPFNppuzNtiTK1h2A16HYs5yRaMgU", - "R58EmVKpxOLz1qeix89guxAcaT4pvWKmtPWJRp+bRi1HWM9+BK96rD8CBBgG+nQZBvrvqcDadsnkDFwZ", - "EtwX0/KSbuSp91ov3KxTOMQMpTw1sAXAVKbqf6UNqOGC4xgp2EruW61twko2zMdmIvkKkto0JJM3UttG", - "UJq0tJn6e0/9+0mSUBCfg+O/zl//huCo0mtgXivcRuZqmelTFEUZaPLQe2/InuNwhozeBAh4w4BGwyC3", - "LqJNGGsmbZx0twsq7k96aD+Zbjo0+qnX000Z7XmAfv9kWhnovZQmI8UvCRsGnzuo9GBK1Swb588++Ana", - "lM1xXhEEaMPI/k1XgQtQJYpj0JwbmEWIW1kbLxBGhQQq+1HGlGGxsnyYh/SWgtqUx1NZJsanIXhQh8Fg", - "6Hyow6AzDAibw2/W0ToMPvspYJXoZhw0U0HN6do5Ex30+5vr0ywtfT0qdOVFvf0+L2lfO19N8bBK17Li", - "YSbnsCL1CppaeEbdugfN52ccueoqP1S8NSqe9VyUlDf4vnwOGPaNiTFwaxqYtmdjp4GttE4MWwBYKlgc", - "LinaGBzUaXAF85bNj7o5v2xW7DXtshCGGDv+27sH/oN+80Icpt9n99UvjgH5NK+u/rjYERbLMWLHbxG/", - "JOpb4Lj+fYlSC6X6kPz7WPjnJbF6X0G0mjTbInN33+SHfoCUD2lbMS9rW/UcxtQ9J0yh5/Brz/7XWTyA", - "l3wR8+nFABkSxnyKYsrsbVzptkgfipaW8JHJ+si/s0kgDndrw5yf//rff8KgKJv+63//qbVp8xds9y0D", - "hgJwwBczgoUaE6wuBuhXQtIujumcuMkAkiaZE7FAu31QM1MBjzwVe+WQDdkbojLBSreWBgJL2gbB9GAw", - "H8oyIm3WjH6RTiw+h3Ewe0x4t5cNKe91R3eWQ4zNDEoT0Kei4wFIuLbFD6z9Ffi9Z2bOFf9Z3Ve+5DFd", - "L18UuVaGe7tmgDcUMEBi376DB3bSaOP8/PlmD4GNYbgCMFhAYy6ascpz74dMWi+TjESpChSgspFNJpVs", - "tf/32L7TzgFsW/yePMAWlusGLmDj8oCqeW4FftgKLdzBfro517DPP3vsciWbHbS3n2+5CxdN1MoQ/nrr", - "7HhvmebmSYlkD2ECow0Xk+4Kp54dnbiKUpsPxvT3cmromdo6HfnRgbgp13pvZtkRZ5OYhgp13VigVERC", - "clOtyiCPRRy8saNG2M2rjnZYPt+2KuA9jSddjuNTHHl3f3rUOr3JMVIgMha89uMkWcc6x1SGXH9b4pZu", - "iFNb5tSoL/k+LXPROoeUCXHPj5yV6pIVzyfHbkPen2vKdp2x+tlwD0LxuCYQH1AQ1ko3ljBMHxM3v8tX", - "0eFCrPBcfVus2b8/Lei+vVg+Nn9MbqyoRjYtBQ2Sc+MB+pKoX8wbd7jQtgfPxM+JcLvaQU7DrPNpmU9R", - "OCPhpZkQXEivtn1PzCvtTF/T3vdk+QJ5bqKxWJL/UFFaGLsFrVYZuCe2XOHd2bfQw43M2693z2sZzENk", - "CDYZO4+1qQSI5YKFm9/VVe+9nGaG2I/yMDvL4tjdeMyJUCiv+l4+A7Y+QVjSet3e7baVx8G7N6+6hIUc", - "4tDyGCq/EmWffGUN3yyYmcoPNmljE5rkZOrOsyYN5wvW35alNhGOPcr/Y+dFTMcCi8V/7LzAcUoZ+Y/d", - "wxgrItXmnTFL/75E831r3I+Y+bTCTatEA9HEoALtOg01f6ulkure/670VDPpG2mqOV1/KKttlNUyuVbq", - "q3Yp7lRjNX080JVMzmw+asMjF5/4nWmq9+vlsxzp4JqprF572Ho8XICfFx5RhjJJHmEAJc05rnxstHRX", - "Fxty5fHhWPfkuAOE7GjSAXaSTRC5J+e1G8e9K7e23/v3XB8mYzrNeCbLuScJVuGMSJusFJOqAH5sandx", - "PDcq3t8wl/bv8+i4d736B9/fkcZfX1AjvM0N1Dqd373VVue372ud36RQ29w1C/DUceB/mw1BhS6Jui0b", - "V3LNl4MdfePy2SLonTZUCnMBgQUxGLL/o+2P3xXByYefXJJM1u/vHMDvhM0//OTyZNipYxXClKAWq/Xw", - "t2O49ptC9jnAuRYpefVxmPoPwHoOwObfzkAqbj7bW0iOC39YSK0spBK5VltIdi3u1kSqgmDdu43k+M1H", - "cAti8sNKug8rSWaTCQ0pYaooj7YUJGarKz7C3DJm74dKwR2Vg7a1lZRvyjUKaAHOf++BPScFGuF9G0eu", - "DsDjjJHnqQXWtuZIcRg22yPfGj/071c4378d8phZzCj8y6RLtU7pq7oJSI9JpiAosUAIgahPJIzWnrfY", - "Q0WxS5mlKRdKGrRIUIANnvxMK8A+ZMkqWKQPHRIwgCmRnSGDegH6scnl37okC4MFSTnLYR/zmVr8R1/u", - "VRWL80G30dfXsfxAo610rHvexhY6+uF0rAcTHfeiaZ1UEPk38o0BBuWY5DuZ58l99CNl081HFYFqhFU+", - "txKekUfV2oJ6exZdd0vmdX2bDtoSwK6tRvlveOIuT9KntTss2hIBUUTxlHGpaOgSd+uY5T9O6NYn9GrK", - "erk5IglXpKtIksYWhdFv2x/Di45Gb93735n66OaNDN2iUpmb7+E4KBnegB2NHN/APSSWCFLlJ1xcPrJr", - "HL2Y5emMcXhZLmEkOaIKkLTGxBXSjfSkc0vVs7cmtlKxf0O94OKyrfroqZz3CLTI8gy/QT+dHh4g7T28", - "uw4cWUbT0kxz7xrmUjnEh5QstK5zhnEWaSXTKZvOTJsInozsjwYLWu8Ki7QL7r/QtvrQwkj3fg/O2N+4", - "QjRJY6ItZBKhruEmvZrWrHYFFagsFQ+9mbDU26acbGaAHqUrvmUlJlxcuwXbgBiW5eXySs2YT9cDzOSd", - "OzQVD8LMkJmCD8RVh7hAuZAF2U5iEip0NaPhDNBmQN5DzWIAgsFpepHDy20O0EvYqWWUPeh8QxJBcax5", - "TfKYGBCZeZJcDJbRkN+fnsJHBmjG4B5fDJBDQM4PCKnfKqPH5FW9frOYOBuakwSPY7OiFwrTuDS/TYsr", - "U8D/DZkPY4aRK9sgnaCLEtzMRQPejBOor/j0wSyZTjNoq5mL4kgA4QxvEhYFTZecNPYjzWz3vRWPWqLe", - "mGHcMejN0mBe8WkOGFthZZymbdnXDhO4eJ4kK3gYbZTKD0sV8Uz9XaqICAEfW+5uYm60gUPzD4UvNaPa", - "0ll5AWdgP+9VvkFw9JJKC9VSnSjzr3mSBJ3AjqeE/HgDY2INelC9weUrZ70yJYigHzbtTcB/qsK+hP5T", - "OzlSwVsas2fmzdya5d+rPXtScnrlBm1Ov+/Loq1ov5SV7fpHky1mVhHh3HDNJ0WZ4giDQtg1Fyr5Knu2", - "kjVxm3fQG/PCd3+N6HwBP/aKqRYoEONFycZHBcgDC7lk1lgZ4NkjbpZdWao03S4KY6lG9Tfg/1kXnJEX", - "HM6rId93lMbyCB5zrqZcms2EizqKy7rwjW+ekb7ekixNtQ2H/ODNm98itWLMNFtReBrqZku4joJizFB+", - "IJxxLktsPyYzPKdc2EIh9nIw50zw/hlHjA3yvtCsemGvGS+spTuwbluEy49sHz343IaG+79wj4ovXpQc", - "V7nE7zjrFMCKJcJoLCiZoBRnkmhtKUsIMoWwbL0JgsMZCnGqMkF6Q/Z2RpAtplzyxeW196lEF9vJRQeN", - "M4ViLKbgODAPTcC3ICFPEsIiUyB9yGYEzykRmihajWPhoisJFMyfk6LOVm/I3tlAGijgi/KS3B3kKrmD", - "r+6iVKf9AqWCABMZzxOrFEUfMpGx/zQAy7rZCzfQC0SkwuOYylle0ijEEWGhF734/NsWY1//PuScqOVS", - "5g8SWnMrWfqQsTblawE3nG8jDOeRxRNz4cowtxDzK5Re2WwaVgP0z4vy7f+GW9rM1c3xgS45cxKv2sXf", - "xu1mznQ/bjjzG04uUJSZ7kq7Etj8e722zAUKyljl5tJeb9z27jIv2JOT+UYyb+uT+/PkFj6yb0QSdhoN", - "+6bSEMWkvwWRa6l6K5n7QM5B60sqecUeUATbQT2c+sRFScp9E2LYbLhcGpdljhIYbCrOfgjjujC2kTa3", - "FcbO47oUS1ISz5R10xg3yWXrnG0UwNYh8G+apFGbXUkQPrjgq96e7d3PrnHizQi8FC9ijr/3e5mQC2Fw", - "B+x97uPBvVTlu8v8gmkDPG6dXEJ0XNLj+9PTzSYpIdRKGSHUI5YQ1erbYeIpKvx6ToSgkatwfHR6bJMs", - "qEQiYz30OqFQdviSkBTqmVGeSQQAEj09P4fAsFyrtQK10AkIU2KRcsrU2lEUr97NYD7fqsLrPctJi/z7", - "3V8egxf+8QkpkB0QbGEmsNqKVFg1xrW6OE/KTFlmrW3hMc9061qyuHrwUzjbJjQmciEVSUyQ6ySLYRMB", - "NrwtHWi/M8AHHUSVRHo/dCBRPCUioVJSzuSQjclEq2EpEbpv/TnUqC/i9bzOe4VzqXlmRN+3EQuqB2Mz", - "N1QT1QABB8pVB4NgC6fpVoQVbog3tMP7giG9gOBOJBfJmMc0RDFllxJtxPTSGB1oLlGs/9hcGR06gu++", - "dmHE2+8sTekTNuHe2lGGZ3Nm/j5yZatizV0iPjqx9pKUN4uTP7DQfrEm18o1QXDcVTQhOUYLyhSN6Ucj", - "6nQjVCoamvTPAiHg/WkBEjBkp0QJ/Q4WBIU8jkmonHNlKxU83Bpm/f5umFIA09olMDgQeM2PE+jx6Owd", - "vJeQhItFZ8j0P6Dht4dn5iZ2gq2PoDRQRtQVF5foZOv1mnj5cyDTv3GUnJngylR974L/uL67OQBH4x6S", - "DVuUp6sMIJ5+92GcVoP74S14nN4CQEDKZ7MxFTgEpVjOMhXxK+b3DMx5nCX6H+aPk3U4WgqHs/fw6jej", - "7ZrhrO3GTfBRbEo7p4iY2nYPckFhCPZY40s14dwUQImpRO55T4FD9T1y99d3ypfp+A1eTVqKurqR38ze", - "uu+Tz47BwUOW6fFYtrnhNDcTxVd7n64wbfY+/Rzz8FKijCkaV7B3tN0GcNX6xwJe2F78gZoAicZIm3Y8", - "U4hcp1QA0FoNxQcRPWMJGBsioQzHWzBn0wgAJTsvFp5zCvn+YUwh45JGBKU8jgEM7mpGGNKzAUeVa6B0", - "TyttoaLyO+UrRsXRmIQ8IQ48etNnuv0DU/WCiyoS9LciF9+W6K/no6eq57kG/Lq5xy8Cwz7F1xDWHGX2", - "mtiNaOMlL340rqAOgrUZBrt9OQw6aBjsJMNAr8ARBhcqVmgfJZRlisgeOjb+LcjoPugjSULOIukwrJ0H", - "b7cvm/K7DVs2JAsfwHf3qfZYrgJSvrGd+MSDfg/p7yHBBm2UN5zdk1EHNl2EeKYggNvtK/tWRBS4Rzbv", - "/Qa2tEd+2PZtJPk/7PatyChYZS0uS0tvJHuOcrzW6+aSKmZcFuDIKMQpDqladBCOYx4W3oNM5rcD3Xwo", - "Y0HwpbahekP2JsdXtokQ6OjsXcc5zVBE5aVpwfrFeuj1nAiZjfPBIZAGxoMHi0GiIVMchTgOM0BeIpMJ", - "CSGHIaYJVbLBr5YP5S6r9RadeBbePczR1R6XM8nPE7B6BVvIGsdtmaXeEiSMMU3KTqU6cUD1hStdcPuO", - "daNcH8OT2F5vhYJLiWxTXRLTKR3H9rJG9tBbrXLghAxZGmPGiECZNHFHeujdVBApM5MYoxuAcuiGozqo", - "wAxKBVfWTRxzLqTx7GoOf3+KpCLpCjZ7Y1o+hTnfEZq9adz29EAGQ20MzceSfQXpBTGcYgiu+Ugf0w8Q", - "7GMG9NCo949l478VdDolQu8KbISsuRo129qR02z6SqZHYymX8/ytdqVc8lZL0dylSOeVmC8j9+IIFOib", - "3MB6Or+kjbBA9tHNsi9+1R+17Lsa5e8fhH30hbP8XipknpeCq9sWgCk4/LHVYimNvLJVKwkK6+EIWmck", - "3GWGQGvcgQeDG3jMKAO4knbQBCfw7TFC/36z4+67GsTj5q0KSkCl/ltDqtR6JNxvggPvBgL3gbNDbwGB", - "+03lKwGE6cPljX5TmUoVP6CrcfXdg9zeVYKSQboFGIumBCUj9WwgwUpD6b19p52ZZFv8njR4e/d8A/3d", - "kf2H1d/CZCgRy++yM7nRDreFJKlauMtFPqldAEr6EZIxfMAPeQzB3eEt3OJ6/euxh+PTxsv1H2Uf7+3+", - "vqiNf3L8+Gs9lvdc5WDZ0qdOF4twRuek2ele3cGWRKkg3ZSncLkSGYJZerizTGHRm35EtnmLVWX/hahD", - "CycRiqggoYoXBktUSwTTx18lElxbAvCci4XPmV7euS8ETw7tbNach3ZPWWdYceebLLoRVrg7d9JmhQvt", - "C27a3d22FniIMvTyZ7RBrpUw4NVooi0fRCc5Scl1SEgkgSc3ywPe7jd4NulHMpqO24xyBQz5awvzjsJM", - "Kp64tT85RhtQE2hKmF4LrepPQJNNBZ/TyNTLLog657Gh6nYDQW/qd9VKRV7QyRkXZnAPosO0OZCmH2la", - "FQsmdCEYBGPKMAxuLeB3dU+ZhCrdH6aQ1lDsHcc5wY8jzFp+G87Y0ZyojRxHRMW5gcbb/HHMPeZjrhyY", - "6s60ymnXrqJxu1jVliGkdwGYm8cx36/b+v23E15J5aOMrLSu83lukDa5zb8tFuzf3/lw3+7y9484HP8l", - "ccZ3yVUODegWfQzzioc4RhGZk5inUOzYvBt0gkzEwSCYKZUOtrZi/d6MSzV42n/aDz5/+Pz/BwAA//94", - "gu7dTX4BAA==", + "H4sIAAAAAAAC/+y963IbOZI/+ioIntkYaYakSN1sa6Pjf2TJdmu7ZetYsufsNn0osAokMaoCqgEUJdrh", + "r/sA+4j7JCeQAOqKIkuyJVtjx25My6wqXBKJRGYi85efOgGPE84IU7Jz8KkjgzmJMfx5qBQO5u95lMbk", + "LfkzJVLpnxPBEyIUJfBSzFOmxglWc/2vkMhA0ERRzjoHnTOs5uh6TgRBC2gFyTlPoxBNCILvSNjpdsgN", + "jpOIdA46WzFTWyFWuNPtqGWif5JKUDbrfO52BMEhZ9HSdDPFaaQ6B1McSdKtdHuqm0ZYIv1JD77J2ptw", + "HhHMOp+hxT9TKkjYOfijOI0P2ct88k8SKN35Yar4ucIsnCzPeESDZX2yv1OW3kBvCKeKx1jRAEnzDUrg", + "IzTBkoSIM4QDRRcEUTbhKQvRxdEZCjhjJNCNyRHjE0nEgoRoKniM1JygOZcK3lECB1dI4UlE+iPW6VbW", + "gzD9JFxPpX/MiZoT4Rkslci2gqZcIDWnElGmnwakX1wwJVJSp2y3Q8OIjBWNCU9VnVC/8msUcTaDabl2", + "UZxKheZ4QdBHIjj6M8URnS4pmzUTaUKmXBD06zIhMWYoiXBAJKIKUaa4m42hUc5je7GPueiMcUHGIZGK", + "MqzbHydcmB1RHv0b+ANHqPAuDA3eR2qOleNyxhW6IiQpTxRf46syGf/Y3u4+GwwGH7odqkhsthW+oXEa", + "dw729/Z29rqdmDLz72E2esoUmRGhh29/wULgZWE6kqciIOOAhmLVTIKIEqbQ0cnx2ztOoDMc9OH/tp52", + "up3hs+3+cP8p/Hu43ylOq0b48sg/r9565wqrVNZlkNlNY8so4wKT1Gf9Oo0nRCA+RUEqBGEqWiLYUiRs", + "wXSlaQ98SxFwNqWzVLgt6NtyJXLOsUSYGaHRq8iLvLFW+y7QQizk12wsSIwp0zSuDeKte4T0DkV2E+kh", + "BZwpwaNICwWlSJwo6XZRV4txhnCSRDQA0VPaVLvxQHa6HZZGkX5YGWG+2iSiMwovtCINlYVFct8ixRFh", + "iohsh7chTUksNnWck9u7GrlcbC8FJWWBf7qsSvNYS3hBAjPd7AQoUWRCAh4TpJsur8D2YHu/N9jtDfYv", + "hk8OBrsHg73/6nQ7Uy5irDoHnRAr0tML3maZVsvvo5xK+kVkX8yPKg/t+hUZ3I5dIixVtqthk1O1HGPP", + "mC5oTKTCcaI3th5DgZhN29o1WF0HR/mVBB5+EYEZuVFjSyHvfHz8QW4SEugjhrvtmZ3Yur0uolOEUSYD", + "NLsawbhyIs++aCKCYKkHrPUOfTr90UmZTBN9FpJwnERY6Xa1kgJsMI6plPrT7IeQSrMxux3H5GPG1Vik", + "jJkXGVHXXFwV37StjGnS6XbmWI4XsyTtdFedA2Wmhi5IhBMJ7dkVF2MiBBcdo2sux1Mu3CLpQywn4Yqm", + "ahSS2ZnloVCn2ykRIJOPbi5u3NmqegcHvQAvCaOmG70aJlMfeLGt+nCzoa2WlEYsG63ULTOyH8uyBAgp", + "njEuFQ1kK7kJp7Fe3piHHtF5nDWHaEiYolNKhFVUCRIpg2PNNYJ0I4gylMrKPsh06TFZaONnvNgdqyCp", + "E6ViKRQXr3DY50dM4ZjLlj/bKWuYtDx3ryWywBT25DFZUHO0lJUhuzTjUNAFER7xnZ2oRhSa99CG3uta", + "hDDOyGaJUmxBQ4rbiIMQxjSmHu45OzpB5jE6OUYbc3JT7mT7yeRpp7lJhmMPL/yaxpj19IbQw3Ltw7vF", + "tn/f9er8PI7T8UzwNKm3fPLm9PQdgoeIgcpYbPHptk/1SwI6xmEoiJT++buHxbENBoPBAd4+GAz6A98o", + "F4SFXDSS1Dz2k3Q4CMmKJluR1LZfI+nr9yfHJ4foiIuECzCC1m6cInmK8yqyTXlVfPz/PKVRWOf6if6Z", + "iHF2iPgIduLUqJNjpyfY79D7U7ShZUhIJulsRtlssw2/B1yTQx91vkMchorsO9pMVE5LufN5GwiC13Sn", + "32jVWX2rpWYlx7Fsat29oiVqTKOIShJwFspiH5Sp/d3myRQ2jDmhal290D+jmEiJZwRtgEsFzA8jTLVi", + "M8U0IuFmO2W2aTL/5JPCEVJib2CLHp4Ew+0dr+yI8YyMQzqzPrHqEaV/1yym21EI3vZPBA7zdvOALgWZ", + "1vt7CaIbOhFkSgTRPP6F3SWCLwjD1nr5C/Tb+b+2cmfhlvUUbgExz/LXP3c7f6YkJeOES2pGWJNc9olm", + "IyA1gi/8Y4ZHq9a6wFFSYbF6f8AbX2En5nrdWtpYt4VWbfBs7ScX+p2q7ATRmOkSBSnQKCJfaKXGox1w", + "puyDivuSz1BEmbE4tGpn1gL0qmVCfok4iMSvRIeM/PXNr8d9B+FlfmhoTT/rZgp4xGdFas4JFmpCSsRs", + "OMJsQ/noGsl/Vto+lbMKSzJeLUHOKGMkBH+x3djmTa3Ges0M2EVXVI0XREjvnoNh/UYVsm80NhXx4GpK", + "IzKeYzm3DrYwpMZZeFaaiUdbKzniMdjjrkHQIsB+Pf/1cHtvH9kOPDS0nkv9Qn0mha918+ZdpLCY4Cjy", + "8kYzu93+jK5ziJ8Dcmdl09mTcaBjTCPpOnY1rZ2cyrn5C2S3HhWcfVoMaPaK9N8fPJM+AiFhrITG2xu/", + "Dph5hmcR1zRdopTRP9OSgt1HJ1NwEOuDgoYk7CIMD8DvoO2/GWFEaDmVe4YKSjDaIP1Zv4tGWi/saS24", + "h7d7g0FvMOqU1dhot2fM+wQrRYQe4P/3B+59POz916D37EP+57jf+/D3v/gYoK1m7rRCO88Nt/e7yA22", + "qK5XB7pOlb+z9C8O3ydxzFKfaDlx25U+OqkrDmauIQ+uiOhTvhXRicBiucVmlN0cRFgRqcozX/3uV6UF", + "zGMFEdhMk+mWZKgYPcDGGxG/JiLQEjgimvFkVwthqmQXYW03g/BC+pT8dxRgpveCUS64QISF6JqqOcLw", + "Xpla8bKHE9qjZqidbifGN78TNlPzzsH+To3PNZNv2D96H/7mftr8P15WF2lEPEz+lqeKshmCx8VrPTeG", + "7Ipm1Yo46qYRqHkxZSfms2H9DurLVthNZNVKG2Oucam1EMpcZGsGUr/f1cZW7DEd3iyIEDR0x/LR6THa", + "iOgVsfsFiZShUToY7ATwAvxJ7C8Bj2PMQvPbZh+9ianSx2Gan/LmyrZyu0aCOQdFJYr4ba7TQFMEAwdH", + "K8/xVaTxUvsoa7d+6v/KperFmOEZAXPUvogmgl8RPVBzJ0CJRFdkqbWcJZrpRnsLKuGGh7AFWmDjdeiP", + "2MWcS2JecY8k+PbpgqCYB1fm6nfOwZJf4Cglsouu51rlAJ8gwZH9GZmLsRGb60HKgCck1EaIeQ2mhi4J", + "W1yiGCewzbEgsMdRjBURFEf0o7nCh1sGElJ9wo0YgY2BEqz3fBBwEcING0cEB/MCFf4q0aVRWC6h+UvK", + "NFtfmo1Zuaz+1Hnz7uL5m3evj8dvzl68PjwZ//biP/XP5qPOwR+fOiZUI9NUnhMsiEB/+QTz/WzU25CI", + "zkHnMFVzLuhH46353O1oGkjNXzihfZ4Qhmk/4HGn2/lb8Z8fPn9wCplxYy/0NvAM7LNXGTJnqUckHTtv", + "oETWw+TuNjTJtIh6dfZuS5/OCZZSzQVPZ/PyxrCqwa22REjl1Zjy8STxjYnKK3Sy9QZpxQVFVG/QTFEZ", + "Dganz7fkqKP/sef+sdlHx2bXwvC1DOLC6k9yrtkni/o4OnuHcBTxwPpQpk0XvK4rn4AnTIllwqnPiKsI", + "p/zVuozq9fKntxBFWxPKtqRehl5wO7oD39zZlHjBFlRwFmtzboEF1ee0LO+V12+OX4xfvH7fOdAHQZgG", + "1it59ubtReegszMYDDo+BtUctEYGvjp7Z249YdsQHKn5OJiT4Grdh7/Cu0fwKuw4lUTpbCzpR48WcpiR", + "BsUk5sJY3/YbtDEvKylmyyNY11Fn59Vzw5fDV8CSbj3t9VLWimm4ciP46rmP0ebLhIgFlT4X3a/ZM8c0", + "9Uih0rYwF2wZv8MG6BdMnyDiadgrdNntTKkgAURm6H/9SWJtAyw+lm+0PN/5PWetdN81Si2OEsrICq32", + "O9Eur7m4ijgOe8OvrFzau1hPVI15UF7f7FLOsUQtWG2CWXhNQzUfh/ya6SF7RLJ9grKXM7l8o2eCo//9", + "7/95f5qbaMNXk8QK6eH23hcK6YpY1k173S/ZRNLEP413iX8S70//97//x83k207C6DB30gft+r8wLVTj", + "bWwYovGkNlwqZwd/FuuiuLXF4XPkeG/tDbJPxvMFERFeFgSvHVNnOADpVxmVoBBgiex3WoxeIf3xGjGs", + "W3P6wauqf2B74Be0nkF5xvRcywp7LrQZSTaQ4fap/XO7PqSGEV3RZAwK9xjPMnfxqmjS8yuaWC0evjDL", + "GEVGEIQp6P0TzlV/xExwi147WGByQwKQeVJhhQ7PTiS6plEEziUQKvWjRdsEhagoeF0q/b8iZV00SZVW", + "9LkiyJpc0EkKY4GXJwSlDLur9IrabSdYj0wAslwRwUg0Nmq1bEkZ8xGyHzUSB6Y6xdJGtwmVJmV6Hf92", + "eo42jpcMxzRAv5lWT3mYRgSdm8CEzTL1uiOWCIhw0J3o/Uxtv3yKeKp6fNpTghA3xBgay9xz9p538ers", + "nY0UkJv9EXtLNGEJC22MsDtxbPxoyNlf9Y4lYbnZYv8VojdFg0iGEznnapxkcderpNO5fT234tv7Ibqd", + "RZCk5SXd7jbGjy6oUCmOtKwtaaLe2AAT/+6xOEx4fdHysXIvj7dV5Uvdts4a0zIEw3sjbT0+F6Mptfa5", + "FLwANe+LM1E/tRvsmvZPmBvISp9TbqV+QV/nppFa3I/5uetmdgcqnWQ0qXiqvg55DmXBqm8Vt27Ct4xG", + "KNHGJU5o3/KxNv0vu+jyb6Uf9N53poXWL66RoQbIE6Z/KrZf9Wes9TTcKlK8uDhY3n09DmVjkBRaDJES", + "mEkT3jbHCemjX0GII0XiREsyNkNUoiwqDDF+/e+IG6XGfTpiemjShJhYcmT+JklnjLLZplbz9cGEw9A4", + "paapSoV+b0FlTs0y6zjHTy0g1oyOGHkMyRWUBVEaEnTpnEOXZb2w7jqqm4TWl1SzcAxJwLIBY09txanS", + "3esJx1gFc00nnioTc2anXo4HrDio1t3F2rFkt3R3WP/zTFxUc2gWHhNHT87eD4FHseDabPIgWkXF7928", + "IktYcufJxDVfZtGJ6Xc1CiJ5tCD22C26QSeQJcSN4pR7QI0v07ov9fav5sf4HHvrlkLTqzX5y6aCJztI", + "qp6bbM4xVvt34eROCunJmf662jCWBIgPpscBAnXssmtsJQIeCMQ0s0QopIIEqtY8ZbMRg/CTS/tL37Z2", + "qTe51lG+Ss4VpDCA0l5cWlRYWaf2QTN6ajymSpGwW9YNrghJ5PpJafXa+rw9jnlBrgV1gszFI7dUzwib", + "chGQ2BoJX2Y4vig05jXjbtdEPRrE0LcwZpfaAYktJDShR2Y9wENbyvioJj6GFavNRB+Uu7zEUXSJNuxL", + "m0iQf0IQv10rxlnO7BdHZ44Fshvz96ddzZFaClzOlUrG+n/kWO/iy2pj9lu3w/OktKcDsK92d3fsqlqn", + "mxlwpdmyf80bUdG8NE79bryU03yhR2lDVNqo8kf5J7kT9oqysG0Dv+l3G71zmWLkLI37dtAlgvTSZCYw", + "ROd+Tffcna9cgZrNEnxNCrAvwjJPLkyl4nExVH+jEh1Cy3EkZWIteNQLscLgymzpbzXDrccsx0vTlLHF", + "mjwx49nEE3JEP0IWwYzO8GSpylcPQ28i4Jfef7ux+JalKfbfWJAkHCu+OvqZTpF7t02wo0lVUHy8mFK+", + "OjPEhs6UUvfMcWTtWt1ELwmodSeAjhPMTXCqIQIoje9Pi9d+/RHrwfF7gI6zDrJmsyYx6JY4NDcnG1wU", + "BmFyQNBkuYkwen/aRxfZaP8qkTZYFsQlQ8yxRBNCGErB9QynYc+cxcUBpBIOTVX93PpOTN7EJtxucvus", + "n6Urg5cmS76GKKsJrczHJF3CQtnrZMyKXrBWXqtVMeNvyYxKJSoR42jj7cujnZ2dZ1X/5fZebzDsDfcu", + "hoODgf7//2ofXP71U0N8bR2WZYuNWytKn6N3J8fb1lla7kd93MXPnt7cYPVsn17LZx/jiZj9cwc/SPKI", + "X5Qd5wF3aCOVRPScmNRc5QuzK0SzNYTR3Tk67p6C3fLY3VXvGkpc6DfvIyvGF29to31vn7dSFZhrI7YL", + "k6tb8ssE7M58lxQ0OBsYGVBvCOgxlVfPBcFXkO1XP7djPCNybM4zfyhEKk18Drmx3g3BuZpKc29a9noO", + "d5/sPt3Z3306GHiSQeoMzwM6DvQJ1GoAb45OUISXRCD4Bm3AhVeIJhGflBl9b2f/6ZPBs+F223GYK552", + "dMgML/cV2rAU+buDOHFPSoPa3n6yv7OzM9jf395tNSrrL241KOdbLqkkT3ae7A6fbu+2ooJPoX/hknOq", + "CrwvKfPQAAPof/VkQgI6pQGC9B6kP0AbMRxhJLutKu/JCQ5d6qr/7FCYRnJlxITpzL5pHG1xGimaRMQ8", + "gwVp5YuGmR9DS15wDcayVOHbtWRTmtZGCLi5ZK+gUmpaiXSnJhe6oDxREoUHZoeulXOwmvnAPjTxgZ1D", + "S274XZtOvYgsSFRkAnN0maRcQVDGJ2bRSrOibIEjGo4pS1IvSzSS8mUqQBc1jSI84aky14w2tzvvBAKm", + "wfaYanHdzs59ycXV2tBTfRJnKexrvUKH4EifWlcNnOIY2a9ddkNB6cuuA82lqX0u0VvzhfEQ5T8naRkQ", + "pws9WU8SQ4JIxUGSWoehbaatdunXW8BZ6sI/TH+57Hyg2Jfe1IQLfF0LW8wIQDeotRqL5pQLeP8cXm8d", + "ya4/XOtIaUF3Rq4fgugQ6t/TbNuTDCf3Q/FVwWiZryF/CU5hQUPSR7C7ICrGpRZWdtq54klCwsz/0x8x", + "Gwqe/STNDYr+0NBBzQkViAs6o+WOyw62+4xquw0rOm66MzsWP6xrqPAQwjeaNz2eKgPTcOWyrUgx9cku", + "QqfbOc9ALawkKpPmbQYMUqNIHqVZG+Krs3e3jU1LBJ9SH1QRxELYp9Yyc1Fbv+8OznvD/8dEYGp+AxWN", + "MhM/EfOwgkFh32938rw6e3fWNKYMFQIVR1ebUxbxsgoXy1HEXirZW0lrwTj21wdL1kmuez/z6bJTgWMy", + "SadTIsaxx7n2Uj9H5gUT2kQZOn1e1me13tzWaj4rLQ6YzVMc2KT+dtT3OOQq0+gWqPnBv1xviTmGm1IB", + "9VIJ+47NBuyj1xkOB3p19k6iPErJ46krL29jqP3ZfClpgCPTosnspazoYAPmbK0hn+UfWlekR0/2w7e4", + "jYA2FrMkhW14/rZ38ub9VhySRbc0JogsmvOI6HFvFqTFwiUE5nkBJSGxaPJ0GMaQbTdQgVbZDm5NpMJ+", + "9VBHcYWjsYy4L1jjQj9E8BBtvH9pErb0CLooKS2l/r1AhRJ/73t3jJZITd2eQ4dVl2lpg3ttxzKQpnGv", + "FKZX6tS3VUyUfF3HqWMn8avyQvOr9Xg9ppHmfo9cIH/FqW0VSWTi/RHE+7v7Z2Q+NV5r6xqRJMECKxIt", + "jWaRHX0RnZJgGURmj5P6VSK5IcEtMgle6Nc/m0TgVJCxmgsi5zwqX0DvdOu4bRKCIBfE4meYORUc74qj", + "GIsrOBidIo1SZihQzhrYWYdJOVcqucWkfr24ODPWtSJiYSLKilG6sna1ekwivEQToq4JYW4qWCKMXvEM", + "qKSaViMbMAyEGidEUF6mYWfH0++5CcxEM4EDgsxXDkbRLomEU7MtKW0vHiyuICBSNqzvcNX62k+nadRu", + "jX3DGq5FHQ1us8AXR2cuFz+DFXRk3q5T+YyIntlyDl9w9dJuy9WoEK4rxhmpdyb4hABMhM2cKeb4uMAS", + "gMHQn5fyagrCQRYTWmw/sAsMqbpmn39opexVt7vvIj3GLPTBM5qIZ5O8OEtjvSB6yCKFuyMamqgTk09p", + "1PJi4LYgOKSMSFnJ/ApSEXW6nd7UzupgayviAY7mXKqD3Z3h063V8XsrAzdtnMo4pKvsOxfNYuIdXIKU", + "gUmESZdZYgsnSQsPmKHjmvMBxFM9UAwgGfXZVtDwXOTwYFBLYrvBgXLINuASK1154uK2TTRLluYDDWa4", + "wHvPnhX358B7B53Dcjv236rxvp6ZCSbTPGLcDRU6Gib/6NWouPDBl3ChbIrKhLhos+w8dLFc9lKl1NnT", + "wdPiLFuhH4OwqWxzu+88UzVvl9PWSIHcdv/aBiAupqxzuC29GuxM02UNT2mJWGOp1QTlCWG3oufe7s72", + "7ejZdiInLquuIpd8SfNHp8dGJwo4U5gyIlBMFLZQ7wUhA74kLWW0gR9iEkOewvTfV4uWhviFYhZ84w34", + "UQ0v7V5uvxtwft6a+M0QxZjRqRbI9s1iz3KOt/f2DwwaWUimu3v7/X7/trnBL/Jk4FZLsWXyHwtpwn05", + "/7J1uIcU4DZz+dQ5O7z4VQuyVApzaG3JCWUHhX9n/8wfwB/mnxPKvKnDrQDs6LQGXFeOB9MGv/n9oIA1", + "7vSeVnjCfmcwhIUCXoEX50XhmdZtDMd9KaDLnSHfctxRVYB6K2bTtIB9ox9XXyM7rxK8Y/tMmaJRjohX", + "v0C+E6ahXAn7VIN8SgjLgJ6iyPwVcLbQu8KH+lQ6idyzLwq+WKl6/aOuca3fb07xWsO2fi9bJv/aot1Z", + "TBrPSfTNpf5dApTKvb+Z/cef/688e/LP4Z+/v3//n4tX/3H8mv7n++jszReln6+GI/qmmEJfDUYIonJK", + "WEJtWekUq8DjjdKGTgOF7RNjW6tg3kdH4DU/GLEe+p0qInB0gEadSn7VqIM2CNgE8JVW7HRTNk10U398", + "Zu7O9MefnML3udpGaPNBhV2QLA1cppOQx5iyzREbMdsWchORoAHrv0IU4ESB44IypC29JZoIKKdi7zby", + "zrvoE06Sz5sjBtcD5EYJPYMEC5Xhp7kegCnsqEzApX2dhA6Qx1wvjFh2LmV4POaCq5+puRDYUE1X8RNl", + "taVibYSnAx9yEYTM64WMqFRG2c44W7NRFsuPng4265bLGm0646EV7Ac7oV5nyTFli71kGBi6NoJ77Jxx", + "awIRtGwyewSBraQ4/PccuYZyWmRLbDzkJoFCmgtWFclC6sRmx4vyDqvbckLmhhE+i1qkXL8wuTUXv58j", + "RUTssh03Ak3OKQ30/CB2kkqZalakGB0enb7Y7LcoFAW0zca/Yh0vshlWM2PtjWPTRWpu2OGYdNHJMeQ2", + "2R2aK3AQk/ySCxQZAZPv6wP0TpKKjQjZaxASaVYyWubXluYEGHU2XYtJVVIcoLeZ3oizoZSKU5VvQvN9", + "Cc3aqBUTMF1rvVsr+yKcXWRFG4RHY5Vl2OkTt1kUtHdUWIrDnq+Y1bfe28Wb5EajubD2Xxut7uurOzu3", + "U3dc5YFkjqWPu+fFqxB4aUW5H1rx74rme/9Sv2sr7mTdQREhffQVP/fVStnrDYcXw93b2/y3BRorA3oU", + "wGAyrLH2IGH3AbZVt39vqBo3RoQi/djGfzor7/0pmmPJ/qrgYcXWG+48aQUbr3ttG0tZjKLkUzOkTEo5", + "dJAsBtDgpFzRKDKhtZLOGI7QM7RxfvLqt5Pff99EPfTmzWl1KVZ94VufFphjTlS8OnsH12lYjl04UnMG", + "Ds6z2MgNlUrWwVNaRfV9CcaZ+bQdLribpGkjBwhfDZT2awnMzIuGs/kVEc5cKGaNjA+BXfYtc1y+P9y0", + "lUhnXwpXZo2Xe0IraxTuPqSvspw3P39d3LF7Gc7aKnvFs94lIN4Z6KvboZ7kq0OpRTAJ0clZjjWeOxld", + "85U52ZKVw8GgPxy0cbnGOFjR9+nhUfvOB9tGtTjAk4MgPCDTL3D5WsY2yjiOrvFSopEzl0YdY58VDLPC", + "trUmVavr6Tqe2t3g06oKjV9OG8XO3ftLW5+mSblpkV1TBXuJ08gkcBZr2tTLdMrEIJsZUN9Mlx0xGGDX", + "IqxkpTVxEIg092e4mmlG800Ty/cjJohMOJOmkmIf/UaWEsUU7hCy7iFySKIsiDscsQ3hAv6zyP4Ep5KE", + "+geIpu26qE09NKoAvFh/MGJynkIJuM0+OuJMpjER1tWDJhT80JtIpsa4g/ECNaAgqaQhESOmX/Ngp33K", + "FPWD/cFgMMhK03UOdvS/Bz5u8nPROsC7dkh2XwoftqqizHm5lkxrm+oLCi62Cvd2WpUN9LZfjW9zW0VQ", + "wNMo1Ir6RItm40choXX3SKLyMj0gzd+xK6YZqjR1G2amOPozJWKJ3p+elq64BJnaKiQtJg4s3bAOPLnV", + "MmyvMW3XjuaOoHIPASRXPVYL6sxXh40r+thdwpnh0Ba+9ty88QYJU2aWRvPJijlVvKQhWYzT1Kc160cu", + "zfzdu5PjEnNgvD98Onj6rPd0Mtzv7YaDYQ8Pd/Z723t4MN0Jnuw01AFrnyRw97h/r43kK+zkov/GLgrR", + "W7u3IQa0ckjZuLZrykJ+3arqdda7DWJa1309RLH1ELyBzVAsGFpqkBKnhZLAJrCvUvanwfezfzEYrvH9", + "tKtk3CB/L0TKAoP/BJI486oWaxgXF6tex/T24hQG5AKI11Gr2Hl7og0O9p4d7H0p0VwQ7LoxVtnpARe3", + "6crfgQhWomxdpkfBg1AoAgz6hnF02qDcTreTxQ3D33DQVmLSssetguGbNmzXL0ZWye+GnLCTkuIK96AG", + "Syg80FpAlk40SRXKUg21enEU8TREBe+LgVaBq4mTghKrm4GbAuucMRhZJqhVK7sAygiQwJRpQQxXMroR", + "m0B2gF7Bu/AIx0a/t4PALKzcRuBwaW5j9f5yXRtte/WQz62iDd9orRtBbW49bU0G66Rb3YTRfA7Qaw7f", + "ZGo/41Vvn3kd9O3661XP4IbN7HIZwNCZw408QNUcSE2pmCtzCGPI5OuBLphgzeVZ0ug1viJ2NfQ70nSn", + "27Yq4gF6mamFmWJpFckNSeyfYysM86T+zVJqpeWmjubEnCsKWYPdjlmtTrfjFgGyC+t5hm7OnW7nXb65", + "atu8yPG+aAiCIxAZeUpXqmhk4SxhUhSKYNtYX81DTWqMhW4n4djYP02xTSZPyNpI2UdOS3p/ijYAuOrv", + "yHpM9b82szio0pG6/Wz32f6T7Wf7reAp8gGu126PIIutPri1qm6QpGNXX7Zh6kdn74ytHRgrFlzwdu6F", + "bOBEcC3h9MzzgrV558/6z4qoHCFPTfFuOyQL4fO5UKJ+ZXXhhmCeP2m0oNMp+/NjcLX9T0Hj4c2+3J54", + "PZd5LXyvm+ekeK1b84mSSc+U2/ADJwBDCdmILfKWSJgBOicKAf/0EA7AQsmSzyzLOQQSS3EvY+3u7Ow8", + "fbK33Yqv7OgKG2cMTh/P2W9HUNhi8CbaeHt+jrYKDGfadBm5gATKrPXq32fIFu0alPXe/nCw4+OSBv0g", + "5xrb9iJuJPl7awHaSVmiQw5dZh3WdrmX2js7gye7e0/32m1j644ci5vVEsZFmBvyWMDa4spvgNJ6cXiG", + "IH9rioOye2a4vbO7t//k6a1GpW41KgBbNiCptxjY0yf7e7s728N2IDm+634L/1TasGXZ5dl0HqbwrIaH", + "FHXR2206LXxam2GwtySIMI0PAxeqWzl9DBjqWJjX8kVoczBYD3jt4GrxbSv/VKU4tNESuEApyyC4++vv", + "+u52ddcsps15sF6M1x0IEWaaXBbNwdTcuAPtEkEWlKfyKzTElclpmkaci1t922QIvSUyjZS5X6MSvT/9", + "KwgRzVxIKpKUbTPLfiswL+44uVtt4BJP+Lm6iVitVqPN0q+acLdhm3ZXJTyXtn8jtEyoRVXK1sfZHeEo", + "SAFlHmfrqWcFIBGQspkk0dJEpEYR5wwFc8xmBAr+mZoWbIYwmvMo7HujBPWT8dR7P8+vUcQNKOYVIYkF", + "YDeD0J9pnYUuCNoopIwiw0qVQkp7sZEqFmK7zI173mBSQbD0ZTlkqYqanljxAl6j+aTkyoz4TIKxqSDW", + "tl+FCdaWlbEWmCkosIiNjVrG2NnWp71niBXp7TtCzdHJp9ZwtjoGJAIaSuJAcCkRiegMwOvfn1byy1Yk", + "S2RZZutj58qDbcG65sbMc3bBmSZb1x3xHYieKPQvORKBhyHZZEVUmnN6xpilAMleYGRyk1Bh2KNd5Nmc", + "SzXOcENuOVipxgC3nQqSgwtlWZGZn8m94z0XnWi7C7lsiOedvq5xlb+ppgE2y1QvRf3U6mY86GPjOnLK", + "SrCWHP2lCvVxG2yfHJ+ZSmiVFmBl0AbjqiSWChjDm22iMPw2qu6nZp7aclK/7w7O28LurEbZOcNqfsKm", + "3JObfYvbTufhtoGBCRExBcB5FBJGSeiMx+za07rQIKEvkgSFKbGUMwqpwJbg2GxvyK9mzvdG2awi66sd", + "tnE7mzGsRuOGfu2LbeJppD8N7EKkQCsTAScRzhPCWoUTUjn2X5PVGxZklkZYoCq01Iohy2UcUXbVpnW5", + "jCc8ogHSH1Tvsqc8ivj1WD+Sv8BcNlvNTn8wzvMZKnfTZnA2m8UsSKXffAq/6FluVnLpwPWyZb7fgoz+", + "NuFJ3qDclzQiFn3pHaM3BUYvw9Xubg+a0iwbGi0lWNaRu24ruS3L+na8A9U6zMpbei5DTXht5Uq27Ihc", + "e6EI8durkkrrrhi04SKeHBxwma4FWN5WnpB2IeTV2D43mi1JgnLvu0/3nuy3xEX+Il+nyVX/2p7NRbzC", + "o9mwUqdt3GZP954+e7azu/ds+1YOKhcG2rA+TaGgxfWpVLGtOM32IGRqcKtBmUBQ/5AagkHLAypVpL3z", + "gD6v2LpNMQz53my6S42KK+nuWcoe0HY+xhXa0mFJ5SrUa98g0ykBo3Js6NbLB1NJBWw1hgAnOKBq6XGY", + "4GtTmy97pYLr1sabVh6sh6S2bYsBpCWXTCd5tsSG6xz9zbjWK7zwtDW8ukwnTW78N9VejRM/9wEVr4ha", + "3NDkFSDr7oJsPtdYlkLa9N8BhEnm9firwbHmjdUYU1VkD7gEtFUECgEbPmzCyvlnPyouf2U5C27fkpJc", + "pfiqI7R5C97KhvacyB4TOlif8lKRD/YAvNtX40mx8MHKyhKlKgn5qXv7fltk9dRRQbMT7Pb9FfIbbvNh", + "FQEL+NGOwZI8b7tbYokGblJcrC/9dQ9IziZ04U5Yzjbq4UHgnO3P9wLhXFuOc6Lcu+faok+jFZW7SmCL", + "JbPFRX24V8p+VCOJu8i6+NAw3qwEWO3O/bqahQtpGTurtcBxIsiU3qzgFvOCOa7LuVHSUiAsV3eTaCPG", + "N2j3CQrmWMjK2BmdzVW0LDtZdz2piV8EbC6I0qpz+zp4+Wq6D+s3GnY5i637tux5IZHQX5+PhONVoDpH", + "2WvOZ5zgJeiWjYbgk53dwWBne3AnVJ2vVTaw0E5TOGbhO+vMKV09FlvIgt/rtSWuBTXV5x2ZpBIExwcQ", + "tJXggKCITCHnPKvps9amr3W9evD2ktRmteVhXHah7Lo5P0sZtjLrygISuWl03C1tOZGx+Lw+7BWJ6ZmY", + "CWoZ6p54zZ3eYP9iuHOwt38wHN4HDE9GpKYQnicfh9dPom083Y2eLp/8OZw/mW3HO95cuHuoUFnOf6kW", + "rLRzSIioFg2pFtuRJKKM9GQWAbc+xHWFLDA3SWv3/+28D2YGK5WF8/IkizoDVjlxqrXzHyJr145+pQul", + "OvyT49XDvlMcWXUgfgarDgX4qd1gABZu+KUYZClree68K7zY+uRZGdu47uzxZbzA1vaucgPFffxcEoyl", + "HbbqxK6fah4TbsYFVfN49fGQvZYhGsFl+EepwnKWaB+dzBiUCCr+nN19FOPY9cedbif6uFveM/b39vnC", + "FsEnY0C71EU1oMXdAFSgWk0FeCU3LYQJT8CCACF+GfaGz+CGPvq4+8ug96yP/lGIFOgaahXJN3Rvl34d", + "tKFhEXfb4bUOn93qGt3RcxUH/UZ9qNH5QWyxfSyP5/VZ3FnhIqhLC5w/rq1xJaXy3ipC29NsXNSSQhLh", + "pa+qpkGOB8h2WY1zLzSAJmRGmewicuOEThWFHEX0iqBRZ2cgRx3EBRp19uJRp48OLSIWWKt5+a1S81B4", + "qcAnNLbFwG3ppeawlO24XfJY1Xi4HUyi+8qjnvX9+tmz9fk066Lo1h2T/S+Iqv4ic7edibsq1et3LFVm", + "kwIgO7zYhXrMrFIVwNbAs5kREN4K+e4HLr07Z1krA2Su+DlPSBfNuEJ5TkS7pCuRMi8/lMdPbiBTaEWC", + "lWGI7a+SPZdlctNV4uvkGCWCh2mQRwFHMOg8PUykFWipFVr9+nvW+3RoQHj9lAu03qHR5MFYjyNBbprX", + "+zW5UYUuNcM2L/VwsH6p78UL0u2kSbhehpmX2kmwWyGfrYkr9fhkymSvaIKFyXxoIdHfFilYN3KhujgK", + "tEqUJrZ8DfBUnZNkvTJNjG/GXhCgYxIRfUzVG0E8CvPQFypzKbpepA73n/rdhvhmHEBGem0gvxGSaFsF", + "kmWhvxizpXdg1ZJvaGPg6nhIBM33DHKqpVZ5cE/WamKNS9W+el7Fq20S4IrFCjMIsK9bOs9+ubaw6X34", + "4b6lkvbGXi5UwF4cFFEGnub6N2G9ABbFosp5vbsn2+lkF9YybgKwqYb2Ft3Mh73/Mm5lNO4fbP3y9/+7", + "9+Fvf/HXTC7ZzZKIXkimcN95RZY9QN9F2kbvl+FbADlQK9O20rAiOAanUXBFjJMqxjfF8e4NMqGxfI3j", + "2hTgojimLPv32gn9/S/N16wFMr4DObmWZb8YWPM+qhYo7o6jjZiImaur6KIDN/sjBgGHV2QpUQEa2ao0", + "jlH/KrNPtIoOTkscoUujBvYJW1yiCQWEeTli2qrFQUASbU1YiFhqSgNxkD6C4KjYjoVodtH8Wi6Z5HSb", + "XVXD/nnz7uL5m3evj8dvzl68PjwZ//biP7U2Qq57poewp3lvd2/fFgQqUnLoWeI7wxD6+MUAm3gYBEJf", + "oUySR+OlEhJXXHx24WW0QeJELV3tAhdBu3k7oJXDrEHvpflXBnUdPPsaGPbvVoLWL3jU0ypxAyaf1wNp", + "aOEN+oKmTDBdp8kzPfNUFT237sAZnWGPM9pb0OxrYM27Aa1FwKmtfyNStD8E77gKWGi2syFVBWCvYlhK", + "1WuO0Iu1JjTOS0eVETdSZoNYLVrLLK2Wq9mKmdqyNSF8iTOhPjpXhy3nu8zBLwDUQIto3JW6eGFmhZE0", + "r82pUzkrSvEKAp1p0lzPiSCFhYAPciC4W5LMhpS2SMcyyO0JEb2MJVw8KpRSFRRiVDNvgSNBFnZcd6Gu", + "Bvo7xTdZD+B+x7J2SQXzyCF3h6+eQ3nkt67yGZ26JmAYFYPAj+pW5qJVNHFcVV+MIlfV523e9248K6tW", + "SL+mvVVhzryPEmv6+PEfmKqXXIAJ0Zz8dO/gcGCehERA9ncV+q0VbhqNSTjO6j827X9X8tFkPmXVNfOq", + "EM5cwsDEWsitR+536Tn5GOqU1uQgQSqoWp5DnTgD/EqwIOIwNRvelZuzP+cdQ5GFz5/B0Tj1xDq+IowI", + "GqDDsxPYjzFmoGWj96cFYHSDkV8DhAH98M3RiTVRHaYQmBxUAevpt2PMdPudbmdBhDHTOoP+dn8Amzkh", + "DCe0c9DZ6Q/7g44pEAhT3IKaTPCnTWPITJ2T0OpBz80r+iuBY6KIkJ2DPzzpAIoIU+NJgsKKZwWTI8FU", + "WJsjiSBJwbAK1d8CSqA7Sg/MeWzr+7X2sEm1tCGbJHljl/WD5gSza2CK24OBxUxT9uCFgFMT5bb1T5va", + "mffbSp8D8ngg82omgdMpLck/dzu7g+GtxrNqGLBjfd2+YzhVcy7oRwLD3LslEe7U6QkzceTIIJLYSJni", + "PgMWKu6wPz7o9ZJpHGOxdOTKaZVw2aQME232M3JtC4z9k0/6yN4fAAq9nPM00tIEmSB55ylQWPRnHxEW", + "wZwuyIjZczpOI0UTLMAPECN9PhuLp7w1TNdm9bP0xec8XFaomzW3pZvrOa9xTuAq2rQkYwBVHDdVD8z9", + "xZRpw15/YqG5szJa9XAcLS7HMuC+qOULwjBTPZmQgE5pgOBlvXutS9rbYCsAIC3wYFmIAFwO52LZ3vRn", + "vgCIuD9p7Dh7hix5y+oEg3ucIErDXOdywdhYTHAUeREiZhGf4Ghs6HNFPCrqK3jDEqWIt+6UG8ZDYrCz", + "k6Wac2b+TicpU6n5eyL4tSRCq0C2JoqlNQlNFRTDutcAXBZDXRJTcU33uWWGuPXpiiw/90fsMIxdNT1p", + "PsGR5PrUtDWMwCBw0buGd/0o7w2BIUepVDy2LJUVdMqHyVOVpMpeikuibCEXeJ1KlKRyTsIRUxx9EmRG", + "pRLLz1uf8h4/g+1CcKj5pPCKmdLWJxp+bhq1HGM9+zG86rH+CBBg1NGny6ij/54JrG2XVM7BCyLB8zEr", + "LulGlrWv9cLNKoUDzFDCE4N4AEw1x5rlSm1AUSwcRUjBVnLfam0TVrJhPjaJKZ40ZjCZlJPKNqIMnT4v", + "bKbB7lP/fpIkEMTn4PiP8zevERxVeg3Ma7nHydxKM32KojAFTR5674/YCxzMkdGbADxv1KHhqJNZF+Em", + "jDWVNsS61wMV9xeoom266dLwl35fN2W05wP0xyfTyoHeS0k8VvyKsFHncxcVHsyomqeT7NkHP0GbEkHO", + "S4IAbRjZv+lKGgIgRX4MmnMDsxBxK2ujJcIol0BFP8qEMixW1mP0kN5SUJvyeCaLxPg0AufrqHMwcu7X", + "Uac76hC2gN+sj3bU+eyngFWimyHUTElKp2tnTLQ/GGyuz9C09PWo0KUX9fb7XNO+tr+a4mGVrrriYSbn", + "YCb1CpriokbdegDN5zkOXbmqnyreGhXPei4Kyht8XzwHDPtGxBi4FQ1M27OR08BWWieGLQD9GSwOl09t", + "DA7qNLiceYvmR9Wcr5sVu027LIAhRo7/dh+A/6DfrLKR6ffZQ/WLIwBNdSi8j4wdYbEcI3b9FvEror4H", + "jhs8lCi1KKzfkn8fC/+8Ilbvy4lWkWZbZOHum/yoEZAtIm0r5mVtq57DmHrnhCn0An7t2/86iwegli8j", + "Prs8QIaEEZ+hiDJ7kVe4LdKHoqUlfGQSRrLvbP6Ig+zaMOfn//73/8CgKJv973//j9amzV+w3bcMjgog", + "CV/OCRZqQrC6PEC/EZL0cEQXxE0GQDjJgogl2hmAmpkIeOQpgS5HbMTeEpUKVrjwNOhZ0jYIpgeD+VCW", + "EmkTbvSLdGqhPYyD2WPCu71sSPmgO7rrwciHGRQmoE9FxwOQq22ruVj7q+P3npk5l/xnVV95zWO6Xr4o", + "cqMM9/bMAG8pYIDEvn0HD+yk0cb5+YvNPgIbw3AFwLeAxpw3Y5Xn/k+ZtF4mGYlSFihAZSObTBbaav/v", + "sX2nnQPYtvgjeYAtotctXMDG5QFlSN0K/LQVWriD/XRzrmGff/bYpVk2O2jvPt9iFy4QqZUh/PXW2fFe", + "nebmSYFk38IERhsunN1Voj47OnEl8ja/GdM/yKmhZ2oLD2VHB+Km/vWDmWVHnE0jGijUc2OBKhMxyUy1", + "MoM8FnHw1o4aYTevKlBi8XzbKuH+NJ50GQRQfuTd/+lR6fQ2x0gO5pjz2s+TZB3rHFMZcP1tgVt6AU5s", + "3WijvmT7tMhF6xxSJjo+O3JWqktWPJ8cuw35cK4p23XKqmfDAwjF44pA/IaCsFKLtgB/+pi4+V22ig5S", + "YoXn6vtizcHDaUEP7cXysfljcmOFFbJpKWiishsP0FdEmVjszj0utO3BM/FzItyudmjVMOtsWuZTUyXO", + "TAgupFfbvifmlXamr2nvR7J8gTy30VgsyX+qKC2M3ZxWqwzcE1t/9f7sW+jhVubt17vntQzmITIEm0yc", + "x9oUEcRyyYLNH+qq90FOM0PsR3mYnaVR5G48FkQo9OboxOys4hmw9QnCktbr9m63rTwO3r39vUdYwCEO", + "LYuh8itR9slX1vDNgpmp/GSTNjahyWum7jxr0nC+YP1tnX0T4din/N+2X0Z0IrBY/tv2SxwllJF/2zmM", + "sCJSbd4bswweSjQ/tMb9iJlPK9y0TDQQTQyK167TULO3Wiqp7v0fSk81k76VpprR9aey2kZZLZJrpb5q", + "l+JeNVbTxze6ksmYzUdteOTiE38wTfVhvXyWIx3SM5Xlaw9byocL8PPCI8pQKskjDKCkGccVj42W7up8", + "Q648Phzrnhx3gZBdTTqAXbIJIg/kvHbjeHDl1vb78J7rw3hCZylPZTH3JMYqmBNpk5UiUhbAj03tzo/n", + "RsX7O+bSwUMeHQ+uV//k+3vS+KsLaoS3uYFap/O7t9rq/PZ9rfObFGqbu2axoboON3CzIajQJVG3ZeNS", + "rnk92NE3Lp8tgt5pQyU3FxBYEAcj9n+0/fGHIjj+8ItLkkkHg+19+J2wxYdfXJ4MO3WsQpgS1MK8Hr4+", + "hmu/GWSfAxJsnpJXHYcpHQGs57Bv/uUMpPzms72F5Ljwp4XUykIqkGu1hWTX4n5NpDJ+1oPbSI7ffAS3", + "ICY/raSHsJJkOp3SgBKm8spqtSAxW5jxEeaWMXs/VAjuKB20ra2kbFOuUUBzXP8HD+w5yYEMH9o4ciUE", + "HmeMPE8sJrc1R/LDsNke+d74YfCwwvnh7ZDHzGJG4a+TLtE6pa9gJ4BExqmCoMQcIQSiPpEwWnvWYh/l", + "dTJlmiRcKGmAJkEBNlD0c60A+0ApyziTPmBJgA+mRHZHDEoN6Mcml3/riiwNjCTlLEOMzGZqoSN9uVdl", + "GM9vuo2+vo7lxyhtpWM98Da2qNPfTsf6ZqLjQTStkxKY/0a2McCgnJBsJ/MsuY9+pGy2+agiUI2wyuZW", + "wDPyqFpbUKrPAvNuyawkcNNBW8DmtYUs/wVP3PokfVq7w6ItEBCFFM8Yl4oGLnG3Cnf+84RufUKvpqyX", + "m0MSc0V6isRJZFEY/bb9MbzoaHTh3v/B1Ec3b2ToFhYq5PwIx0HB8AbsaOT4Bu4hsUSQKj/l4uqRXePo", + "xSxOZ4KDq2L1I8kRVYCkNSGuBm+oJ51Zqp69NbVFjv0b6iUXV23VR0/RvUegRRZn+B366fTwAGnv27vr", + "wJFlNC3NNA+uYdYqKX5LyUKrOmcQpaFWMp2y6cy0qeDx2P5osKD1rrBIu+D+C2yr31oY6d4fwBn7mitE", + "4yQi2kImIeoZbtKrac1qV4uBykLd0dsJS71tislmBuhRurpdVmLCxbVbsA2IYakvl1dqRny2HmAm69yh", + "qXgQZkbM1IogrrDEJcqELMh2EpFAoes5DeaANgPyHsodAxAMTpLLDF5u8wC9gp1aRNmDzjckERRHmtck", + "j4gBkVnE8eVBHQ35/ekpfGSAZgzu8eUBcgjI2QEh9VtF9JisINhri4mzoTlJ8CgyK3qpMI0K89u0uDI5", + "/N+I+TBmGLm2DdIpuizAzVw24M04gfo7n30zS6bbDNpq5qI4EkA4w5uEhZ2mS04a+ZFmhgNvsaSWqDdm", + "GPcMelMbzO98lgHGllgZJ0lb9rXDBC5exPEKHkYbhcrFUoU8VX+XKiRCwMeWu5uYG23gwPxD4SvNqLbq", + "Vlb7GdjPe5VvEBy9pNJCtVBiyvxrEcedbseOp4D8eAtjYg16ULXB+pWzXpkCRNBPm/Y24D9lYV9A/6mc", + "HIngLY3ZM/NmZs3yH9WePSk4vTKDNqPfj2XRlrRfyop2/aPJFjOriHBmuGaTokxxhEEh7JkLlWyVPVvJ", + "mrjNO+iteeGHv0Z0voCfe8UUGhSI8bza46MC5IGFrJk1VgZ49oibZU8WilS3i8Kolbf+Dvw/64IzslrF", + "WSHlh47SqI/gMedqytpsplxUUVzWhW9894z09ZakNtU2HPKTN29/i9SKMZN0Rc1qKLkt4ToK6jhD+YFg", + "zrkssP2EzPGCcmELhdjLwYwzwftnHDE2yPtSs+qlvWa8tJbugXXbIlx8ZPvow+c2NNz/hXuUf/Gy4LjK", + "JH7XWacAViwRRhNByRQlOJVEa0tpTJAphGXrTRAczFGAE5UK0h+xizlBtg5zwReXle2nEl0O48sumqQK", + "RVjMwHFgHpqAb0ECHseEhaa2+ojNCV5QIjRRtBrHgmVPEqi1vyB5na3+iL2zgTRQ+xdl1by7yBWBB1/d", + "ZaHE+yVKBAEmMp4nVqqnPmIiZf9uAJZ1s5duoJeISIUnEZXzrKRRgEPCAi968fn3Lca+/n3IOVH1Kujf", + "JLTmTrL0W8baFK8F3HC+jzCcRxZPzIWr4NxCzK9QemWzaVgO0D/PK7//C25pM1c3x290yZmReNUu/j5u", + "NzOm+3nDmd1wcoHC1HRX2JXA5j/qtWUmUFDKSjeX9nrjrneXWcGejMy3knlbn9yfJ3fwkX0nkrDbaNg3", + "lYbIJ/09iFxL1TvJ3G/kHLS+pIJX7BuKYDuob6c+cVGQct+FGDYbLpPGRZmjBAabirOfwrgqjG2kzV2F", + "sfO41mJJCuKZsl4S4Sa5bJ2zjQLYOgT+RZM0KrMrCMJvLvjKt2e7D7NrnHgzAi/By4jjH/1eJuBCGNwB", + "e5/7eHAvVfHuMrtg2gCPWzeTEF2X9Pj+9HSzSUoItVJGCPWIJUS5+nYQe4oKv1kQIWjoKhwfnR7bJAsq", + "kUhZH72JKZQdviIkgXpmlKcSAYBEX8/PITDUa7WWoBa6HcKUWCacMrV2FPmr9zOYz3eq8PrActIi//7w", + "l8fghX98QgpkBwRbmAmstiIVVo1xrS7OkzJTlllrW3jCU926liyuHvwMzrYpjYhcSkViE+Q6TSPYRIAN", + "b0sH2u8M8EEXUSWR3g9dSBRPiIiplJQzOWITMtVqWEKE7lt/DjXq83g9r/Ne4UxqnhnR933EgurB2MwN", + "1UQ1QMCBctWdg84WTpKtECvcEG9oh/cFQ3oJwZ1ILuMJj2iAIsquJNqI6JUxOtBCokj/sbkyOnQM333t", + "woh331ma0idsyr21owzPZsz8Y+TKlsWau0R8dGLtFSluFid/YKH9Yk2ulWuC4KinaEwyjBaUKhrRj0bU", + "6UaoVDQw6Z85QsD70xwkYMROiRL6HSwICngUkUA558pWIniwNUoHg50goQCmtUNgcCDwmh/H0OPR2Tt4", + "LyYxF8vuiOl/QMMXh2fmJnaKrY+gMFBG1DUXV+hk682aePlzINO/cJScmeDKVH3vgv+8vrs9AEfjHpIN", + "W5QnqwwgnvzwYZxWg/vpLXic3gJAQMpmszETOAClWM5TFfJr5vcMLHiUxvof5o+TdThaCgfz9/Dqd6Pt", + "muGs7cZN8FFsSjunkJjadt/kgsIQ7LHGl2rCuSmAElOK3POeAofqR+Tur++UL9LxO7yatBR1dSO/m731", + "0CefHYODhyzS47Fsc8NpbiaKr/Y+XWPa7H16HvHgSqKUKRqVsHe03QZw1frHHF7YXvyBmgCJxkibdjxV", + "iNwkVADQWgXFBxE9YwkYGyKmDEdbMGfTCAAlOy8WXnAK+f5BRCHjkoYEJTyKAAzuek4Y0rMBR5VroHBP", + "K22houI7xStGxdGEBDwmDjx602e6/QNT9ZKLMhL09yIXLwr01/PRU9XzXAN+3dzjF4Fhn+IbCGsOU3tN", + "7Ea08YrnPxpXUBfB2ow6OwM56nTRqLMdjzp6BY4wuFCxQnsopixVRPbRsfFvQUb3/gBJEnAWSodh7Tx4", + "OwPZlN9t2LIhWXgfvntItcdyFZDyre3EJx70e0h/Dwk2aKO44eyeDLuw6ULEUwUB3G5f2bdCosA9svng", + "N7CFPfLTtm8jyf9ht29JRsEqa3FZWHoj2TOU47VeN5dUMecyB0dGAU5wQNWyi3AU8SD3HqQyux3oZUOZ", + "CIKvtA3VH7G3Gb6yTYRAR2fvus5phkIqr0wL1i/WR28WRMh0kg0OgTQwHjxYDBKOmOIowFGQAvISmU5J", + "ADkMEY2pkg1+tWwo91mtN+/Es/DuYYau9ricSX6egNXL2UJWOG7LLPWWIEGEaVx0KlWJA6ovXOmC23ei", + "G+X6GJ5G9norEFxKZJvqkYjO6CSylzWyjy60yoFjMmJJhBkjAqXSxB3pofcSQaRMTWKMbgDKoRuO6qIc", + "MygRXFk3ccS5kMazqzn8/SmSiiQr2OytafkU5nxPaPamcdvTNzIYKmNoPpbsK0gviOEUQ3DNR/qY/gbB", + "PmZA3xr1/rFs/AtBZzMi9K7ARsiaq1GzrR05zaYvZXo0lnI5z95qV8ola7UQzV2IdF6J+TJ2L45Bgb7N", + "Dayn8yvaCAtkH90u++I3/VHLvstR/v5B2EdfOMsfpULmeSG4um0BmJzDH1stlsLIS1u1lKCwHo6gdUbC", + "fWYItMYd+GZwA48ZZQCX0g6a4AS+P0YYPGx23ENXg3jcvFVCCSjVf2tIlVqPhPtdcOD9QOB+4+zQO0Dg", + "flf5SgBh+u3yRr+rTKWSH9DVuPrhQW7vK0HJIN0CjEVTgpKRejaQYKWh9N6+085Msi3+SBq8vXu+hf7u", + "yP7T6m9hMhSI5XfZmdxoh9tC4kQt3eUin1YuACX9CMkYPuCHLIbg/vAW7nC9/vXYw/Fp4+X6z7KPD3Z/", + "n9fGPzl+/LUei3uudLBs6VOnh0UwpwvS7HQv72BLokSQXsITuFwJDcEsPdxZprDozz4i27zFqrL/QtSh", + "hZMQhVSQQEVLgyWqJYLp468SCa4tAXjOxdLnTC/u3JeCx4d2NmvOQ7unrDMsv/ONl70QK9xbOGmzwoX2", + "BTft7m5bCzxEGXr1HG2QGyUMeDWaassH0WlGUnITEBJK4MnN4oCHgwbPJv1IxrNJm1GugCF/Y2HeUZBK", + "xWO39ifHaANqAs0I02uhVf0paLKJ4AsamnrZOVEXPDJUHTYQ9LZ+V61UZAWdnHFhBvdNdJg2B9LsI03K", + "YsGELnQOOhPKMAxuLeB3eU+ZhCrdH6aQ1pDvHcc5nZ9HmLX8NpyxozlRGzmOiIpzA423+fOYe8zHXDEw", + "1Z1ppdOuXUXjdrGqLUNI7wMwN4tjfli39fvvJ7ySykcZWWld54vMIG1ym39fLDh4uPPhod3l7x9xOP4r", + "4ozvgqscGtAt+hjmdx7gCIVkQSKeQLFj826n20lF1DnozJVKDra2Iv3enEt18HTwdND5/OHz/x8AAP//", + "cmBIDRaNAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/health_check.go b/lib/providers/health_check.go new file mode 100644 index 00000000..81cc21e7 --- /dev/null +++ b/lib/providers/health_check.go @@ -0,0 +1,133 @@ +package providers + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/healthcheck" + "github.com/kernel/hypeman/lib/instances" +) + +type healthCheckRuntimeManager interface { + GetHealthCheckRuntime(ctx context.Context, id string) (*healthcheck.Runtime, error) + SetHealthCheckRuntime(ctx context.Context, id string, runtime *healthcheck.Runtime) error + SubscribeLifecycleEvents(consumer instances.LifecycleEventConsumer) (<-chan instances.LifecycleEvent, func()) +} + +type healthCheckInstanceStore struct { + manager instances.Manager + runtimeManager healthCheckRuntimeManager +} + +func (s healthCheckInstanceStore) ListInstances(ctx context.Context) ([]healthcheck.Instance, error) { + insts, err := s.manager.ListInstances(ctx, nil) + if err != nil { + return nil, err + } + + out := make([]healthcheck.Instance, 0, len(insts)) + for _, inst := range insts { + runtime, err := s.runtimeManager.GetHealthCheckRuntime(ctx, inst.Id) + if err != nil { + return nil, err + } + out = append(out, toHealthCheckInstance(&inst, runtime)) + } + return out, nil +} + +func (s healthCheckInstanceStore) SetRuntime(ctx context.Context, id string, runtime *healthcheck.Runtime) error { + return s.runtimeManager.SetHealthCheckRuntime(ctx, id, runtime) +} + +func (s healthCheckInstanceStore) SubscribeInstanceEvents() (<-chan healthcheck.InstanceEvent, func(), error) { + src, unsub := s.runtimeManager.SubscribeLifecycleEvents(instances.LifecycleEventConsumerHealthCheck) + dst := make(chan healthcheck.InstanceEvent, 32) + go func() { + defer close(dst) + for event := range src { + var inst *healthcheck.Instance + if event.Instance != nil { + runtime, err := s.runtimeManager.GetHealthCheckRuntime(context.Background(), event.Instance.Id) + if err == nil { + converted := toHealthCheckInstance(event.Instance, runtime) + inst = &converted + } + } + dst <- healthcheck.InstanceEvent{ + Action: healthcheck.InstanceEventAction(event.Action), + InstanceID: event.InstanceID, + Instance: inst, + } + } + }() + return dst, unsub, nil +} + +type healthCheckExecRunner struct { + manager instances.Manager +} + +func (r healthCheckExecRunner) Run(ctx context.Context, inst healthcheck.Instance, check healthcheck.ExecCheck, timeout time.Duration) error { + dialer, err := r.manager.GetVsockDialer(ctx, inst.ID) + if err != nil { + return err + } + + timeoutSeconds := int32((timeout + time.Second - 1) / time.Second) + if timeoutSeconds < 1 { + timeoutSeconds = 1 + } + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: check.Command, + Cwd: check.WorkingDir, + Timeout: timeoutSeconds, + }) + if err != nil { + return err + } + if exit == nil { + return fmt.Errorf("exec health check exited without status") + } + if exit.Code != 0 { + return fmt.Errorf("exec health check exited with status %d", exit.Code) + } + return nil +} + +func toHealthCheckInstance(inst *instances.Instance, runtime *healthcheck.Runtime) healthcheck.Instance { + if inst == nil { + return healthcheck.Instance{} + } + return healthcheck.Instance{ + ID: inst.Id, + Name: inst.Name, + State: string(inst.State), + NetworkEnabled: inst.NetworkEnabled, + IP: inst.IP, + StartedAt: inst.StartedAt, + GuestAgentReady: inst.GuestAgentReadyAt != nil, + SkipGuestAgent: inst.SkipGuestAgent, + HealthCheck: inst.HealthCheck, + Runtime: runtime, + } +} + +func ProvideHealthCheckController(instanceManager instances.Manager, log *slog.Logger) *healthcheck.Controller { + if instanceManager == nil || log == nil { + return nil + } + runtimeManager, ok := instanceManager.(healthCheckRuntimeManager) + if !ok { + return nil + } + + return healthcheck.NewController( + healthCheckInstanceStore{manager: instanceManager, runtimeManager: runtimeManager}, + healthcheck.DefaultProbeRunner{ExecRunner: healthCheckExecRunner{manager: instanceManager}}, + healthcheck.ControllerOptions{Log: log.With("controller", "health_check")}, + ) +} diff --git a/openapi.yaml b/openapi.yaml index 23aea061..3b2ce03c 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -236,6 +236,146 @@ components: maximum: 65535 example: [22, 9000] + HealthCheckHTTP: + type: object + required: [port] + properties: + port: + type: integer + minimum: 1 + maximum: 65535 + description: Port to probe on the instance network address. + example: 8080 + path: + type: string + description: HTTP path to request. + default: "/" + example: "/healthz" + scheme: + type: string + enum: [http, https] + default: http + description: HTTP scheme to use for the probe. + example: http + expected_status: + type: integer + minimum: 100 + maximum: 599 + default: 200 + description: Exact status code required for a successful probe. + example: 200 + + HealthCheckTCP: + type: object + required: [port] + properties: + port: + type: integer + minimum: 1 + maximum: 65535 + description: Port to open on the instance network address. + example: 5432 + + HealthCheckExec: + type: object + required: [command] + properties: + command: + type: array + minItems: 1 + items: + type: string + description: Command and arguments to run inside the guest after guest-agent readiness. + example: ["curl", "-f", "http://localhost:4318/"] + working_dir: + type: string + description: Optional working directory for the command. + example: /app + + HealthCheck: + type: object + description: Workload health check policy. Health is reported separately from instance lifecycle state. + properties: + type: + type: string + enum: [none, http, tcp, exec] + default: none + description: Probe type. Omit health_check or set type=none to disable health checks. + interval: + type: string + description: Delay between checks as a Go duration. + default: "10s" + example: "10s" + timeout: + type: string + description: Per-check timeout as a Go duration. + default: "2s" + example: "2s" + start_period: + type: string + description: Startup grace period before failures can mark the workload unhealthy. + default: "30s" + example: "30s" + failure_threshold: + type: integer + minimum: 1 + default: 3 + description: Consecutive failed checks required to mark the workload unhealthy. + example: 3 + success_threshold: + type: integer + minimum: 1 + default: 1 + description: Consecutive successful checks required to mark the workload healthy. + example: 1 + http: + $ref: "#/components/schemas/HealthCheckHTTP" + tcp: + $ref: "#/components/schemas/HealthCheckTCP" + exec: + $ref: "#/components/schemas/HealthCheckExec" + + InstanceHealthStatus: + type: object + required: [status, consecutive_successes, consecutive_failures] + properties: + status: + type: string + enum: [disabled, starting, healthy, unhealthy, unknown] + description: Current workload health status. + example: healthy + consecutive_successes: + type: integer + description: Consecutive successful checks in the current health window. + example: 4 + consecutive_failures: + type: integer + description: Consecutive failed checks in the current health window. + example: 0 + last_checked_at: + type: string + format: date-time + nullable: true + description: Most recent check completion time. + example: "2026-05-16T01:00:00Z" + last_success_at: + type: string + format: date-time + nullable: true + description: Most recent successful check completion time. + example: "2026-05-16T01:00:00Z" + last_failure_at: + type: string + format: date-time + nullable: true + description: Most recent failed check completion time. + example: "2026-05-16T00:59:50Z" + last_error: + type: string + nullable: true + description: Truncated error from the most recent failed check. + example: "connection refused" + AutoStandbyStatus: type: object required: [supported, configured, enabled, eligible, status, reason, active_inbound_connections, tracking_mode] @@ -317,6 +457,8 @@ components: OUTBOUND_OPENAI_KEY: new-rotated-key-456 auto_standby: $ref: "#/components/schemas/AutoStandbyPolicy" + health_check: + $ref: "#/components/schemas/HealthCheck" CreateInstanceRequest: type: object @@ -425,6 +567,8 @@ components: $ref: "#/components/schemas/SnapshotPolicy" auto_standby: $ref: "#/components/schemas/AutoStandbyPolicy" + health_check: + $ref: "#/components/schemas/HealthCheck" skip_kernel_headers: type: boolean description: | @@ -879,6 +1023,10 @@ components: $ref: "#/components/schemas/SnapshotPolicy" auto_standby: $ref: "#/components/schemas/AutoStandbyPolicy" + health_check: + $ref: "#/components/schemas/HealthCheck" + health_status: + $ref: "#/components/schemas/InstanceHealthStatus" phase_durations_ms: type: object description: |