Add instance health checks

cmd/api/api/health_check.go

@@ -0,0 +1,138 @@
+package api
+
+import (
+	"fmt"
+
+	"github.com/kernel/hypeman/lib/healthcheck"
+	"github.com/kernel/hypeman/lib/oapi"
+	"github.com/samber/lo"
+)
+
+func toDomainHealthCheck(policy *oapi.HealthCheck) (*healthcheck.Policy, error) {
+	if policy == nil {
+		return nil, nil
+	}
+
+	out := &healthcheck.Policy{}
+	if policy.Type != nil {
+		out.Type = healthcheck.Type(*policy.Type)
+	}
+	if policy.Interval != nil {
+		out.Interval = *policy.Interval
+	}
+	if policy.Timeout != nil {
+		out.Timeout = *policy.Timeout
+	}
+	if policy.StartPeriod != nil {
+		out.StartPeriod = *policy.StartPeriod
+	}
+	if policy.FailureThreshold != nil {
+		out.FailureThreshold = *policy.FailureThreshold
+	}
+	if policy.SuccessThreshold != nil {
+		out.SuccessThreshold = *policy.SuccessThreshold
+	}
+	if policy.Http != nil {
+		if policy.Http.Port < 1 || policy.Http.Port > 65535 {
+			return nil, fmt.Errorf("health_check.http.port must be between 1 and 65535")
+		}
+		out.HTTP = &healthcheck.HTTPCheck{
+			Port: uint16(policy.Http.Port),
+		}
+		if policy.Http.Path != nil {
+			out.HTTP.Path = *policy.Http.Path
+		}
+		if policy.Http.Scheme != nil {
+			out.HTTP.Scheme = string(*policy.Http.Scheme)
+		}
+		if policy.Http.ExpectedStatus != nil {
+			out.HTTP.ExpectedStatus = *policy.Http.ExpectedStatus
+		}
+	}
+	if policy.Tcp != nil {
+		if policy.Tcp.Port < 1 || policy.Tcp.Port > 65535 {
+			return nil, fmt.Errorf("health_check.tcp.port must be between 1 and 65535")
+		}
+		out.TCP = &healthcheck.TCPCheck{Port: uint16(policy.Tcp.Port)}
+	}
+	if policy.Exec != nil {
+		out.Exec = &healthcheck.ExecCheck{
+			Command: append([]string(nil), policy.Exec.Command...),
+		}
+		if policy.Exec.WorkingDir != nil {
+			out.Exec.WorkingDir = *policy.Exec.WorkingDir
+		}
+	}
+
+	return healthcheck.NormalizePolicy(out)
+}
+
+func toOAPIHealthCheck(policy *healthcheck.Policy) *oapi.HealthCheck {
+	if policy == nil {
+		return nil
+	}
+
+	typ := oapi.HealthCheckType(policy.Type)
+	out := &oapi.HealthCheck{
+		Type: &typ,
+	}
+	if policy.Interval != "" {
+		out.Interval = lo.ToPtr(policy.Interval)
+	}
+	if policy.Timeout != "" {
+		out.Timeout = lo.ToPtr(policy.Timeout)
+	}
+	if policy.StartPeriod != "" {
+		out.StartPeriod = lo.ToPtr(policy.StartPeriod)
+	}
+	if policy.FailureThreshold != 0 {
+		out.FailureThreshold = lo.ToPtr(policy.FailureThreshold)
+	}
+	if policy.SuccessThreshold != 0 {
+		out.SuccessThreshold = lo.ToPtr(policy.SuccessThreshold)
+	}
+	if policy.HTTP != nil {
+		out.Http = &oapi.HealthCheckHTTP{
+			Port: int(policy.HTTP.Port),
+		}
+		if policy.HTTP.Path != "" {
+			out.Http.Path = lo.ToPtr(policy.HTTP.Path)
+		}
+		if policy.HTTP.Scheme != "" {
+			scheme := oapi.HealthCheckHTTPScheme(policy.HTTP.Scheme)
+			out.Http.Scheme = &scheme
+		}
+		if policy.HTTP.ExpectedStatus != 0 {
+			out.Http.ExpectedStatus = lo.ToPtr(policy.HTTP.ExpectedStatus)
+		}
+	}
+	if policy.TCP != nil {
+		out.Tcp = &oapi.HealthCheckTCP{
+			Port: int(policy.TCP.Port),
+		}
+	}
+	if policy.Exec != nil {
+		out.Exec = &oapi.HealthCheckExec{
+			Command: append([]string(nil), policy.Exec.Command...),
+		}
+		if policy.Exec.WorkingDir != "" {
+			out.Exec.WorkingDir = lo.ToPtr(policy.Exec.WorkingDir)
+		}
+	}
+	return out
+}
+
+func toOAPIHealthStatus(snapshot healthcheck.StatusSnapshot) *oapi.InstanceHealthStatus {
+	out := &oapi.InstanceHealthStatus{
+		Status:               oapi.InstanceHealthStatusStatus(snapshot.Status),
+		ConsecutiveSuccesses: snapshot.ConsecutiveSuccesses,
+		ConsecutiveFailures:  snapshot.ConsecutiveFailures,
+		LastCheckedAt:        snapshot.LastCheckedAt,
+		LastSuccessAt:        snapshot.LastSuccessAt,
+		LastFailureAt:        snapshot.LastFailureAt,
+	}
+	if snapshot.LastError != "" {
+		out.LastError = lo.ToPtr(snapshot.LastError)
+	}
+	return out
+}

cmd/api/api/instances.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/c2h5oh/datasize"
 	"github.com/kernel/hypeman/lib/guest"
+	"github.com/kernel/hypeman/lib/healthcheck"
 	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/instances"
 	"github.com/kernel/hypeman/lib/logger"
@@ -285,6 +286,13 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 			Message: err.Error(),
 		}, nil
 	}
+	healthCheck, err := toDomainHealthCheck(request.Body.HealthCheck)
+	if err != nil {
+		return oapi.CreateInstance400JSONResponse{
+			Code:    "invalid_health_check",
+			Message: err.Error(),
+		}, nil
+	}
 
 	domainReq := instances.CreateInstanceRequest{
 		Name:                     request.Body.Name,
@@ -310,6 +318,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		SkipKernelHeaders:        request.Body.SkipKernelHeaders != nil && *request.Body.SkipKernelHeaders,
 		SkipGuestAgent:           request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent,
 		AutoStandby:              autoStandby,
+		HealthCheck:              healthCheck,
 	}
 	if request.Body.SnapshotPolicy != nil {
 		snapshotPolicy, err := toInstanceSnapshotPolicy(*request.Body.SnapshotPolicy)
@@ -1028,10 +1037,18 @@ func (s *ApiService) UpdateInstance(ctx context.Context, request oapi.UpdateInst
 			Message: err.Error(),
 		}, nil
 	}
+	healthCheck, err := toDomainHealthCheck(request.Body.HealthCheck)
+	if err != nil {
+		return oapi.UpdateInstance400JSONResponse{
+			Code:    "invalid_health_check",
+			Message: err.Error(),
+		}, nil
+	}
 
 	result, err := s.InstanceManager.UpdateInstance(ctx, inst.Id, instances.UpdateInstanceRequest{
 		Env:         env,
 		AutoStandby: autoStandby,
+		HealthCheck: healthCheck,
 	})
 	if err != nil {
 		switch {
@@ -1163,6 +1180,8 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance {
 		oapiInst.SnapshotPolicy = &oapiPolicy
 	}
 	oapiInst.AutoStandby = toOAPIAutoStandbyPolicy(inst.AutoStandby)
+	oapiInst.HealthCheck = toOAPIHealthCheck(inst.HealthCheck)
+	oapiInst.HealthStatus = toOAPIHealthStatus(healthcheck.Snapshot(inst.HealthCheck, string(inst.State), inst.HealthCheckRuntime))
 
 	// Convert volume attachments
 	if len(inst.Volumes) > 0 {

cmd/api/api/instances_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/c2h5oh/datasize"
 	"github.com/kernel/hypeman/lib/autostandby"
+	"github.com/kernel/hypeman/lib/healthcheck"
 	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/instances"
 	"github.com/kernel/hypeman/lib/instances/phasetracking"
@@ -279,6 +280,7 @@ func (m *captureUpdateManager) UpdateInstance(ctx context.Context, id string, re
 			Image:          "docker.io/library/alpine:latest",
 			Env:            req.Env,
 			AutoStandby:    req.AutoStandby,
+			HealthCheck:    req.HealthCheck,
 			CreatedAt:      now,
 			HypervisorType: hypervisor.TypeCloudHypervisor,
 		},
@@ -301,6 +303,7 @@ func (m *captureCreateManager) CreateInstance(ctx context.Context, req instances
 			OverlaySize:    req.OverlaySize,
 			Vcpus:          req.Vcpus,
 			AutoStandby:    req.AutoStandby,
+			HealthCheck:    req.HealthCheck,
 			CreatedAt:      now,
 			HypervisorType: hypervisor.TypeCloudHypervisor,
 		},
@@ -657,6 +660,51 @@ func TestCreateInstance_MapsAutoStandbyPolicy(t *testing.T) {
 	assert.Equal(t, idleTimeout, *instance.AutoStandby.IdleTimeout)
 }
 
+func TestCreateInstance_MapsHealthCheckPolicy(t *testing.T) {
+	t.Parallel()
+
+	svc := newTestService(t)
+	origMgr := svc.InstanceManager
+	mockMgr := &captureCreateManager{Manager: origMgr}
+	svc.InstanceManager = mockMgr
+
+	typ := oapi.HealthCheckTypeExec
+	interval := "5s"
+	timeout := "1s"
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
+		Body: &oapi.CreateInstanceRequest{
+			Name:  "test-health-check",
+			Image: "docker.io/library/alpine:latest",
+			HealthCheck: &oapi.HealthCheck{
+				Type:     &typ,
+				Interval: &interval,
+				Timeout:  &timeout,
+				Exec: &oapi.HealthCheckExec{
+					Command: []string{"true"},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	created, ok := resp.(oapi.CreateInstance201JSONResponse)
+	require.True(t, ok, "expected 201 response")
+	require.NotNil(t, mockMgr.lastReq)
+	require.NotNil(t, mockMgr.lastReq.HealthCheck)
+	assert.Equal(t, healthcheck.TypeExec, mockMgr.lastReq.HealthCheck.Type)
+	assert.Equal(t, []string{"true"}, mockMgr.lastReq.HealthCheck.Exec.Command)
+	assert.Equal(t, "5s", mockMgr.lastReq.HealthCheck.Interval)
+	assert.Equal(t, "1s", mockMgr.lastReq.HealthCheck.Timeout)
+
+	instance := oapi.Instance(created)
+	require.NotNil(t, instance.HealthCheck)
+	require.NotNil(t, instance.HealthCheck.Type)
+	assert.Equal(t, oapi.HealthCheckTypeExec, *instance.HealthCheck.Type)
+	require.NotNil(t, instance.HealthStatus)
+	assert.Equal(t, oapi.InstanceHealthStatusStatusStarting, instance.HealthStatus.Status)
+}
+
 func TestUpdateInstance_MapsEnvPatch(t *testing.T) {
 	t.Parallel()
 	svc := newTestService(t)
@@ -770,6 +818,71 @@ func TestUpdateInstance_MapsAutoStandbyPatch(t *testing.T) {
 	assert.True(t, *instance.AutoStandby.Enabled)
 }
 
+func TestUpdateInstance_MapsHealthCheckPatch(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+
+	origMgr := svc.InstanceManager
+	now := time.Now()
+	mockMgr := &captureUpdateManager{
+		Manager: origMgr,
+		result: &instances.Instance{
+			StoredMetadata: instances.StoredMetadata{
+				Id:             "inst-update-health-check",
+				Name:           "inst-update-health-check",
+				Image:          "docker.io/library/alpine:latest",
+				CreatedAt:      now,
+				HypervisorType: hypervisor.TypeCloudHypervisor,
+				HealthCheck: &healthcheck.Policy{
+					Type: healthcheck.TypeTCP,
+					TCP:  &healthcheck.TCPCheck{Port: 8080},
+				},
+			},
+			State: instances.StateStopped,
+		},
+	}
+	svc.InstanceManager = mockMgr
+
+	typ := oapi.HealthCheckTypeTcp
+	resolved := &instances.Instance{
+		StoredMetadata: instances.StoredMetadata{
+			Id:             "inst-update-health-check",
+			Name:           "inst-update-health-check",
+			Image:          "docker.io/library/alpine:latest",
+			CreatedAt:      now,
+			HypervisorType: hypervisor.TypeCloudHypervisor,
+			NetworkEnabled: true,
+		},
+		State: instances.StateStopped,
+	}
+
+	resp, err := svc.UpdateInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.UpdateInstanceRequestObject{
+		Id: resolved.Id,
+		Body: &oapi.UpdateInstanceRequest{
+			HealthCheck: &oapi.HealthCheck{
+				Type: &typ,
+				Tcp:  &oapi.HealthCheckTCP{Port: 8080},
+			},
+		},
+	})
+	require.NoError(t, err)
+	updated, ok := resp.(oapi.UpdateInstance200JSONResponse)
+	require.True(t, ok, "expected 200 response")
+
+	require.NotNil(t, mockMgr.lastReq)
+	require.NotNil(t, mockMgr.lastReq.HealthCheck)
+	assert.Equal(t, resolved.Id, mockMgr.lastID)
+	assert.Equal(t, healthcheck.TypeTCP, mockMgr.lastReq.HealthCheck.Type)
+	assert.Equal(t, uint16(8080), mockMgr.lastReq.HealthCheck.TCP.Port)
+
+	instance := oapi.Instance(updated)
+	require.NotNil(t, instance.HealthCheck)
+	require.NotNil(t, instance.HealthCheck.Type)
+	assert.Equal(t, oapi.HealthCheckTypeTcp, *instance.HealthCheck.Type)
+	require.NotNil(t, instance.HealthStatus)
+	assert.Equal(t, oapi.InstanceHealthStatusStatusUnknown, instance.HealthStatus.Status)
+}
+
 func TestUpdateInstance_RejectsZeroAutoStandbyIgnoreDestinationPort(t *testing.T) {
 	t.Parallel()
 	svc := newTestService(t)

cmd/api/main.go

@@ -565,6 +565,12 @@ func run() error {
 			return app.AutoStandbyController.Run(gctx)
 		})
 	}
+	if app.HealthCheckController != nil {
+		grp.Go(func() error {
+			logger.Info("starting health check controller")
+			return app.HealthCheckController.Run(gctx)
+		})
+	}
 
 	// Run the server
 	grp.Go(func() error {

cmd/api/wire.go

@@ -13,6 +13,7 @@ import (
 	"github.com/kernel/hypeman/lib/builds"
 	"github.com/kernel/hypeman/lib/devices"
 	"github.com/kernel/hypeman/lib/guestmemory"
+	"github.com/kernel/hypeman/lib/healthcheck"
 	"github.com/kernel/hypeman/lib/images"
 	"github.com/kernel/hypeman/lib/ingress"
 	"github.com/kernel/hypeman/lib/instances"
@@ -41,6 +42,7 @@ type application struct {
 	ResourceManager       *resources.Manager
 	GuestMemoryController guestmemory.Controller
 	AutoStandbyController *autostandby.Controller
+	HealthCheckController *healthcheck.Controller
 	VMMetricsManager      *vm_metrics.Manager
 	Registry              *registry.Registry
 	ApiService            *api.ApiService
@@ -64,6 +66,7 @@ func initializeApp() (*application, func(), error) {
 		providers.ProvideResourceManager,
 		providers.ProvideGuestMemoryController,
 		providers.ProvideAutoStandbyController,
+		providers.ProvideHealthCheckController,
 		providers.ProvideVMMetricsManager,
 		providers.ProvideRegistry,
 		api.New,