Re-iterate status evaluation. (#86)

Author: Gerrit91

api/v2/types_firewall.go

@@ -1,8 +1,10 @@
 package v2
 
 import (
+	"fmt"
 	"sort"
 	"strconv"
+	"time"
 
 	"github.com/metal-stack/metal-lib/pkg/pointer"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -185,7 +187,7 @@ const (
 	FirewallDistanceConfigured ConditionType = "Distance"
 	// FirewallProvisioned indicates that all health conditions have been met at least once.
 	// Once set to true, it stays true and is used to detect condition degradation.
-	FirewallHealthy ConditionType = "Healthy"
+	FirewallProvisioned ConditionType = "Provisioned"
 )
 
 // ShootAccess contains secret references to construct a shoot client in the firewall-controller to update its firewall monitor.
@@ -354,3 +356,169 @@ func SortFirewallsByImportance(fws []*Firewall) {
 		return !a.CreationTimestamp.Before(&b.CreationTimestamp)
 	})
 }
+
+type (
+	FirewallStatusResult string
+
+	FirewallStatusEvalResult struct {
+		Result    FirewallStatusResult
+		Reason    string
+		TimeoutIn *time.Duration
+	}
+)
+
+const (
+	FirewallStatusReady         FirewallStatusResult = "ready"
+	FirewallStatusProgressing   FirewallStatusResult = "progressing"
+	FirewallStatusUnhealthy     FirewallStatusResult = "unhealthy"
+	FirewallStatusHealthTimeout FirewallStatusResult = "health-timeout"
+	FirewallStatusCreateTimeout FirewallStatusResult = "create-timeout"
+)
+
+func EvaluateFirewallStatus(fw *Firewall, createTimeout, healthTimeout time.Duration) *FirewallStatusEvalResult {
+	var (
+		checkForTimeout = func(fw *Firewall, condition ConditionType, timeout time.Duration) (time.Duration, bool) {
+			if timeout == 0 {
+				return 0, false
+			}
+
+			var (
+				cond           = pointer.SafeDeref(fw.Status.Conditions.Get(condition))
+				transitionTime = cond.LastTransitionTime.Time
+				deadline       = time.Until(transitionTime.Add(timeout))
+			)
+
+			if deadline < 0 {
+				return 0, true
+			}
+
+			return deadline, false
+		}
+
+		collectUnhealthyConditions = func(cts ...ConditionType) []*Condition {
+			var res []*Condition
+
+			for _, ct := range cts {
+				cond := fw.Status.Conditions.Get(ct)
+				if cond == nil {
+					res = append(res, &Condition{Type: ct})
+				} else if cond.Status != ConditionTrue {
+					res = append(res, cond)
+				}
+			}
+
+			return res
+		}
+
+		unhealthyTypes []string
+		timeoutIn      *time.Duration
+	)
+
+	switch fw.Status.Phase {
+	case FirewallPhaseCreating, FirewallPhaseCrashing:
+		unhealthyConds := collectUnhealthyConditions(
+			FirewallCreated,
+			FirewallReady,
+			FirewallProvisioned,
+		)
+
+		if len(unhealthyConds) == 0 {
+			return &FirewallStatusEvalResult{
+				Result: FirewallStatusReady,
+				Reason: "",
+			}
+		}
+
+		if createTimeout > 0 {
+			if t, ok := checkForTimeout(fw, FirewallReady, createTimeout); ok {
+				return &FirewallStatusEvalResult{
+					Result: FirewallStatusCreateTimeout,
+					Reason: fmt.Sprintf("%s create timeout exceeded, firewall not provisioned in time", createTimeout.String()),
+				}
+			} else if createTimeout != 0 {
+				timeoutIn = &t
+			}
+		}
+
+		for _, c := range unhealthyConds {
+			unhealthyTypes = append(unhealthyTypes, string(c.Type))
+		}
+
+		return &FirewallStatusEvalResult{
+			Result:    FirewallStatusProgressing,
+			Reason:    fmt.Sprintf("not all health conditions are true: %v", unhealthyTypes),
+			TimeoutIn: timeoutIn,
+		}
+
+	case FirewallPhaseRunning:
+		fallthrough
+
+	default:
+		unhealthyConds := collectUnhealthyConditions(
+			FirewallCreated,
+			FirewallReady,
+			FirewallProvisioned,
+			FirewallControllerConnected,
+			FirewallControllerSeedConnected,
+			FirewallDistanceConfigured,
+		)
+
+		if len(unhealthyConds) == 0 {
+			return &FirewallStatusEvalResult{
+				Result: FirewallStatusReady,
+				Reason: "",
+			}
+		}
+
+		var (
+			ready         = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallReady)).Status == ConditionTrue
+			provisioned   = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallProvisioned)).Status == ConditionTrue
+			connected     = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallControllerConnected)).Status == ConditionTrue
+			seedConnected = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallControllerSeedConnected)).Status == ConditionTrue
+		)
+
+		if provisioned {
+			switch {
+			case !seedConnected:
+				if t, ok := checkForTimeout(fw, FirewallControllerSeedConnected, healthTimeout); ok {
+					return &FirewallStatusEvalResult{
+						Result: FirewallStatusHealthTimeout,
+						Reason: fmt.Sprintf("%s health timeout exceeded, seed connection lost", healthTimeout.String()),
+					}
+				} else if healthTimeout != 0 {
+					timeoutIn = &t
+				}
+
+			case !connected:
+				if t, ok := checkForTimeout(fw, FirewallControllerConnected, healthTimeout); ok {
+					return &FirewallStatusEvalResult{
+						Result: FirewallStatusHealthTimeout,
+						Reason: fmt.Sprintf("%s health timeout exceeded, firewall monitor not reconciled anymore", healthTimeout.String()),
+					}
+				} else if healthTimeout != 0 {
+					timeoutIn = &t
+				}
+
+			case !ready:
+				if t, ok := checkForTimeout(fw, FirewallReady, healthTimeout); ok {
+					return &FirewallStatusEvalResult{
+						Result: FirewallStatusHealthTimeout,
+						Reason: fmt.Sprintf("%s health timeout exceeded, firewall is not ready from perspective of the metal-api", healthTimeout.String()),
+					}
+				} else if healthTimeout != 0 {
+					timeoutIn = &t
+				}
+			}
+		}
+
+		for _, c := range unhealthyConds {
+			unhealthyTypes = append(unhealthyTypes, string(c.Type))
+		}
+
+		return &FirewallStatusEvalResult{
+			Result:    FirewallStatusUnhealthy,
+			Reason:    fmt.Sprintf("not all health conditions are true: %v", unhealthyTypes),
+			TimeoutIn: timeoutIn,
+		}
+	}
+}

api/v2/types_firewall_test.go

@@ -6,7 +6,10 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/metal-stack/metal-lib/pkg/pointer"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"testing/synctest"
 )
 
 func Test_SortFirewallsByImportance(t *testing.T) {
@@ -107,3 +110,183 @@ func Test_SortFirewallsByImportance(t *testing.T) {
 		})
 	}
 }
+
+func Test_EvaluateFirewallStatus(t *testing.T) {
+	tests := []struct {
+		name          string
+		modFn         func(fw *Firewall)
+		healthTimeout time.Duration
+		createTimeout time.Duration
+		want          *FirewallStatusEvalResult
+		wantReason    string
+	}{
+		{
+			name:  "ready firewall in running phase",
+			modFn: nil,
+			want: &FirewallStatusEvalResult{
+				Result: FirewallStatusReady,
+			},
+		},
+		{
+			name: "unhealthy firewall in running phase due to firewall monitor not reconciling",
+			modFn: func(fw *Firewall) {
+				fw.Status.Conditions.Set(Condition{
+					Type:   FirewallControllerConnected,
+					Status: ConditionFalse,
+				})
+			},
+			want: &FirewallStatusEvalResult{
+				Result: FirewallStatusUnhealthy,
+				Reason: "not all health conditions are true: [Connected]",
+			},
+		},
+		{
+			name: "unhealthy firewall in running phase due to firewall not reconciling",
+			modFn: func(fw *Firewall) {
+				fw.Status.Conditions.Set(Condition{
+					Type:   FirewallControllerSeedConnected,
+					Status: ConditionFalse,
+				})
+			},
+			want: &FirewallStatusEvalResult{
+				Result: FirewallStatusUnhealthy,
+				Reason: "not all health conditions are true: [SeedConnected]",
+			},
+		},
+		{
+			name: "unhealthy firewall in running phase due to readiness condition false",
+			modFn: func(fw *Firewall) {
+				fw.Status.Conditions.Set(Condition{
+					Type:   FirewallReady,
+					Status: ConditionFalse,
+				})
+			},
+			want: &FirewallStatusEvalResult{
+				Result: FirewallStatusUnhealthy,
+				Reason: "not all health conditions are true: [Ready]",
+			},
+		},
+		{
+			name:          "health timeout reached because seed not connected",
+			healthTimeout: 5 * time.Minute,
+			modFn: func(fw *Firewall) {
+				cond := fw.Status.Conditions.Get(FirewallControllerSeedConnected)
+				cond.Status = ConditionFalse
+				fw.Status.Conditions.Set(*cond)
+			},
+			want: &FirewallStatusEvalResult{
+				Result: FirewallStatusHealthTimeout,
+				Reason: "5m0s health timeout exceeded, seed connection lost",
+			},
+		},
+		{
+			name:          "health timeout not yet reached",
+			healthTimeout: 15 * time.Minute,
+			modFn: func(fw *Firewall) {
+				cond := fw.Status.Conditions.Get(FirewallControllerSeedConnected)
+				cond.Status = ConditionFalse
+				fw.Status.Conditions.Set(*cond)
+			},
+			want: &FirewallStatusEvalResult{
+				Result:    FirewallStatusUnhealthy,
+				Reason:    "not all health conditions are true: [SeedConnected]",
+				TimeoutIn: pointer.Pointer(5 * time.Minute),
+			},
+		},
+		{
+			name:          "create timeout reached because not provisioned",
+			createTimeout: 5 * time.Minute,
+			modFn: func(fw *Firewall) {
+				fw.Status.Phase = FirewallPhaseCreating
+				cond := fw.Status.Conditions.Get(FirewallProvisioned)
+				cond.Status = ConditionFalse
+				fw.Status.Conditions.Set(*cond)
+			},
+			want: &FirewallStatusEvalResult{
+				Result: FirewallStatusCreateTimeout,
+				Reason: "5m0s create timeout exceeded, firewall not provisioned in time",
+			},
+		},
+		{
+			name:          "create timeout not yet reached",
+			createTimeout: 15 * time.Minute,
+			modFn: func(fw *Firewall) {
+				fw.Status.Phase = FirewallPhaseCreating
+				cond := fw.Status.Conditions.Get(FirewallProvisioned)
+				cond.Status = ConditionFalse
+				fw.Status.Conditions.Set(*cond)
+			},
+			want: &FirewallStatusEvalResult{
+				Result:    FirewallStatusProgressing,
+				Reason:    "not all health conditions are true: [Provisioned]",
+				TimeoutIn: pointer.Pointer(5 * time.Minute),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			synctest.Test(t, func(t *testing.T) {
+				tenMinutesAgo := time.Now().Add(-10 * time.Minute)
+
+				fw := &Firewall{
+					Status: FirewallStatus{
+						Phase: FirewallPhaseRunning,
+						Conditions: Conditions{
+							{
+								Type:               FirewallControllerConnected,
+								Status:             ConditionTrue,
+								LastTransitionTime: metav1.NewTime(tenMinutesAgo),
+								LastUpdateTime:     metav1.NewTime(tenMinutesAgo),
+							},
+							{
+								Type:               FirewallControllerSeedConnected,
+								Status:             ConditionTrue,
+								LastTransitionTime: metav1.NewTime(tenMinutesAgo),
+								LastUpdateTime:     metav1.NewTime(tenMinutesAgo),
+							},
+							{
+								Type:               FirewallCreated,
+								Status:             ConditionTrue,
+								LastTransitionTime: metav1.NewTime(tenMinutesAgo),
+								LastUpdateTime:     metav1.NewTime(tenMinutesAgo),
+							},
+							{
+								Type:               FirewallReady,
+								Status:             ConditionTrue,
+								LastTransitionTime: metav1.NewTime(tenMinutesAgo),
+								LastUpdateTime:     metav1.NewTime(tenMinutesAgo),
+							},
+							{
+								Type:               FirewallProvisioned,
+								Status:             ConditionTrue,
+								LastTransitionTime: metav1.NewTime(tenMinutesAgo),
+								LastUpdateTime:     metav1.NewTime(tenMinutesAgo),
+							},
+							{
+								Type:               FirewallDistanceConfigured,
+								Status:             ConditionTrue,
+								LastTransitionTime: metav1.NewTime(tenMinutesAgo),
+								LastUpdateTime:     metav1.NewTime(tenMinutesAgo),
+							},
+							{
+								Type:               FirewallMonitorDeployed,
+								Status:             ConditionTrue,
+								LastTransitionTime: metav1.NewTime(tenMinutesAgo),
+								LastUpdateTime:     metav1.NewTime(tenMinutesAgo),
+							},
+						},
+					},
+				}
+
+				if tt.modFn != nil {
+					tt.modFn(fw)
+				}
+
+				got := EvaluateFirewallStatus(fw, tt.createTimeout, tt.healthTimeout)
+				if diff := cmp.Diff(tt.want, got); diff != "" {
+					t.Errorf("diff = %s", diff)
+				}
+			})
+		})
+	}
+}