Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions pkg/apps/deployment/progressing.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package deployment

import (
"fmt"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/utils/ptr"

operatorv1 "github.com/openshift/api/operator/v1"
)

// DeploymentProgressingCondition computes an operator Progressing condition from
// the deployment's own status conditions and replica counts.
func DeploymentProgressingCondition(deployment *appsv1.Deployment) operatorv1.OperatorCondition {
desiredReplicas := ptr.Deref(deployment.Spec.Replicas, 1)
timedOutMessage, timedOut := HasDeploymentTimedOutProgressing(deployment.Status)

switch {
case !HasDeploymentProgressed(deployment.Status) && !timedOut:
return operatorv1.OperatorCondition{
Type: operatorv1.OperatorStatusTypeProgressing,
Status: operatorv1.ConditionTrue,
Reason: "PodsUpdating",
Message: fmt.Sprintf("deployment/%s.%s: %d/%d pods have been updated to the latest revision and %d/%d pods are available", deployment.Name, deployment.Namespace, deployment.Status.UpdatedReplicas, desiredReplicas, deployment.Status.AvailableReplicas, desiredReplicas),
}

case timedOut:
return operatorv1.OperatorCondition{
Type: operatorv1.OperatorStatusTypeProgressing,
Status: operatorv1.ConditionFalse,
Reason: "ProgressDeadlineExceeded",
Message: fmt.Sprintf("deployment/%s.%s has timed out progressing: %s", deployment.Name, deployment.Namespace, timedOutMessage),
}

default:
return operatorv1.OperatorCondition{
Type: operatorv1.OperatorStatusTypeProgressing,
Status: operatorv1.ConditionFalse,
Reason: "AsExpected",
}
}
}

// HasDeploymentProgressed returns true if the deployment reports NewReplicaSetAvailable
// via the DeploymentProgressing condition.
func HasDeploymentProgressed(status appsv1.DeploymentStatus) bool {
for _, cond := range status.Conditions {
if cond.Type == appsv1.DeploymentProgressing {
return cond.Status == corev1.ConditionTrue && cond.Reason == "NewReplicaSetAvailable"
}
}
return false
}

// HasDeploymentTimedOutProgressing returns true if the deployment reports ProgressDeadlineExceeded.
// The function returns the Progressing condition message as the first return value.
func HasDeploymentTimedOutProgressing(status appsv1.DeploymentStatus) (string, bool) {
for _, cond := range status.Conditions {
if cond.Type == appsv1.DeploymentProgressing {
return cond.Message, cond.Status == corev1.ConditionFalse && cond.Reason == "ProgressDeadlineExceeded"
}
}
return "", false
}
225 changes: 225 additions & 0 deletions pkg/apps/deployment/progressing_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,225 @@
package deployment

import (
"testing"

"github.com/google/go-cmp/cmp"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"

operatorv1 "github.com/openshift/api/operator/v1"
)

func TestDeploymentProgressingCondition(t *testing.T) {
tests := []struct {
name string
deploy *appsv1.Deployment
expected operatorv1.OperatorCondition
}{
{
name: "active rollout, not all pods updated",
deploy: &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "ns"},
Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)},
Status: appsv1.DeploymentStatus{
UpdatedReplicas: 1,
AvailableReplicas: 2,
Conditions: []appsv1.DeploymentCondition{
{Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, Reason: "ReplicaSetUpdated"},
},
},
},
expected: operatorv1.OperatorCondition{
Type: operatorv1.OperatorStatusTypeProgressing,
Status: operatorv1.ConditionTrue,
Reason: "PodsUpdating",
Message: "deployment/web.ns: 1/3 pods have been updated to the latest revision and 2/3 pods are available",
},
},
{
name: "rollout complete",
deploy: &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "ns"},
Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)},
Status: appsv1.DeploymentStatus{
UpdatedReplicas: 3,
AvailableReplicas: 3,
Conditions: []appsv1.DeploymentCondition{
{Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, Reason: "NewReplicaSetAvailable"},
},
},
},
expected: operatorv1.OperatorCondition{
Type: operatorv1.OperatorStatusTypeProgressing,
Status: operatorv1.ConditionFalse,
Reason: "AsExpected",
},
},
{
name: "progress deadline exceeded",
deploy: &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "ns"},
Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](3)},
Status: appsv1.DeploymentStatus{
UpdatedReplicas: 1,
AvailableReplicas: 2,
Conditions: []appsv1.DeploymentCondition{
{Type: appsv1.DeploymentProgressing, Status: corev1.ConditionFalse, Reason: "ProgressDeadlineExceeded", Message: "timed out waiting"},
},
},
},
expected: operatorv1.OperatorCondition{
Type: operatorv1.OperatorStatusTypeProgressing,
Status: operatorv1.ConditionFalse,
Reason: "ProgressDeadlineExceeded",
Message: "deployment/web.ns has timed out progressing: timed out waiting",
},
},
{
name: "no progressing condition on deployment",
deploy: &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "ns"},
Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](1)},
Status: appsv1.DeploymentStatus{},
},
expected: operatorv1.OperatorCondition{
Type: operatorv1.OperatorStatusTypeProgressing,
Status: operatorv1.ConditionTrue,
Reason: "PodsUpdating",
Message: "deployment/web.ns: 0/1 pods have been updated to the latest revision and 0/1 pods are available",
},
},
{
name: "nil replicas defaults to 1",
deploy: &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "ns"},
Spec: appsv1.DeploymentSpec{},
Status: appsv1.DeploymentStatus{
UpdatedReplicas: 1,
AvailableReplicas: 1,
Conditions: []appsv1.DeploymentCondition{
{Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, Reason: "NewReplicaSetAvailable"},
},
},
},
expected: operatorv1.OperatorCondition{
Type: operatorv1.OperatorStatusTypeProgressing,
Status: operatorv1.ConditionFalse,
Reason: "AsExpected",
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := DeploymentProgressingCondition(tt.deploy)
if d := cmp.Diff(tt.expected, got); d != "" {
t.Errorf("unexpected condition (-want +got):\n%s", d)
}
})
}
}

func TestHasDeploymentProgressed(t *testing.T) {
tests := []struct {
name string
conditions []appsv1.DeploymentCondition
want bool
}{
{
name: "NewReplicaSetAvailable and True",
conditions: []appsv1.DeploymentCondition{
{Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, Reason: "NewReplicaSetAvailable"},
},
want: true,
},
{
name: "ReplicaSetUpdated and True",
conditions: []appsv1.DeploymentCondition{
{Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, Reason: "ReplicaSetUpdated"},
},
want: false,
},
{
name: "NewReplicaSetAvailable but False",
conditions: []appsv1.DeploymentCondition{
{Type: appsv1.DeploymentProgressing, Status: corev1.ConditionFalse, Reason: "NewReplicaSetAvailable"},
},
want: false,
},
{
name: "no conditions",
conditions: nil,
want: false,
},
{
name: "unrelated condition only",
conditions: []appsv1.DeploymentCondition{
{Type: appsv1.DeploymentAvailable, Status: corev1.ConditionTrue},
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := HasDeploymentProgressed(appsv1.DeploymentStatus{Conditions: tt.conditions})
if got != tt.want {
t.Errorf("got %v, want %v", got, tt.want)
}
})
}
}

func TestHasDeploymentTimedOutProgressing(t *testing.T) {
tests := []struct {
name string
conditions []appsv1.DeploymentCondition
wantMessage string
wantTimedOut bool
}{
{
name: "ProgressDeadlineExceeded",
conditions: []appsv1.DeploymentCondition{
{Type: appsv1.DeploymentProgressing, Status: corev1.ConditionFalse, Reason: "ProgressDeadlineExceeded", Message: "timed out"},
},
wantMessage: "timed out",
wantTimedOut: true,
},
{
name: "ProgressDeadlineExceeded but True status",
conditions: []appsv1.DeploymentCondition{
{Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, Reason: "ProgressDeadlineExceeded", Message: "timed out"},
},
wantMessage: "timed out",
wantTimedOut: false,
},
{
name: "normal progressing",
conditions: []appsv1.DeploymentCondition{
{Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, Reason: "ReplicaSetUpdated", Message: "progressing"},
},
wantMessage: "progressing",
wantTimedOut: false,
},
{
name: "no conditions",
conditions: nil,
wantMessage: "",
wantTimedOut: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotMsg, gotTimedOut := HasDeploymentTimedOutProgressing(appsv1.DeploymentStatus{Conditions: tt.conditions})
if gotMsg != tt.wantMessage {
t.Errorf("message: got %q, want %q", gotMsg, tt.wantMessage)
}
if gotTimedOut != tt.wantTimedOut {
t.Errorf("timedOut: got %v, want %v", gotTimedOut, tt.wantTimedOut)
}
})
}
}
50 changes: 11 additions & 39 deletions pkg/operator/apiserver/controller/workload/workload.go
Original file line number Diff line number Diff line change
Expand Up @@ -301,34 +301,16 @@ func (c *Controller) updateOperatorStatus(ctx context.Context, previousStatus *o

desiredReplicas := ptr.Deref(workload.Spec.Replicas, 1)

// If the workload is up to date, then we are no longer progressing
workloadAtHighestGeneration := workload.ObjectMeta.Generation == workload.Status.ObservedGeneration
// Update is done when all pods have been updated to the latest revision
// and the deployment controller has reported NewReplicaSetAvailable
workloadIsBeingUpdated := !workloadAtHighestGeneration || !hasDeploymentProgressed(workload.Status)
workloadIsBeingUpdatedTooLong := v1helpers.IsUpdatingTooLong(previousStatus, *deploymentProgressingCondition.Type)
if !workloadAtHighestGeneration {
deploymentProgressingCondition = deploymentProgressingCondition.
WithStatus(operatorv1.ConditionTrue).
WithReason("NewGeneration").
WithMessage(fmt.Sprintf("deployment/%s.%s: observed generation is %d, desired generation is %d.", workload.Name, c.targetNamespace, workload.Status.ObservedGeneration, workload.ObjectMeta.Generation))
} else if workloadIsBeingUpdated {
deploymentProgressingCondition = deploymentProgressingCondition.
WithStatus(operatorv1.ConditionTrue).
WithReason("PodsUpdating").
WithMessage(fmt.Sprintf("deployment/%s.%s: %d/%d pods have been updated to the latest generation and %d/%d pods are available", workload.Name, c.targetNamespace, workload.Status.UpdatedReplicas, desiredReplicas, workload.Status.AvailableReplicas, desiredReplicas))
} else {
// Terminating pods don't account for any of the other status fields but
// still can exist in a state when they are accepting connections and would
// contribute to unexpected behavior when we report Progressing=False.
// The case of too many pods might occur for example if `TerminationGracePeriodSeconds` is set
//
// The workload should ensure this does not happen by using for example EnsureAtMostOnePodPerNode
// so that the old pods terminate before the new ones are started.
deploymentProgressingCondition = deploymentProgressingCondition.
WithStatus(operatorv1.ConditionFalse).
WithReason("AsExpected")
}
// Update is done when the deployment controller has reported NewReplicaSetAvailable.
// Checking the current vs. observed generation here is not possible since we don't want to be Progressing on scaling.
progressingCond := deployment.DeploymentProgressingCondition(workload)
_, workloadIsBeingUpdatedTooLong := deployment.HasDeploymentTimedOutProgressing(workload.Status)
workloadIsBeingUpdated := !deployment.HasDeploymentProgressed(workload.Status) && !workloadIsBeingUpdatedTooLong
Comment thread
coderabbitai[bot] marked this conversation as resolved.

deploymentProgressingCondition = deploymentProgressingCondition.
WithStatus(progressingCond.Status).
WithReason(progressingCond.Reason).
WithMessage(progressingCond.Message)

// During a rollout the default maxSurge (25%) will allow the available
// replicas to temporarily exceed the desired replica count. If this were
Expand All @@ -354,6 +336,7 @@ func (c *Controller) updateOperatorStatus(ctx context.Context, previousStatus *o
// if the deployment is all available and at the expected generation, then update the version to the latest
// when we update, the image pull spec should immediately be different, which should immediately cause a deployment rollout
// which should immediately result in a deployment generation diff, which should cause this block to be skipped until it is ready.
workloadAtHighestGeneration := workload.ObjectMeta.Generation == workload.Status.ObservedGeneration
workloadHasAllPodsUpdated := workload.Status.UpdatedReplicas == desiredReplicas
if workloadAtHighestGeneration && workloadHasAllPodsAvailable && workloadHasAllPodsUpdated && operatorConfigAtHighestGeneration {
c.versionRecorder.SetVersion(c.constructOperandNameFor(workload.Name), c.targetOperandVersion)
Expand All @@ -373,17 +356,6 @@ func (c *Controller) constructOperandNameFor(name string) string {
return name
}

// hasDeploymentProgressed returns true if the deployment reports NewReplicaSetAvailable
// via the DeploymentProgressing condition
func hasDeploymentProgressed(status appsv1.DeploymentStatus) bool {
for _, cond := range status.Conditions {
if cond.Type == appsv1.DeploymentProgressing {
return cond.Status == corev1.ConditionTrue && cond.Reason == "NewReplicaSetAvailable"
}
}
return false
}

// EnsureAtMostOnePodPerNode updates the deployment spec to prevent more than
// one pod of a given replicaset from landing on a node. It accomplishes this
// by adding a label on the template and updates the pod anti-affinity term to include that label.
Expand Down
Loading