Skip to content

Commit f32caad

Browse files
camilamacedo86camilamacedo86
committed
chore(ci): Add test to check if user changes are preserved
Add a test to ensure that OLM is not reverting user changes like kubectl rollout restart. Assisted-by: Cursor/Claude
1 parent 147cfa8 commit f32caad

2 files changed

Lines changed: 357 additions & 0 deletions

File tree

‎test/e2e/features/user-managed-fields.feature‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
@BoxcutterRuntime
2+
Feature: Preserve user-managed fields on deployed resources
3+
OLM uses Server-Side Apply with specific field ownership. Fields that OLM does
4+
not declare ownership of (e.g. user-applied annotations and labels) belong to
5+
other managers and must be preserved across reconciliation cycles.
6+
Related: https://github.com/operator-framework/operator-lifecycle-manager/issues/3392
7+
8+
Background:
9+
Given OLM is available
10+
And ClusterCatalog "test" serves bundles
11+
And ServiceAccount "olm-sa" with needed permissions is available in ${TEST_NAMESPACE}
12+
13+
Scenario: User-added annotations and labels coexist with bundle-defined labels after reconciliation
14+
When ClusterExtension is applied
15+
"""
16+
apiVersion: olm.operatorframework.io/v1
17+
kind: ClusterExtension
18+
metadata:
19+
name: ${NAME}
20+
spec:
21+
namespace: ${TEST_NAMESPACE}
22+
serviceAccount:
23+
name: olm-sa
24+
source:
25+
sourceType: Catalog
26+
catalog:
27+
packageName: test
28+
selector:
29+
matchLabels:
30+
"olm.operatorframework.io/metadata.name": test-catalog
31+
"""
32+
Then ClusterExtension is rolled out
33+
And ClusterExtension is available
34+
And resource "deployment/test-operator" is available
35+
# The bundle defines labels on the deployment via the CSV spec; verify they are present
36+
And resource "deployment/test-operator" has label "app.kubernetes.io/name" with value "test-operator"
37+
When user adds annotation "example.com/custom-annotation=my-value" to "deployment/test-operator"
38+
And user adds label "example.com/custom-label=my-value" to "deployment/test-operator"
39+
Then resource "deployment/test-operator" has annotation "example.com/custom-annotation" with value "my-value"
40+
And resource "deployment/test-operator" has label "example.com/custom-label" with value "my-value"
41+
When ClusterExtension reconciliation is triggered
42+
And ClusterExtension has been reconciled the latest generation
43+
Then resource "deployment/test-operator" has annotation "example.com/custom-annotation" with value "my-value"
44+
And resource "deployment/test-operator" has label "example.com/custom-label" with value "my-value"
45+
# Bundle-defined labels must still be intact after reconciliation
46+
And resource "deployment/test-operator" has label "app.kubernetes.io/name" with value "test-operator"
47+
48+
Scenario: Deployment rollout restart persists after OLM reconciliation
49+
When ClusterExtension is applied
50+
"""
51+
apiVersion: olm.operatorframework.io/v1
52+
kind: ClusterExtension
53+
metadata:
54+
name: ${NAME}
55+
spec:
56+
namespace: ${TEST_NAMESPACE}
57+
serviceAccount:
58+
name: olm-sa
59+
source:
60+
sourceType: Catalog
61+
catalog:
62+
packageName: test
63+
selector:
64+
matchLabels:
65+
"olm.operatorframework.io/metadata.name": test-catalog
66+
"""
67+
Then ClusterExtension is rolled out
68+
And ClusterExtension is available
69+
And resource "deployment/test-operator" is available
70+
When user performs rollout restart on "deployment/test-operator"
71+
Then deployment "test-operator" has restart annotation
72+
And deployment "test-operator" rollout is complete
73+
And deployment "test-operator" has 2 replica sets
74+
When ClusterExtension reconciliation is triggered
75+
And ClusterExtension has been reconciled the latest generation
76+
Then deployment "test-operator" has restart annotation
77+
And deployment "test-operator" rollout is complete

‎test/e2e/steps/steps.go‎

Lines changed: 280 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,15 @@ func RegisterSteps(sc *godog.ScenarioContext) {
9090
sc.Step(`^(?i)resource apply fails with error msg containing "([^"]+)"$`, ResourceApplyFails)
9191
sc.Step(`^(?i)resource "([^"]+)" is eventually restored$`, ResourceRestored)
9292
sc.Step(`^(?i)resource "([^"]+)" matches$`, ResourceMatches)
93+
sc.Step(`^(?i)user performs rollout restart on "([^"]+)"$`, UserPerformsRolloutRestart)
94+
sc.Step(`^(?i)user adds annotation "([^"]+)" to "([^"]+)"$`, UserAddsAnnotation)
95+
sc.Step(`^(?i)user adds label "([^"]+)" to "([^"]+)"$`, UserAddsLabel)
96+
sc.Step(`^(?i)resource "([^"]+)" has annotation "([^"]+)" with value "([^"]+)"$`, ResourceHasAnnotation)
97+
sc.Step(`^(?i)resource "([^"]+)" has label "([^"]+)" with value "([^"]+)"$`, ResourceHasLabel)
98+
sc.Step(`^(?i)deployment "([^"]+)" has restart annotation$`, DeploymentHasRestartAnnotation)
99+
sc.Step(`^(?i)deployment "([^"]+)" rollout is complete$`, DeploymentRolloutIsComplete)
100+
sc.Step(`^(?i)deployment "([^"]+)" has (\d+) replica sets?$`, DeploymentHasReplicaSets)
101+
sc.Step(`^(?i)ClusterExtension reconciliation is triggered$`, TriggerClusterExtensionReconciliation)
93102

94103
sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in test namespace$`, ServiceAccountWithNeededPermissionsIsAvailableInNamespace)
95104
sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in \${TEST_NAMESPACE}$`, ServiceAccountWithNeededPermissionsIsAvailableInNamespace)
@@ -1309,3 +1318,274 @@ func latestActiveRevisionForExtension(extName string) (*ocv1.ClusterExtensionRev
13091318

13101319
return latest, nil
13111320
}
1321+
1322+
// UserAddsAnnotation adds a custom annotation to a resource using kubectl annotate.
1323+
func UserAddsAnnotation(ctx context.Context, annotation, resourceName string) error {
1324+
sc := scenarioCtx(ctx)
1325+
resourceName = substituteScenarioVars(resourceName, sc)
1326+
1327+
kind, name, ok := strings.Cut(resourceName, "/")
1328+
if !ok {
1329+
return fmt.Errorf("invalid resource name format: %q (expected kind/name)", resourceName)
1330+
}
1331+
1332+
out, err := k8sClient("annotate", kind, name, annotation, "--overwrite", "-n", sc.namespace)
1333+
if err != nil {
1334+
return fmt.Errorf("failed to annotate %s: %w; stderr: %s", resourceName, err, stderrOutput(err))
1335+
}
1336+
logger.V(1).Info("Annotation added", "resource", resourceName, "annotation", annotation, "output", out)
1337+
return nil
1338+
}
1339+
1340+
// UserAddsLabel adds a custom label to a resource using kubectl label.
1341+
func UserAddsLabel(ctx context.Context, label, resourceName string) error {
1342+
sc := scenarioCtx(ctx)
1343+
resourceName = substituteScenarioVars(resourceName, sc)
1344+
1345+
kind, name, ok := strings.Cut(resourceName, "/")
1346+
if !ok {
1347+
return fmt.Errorf("invalid resource name format: %q (expected kind/name)", resourceName)
1348+
}
1349+
1350+
out, err := k8sClient("label", kind, name, label, "--overwrite", "-n", sc.namespace)
1351+
if err != nil {
1352+
return fmt.Errorf("failed to label %s: %w; stderr: %s", resourceName, err, stderrOutput(err))
1353+
}
1354+
logger.V(1).Info("Label added", "resource", resourceName, "label", label, "output", out)
1355+
return nil
1356+
}
1357+
1358+
// ResourceHasAnnotation waits for a resource to have the given annotation key with the expected value.
1359+
func ResourceHasAnnotation(ctx context.Context, resourceName, annotationKey, expectedValue string) error {
1360+
sc := scenarioCtx(ctx)
1361+
resourceName = substituteScenarioVars(resourceName, sc)
1362+
1363+
kind, name, ok := strings.Cut(resourceName, "/")
1364+
if !ok {
1365+
return fmt.Errorf("invalid resource name format: %q (expected kind/name)", resourceName)
1366+
}
1367+
1368+
waitFor(ctx, func() bool {
1369+
out, err := k8sClient("get", kind, name, "-n", sc.namespace, "-o", "json")
1370+
if err != nil {
1371+
return false
1372+
}
1373+
var obj unstructured.Unstructured
1374+
if err := json.Unmarshal([]byte(out), &obj); err != nil {
1375+
return false
1376+
}
1377+
annotations := obj.GetAnnotations()
1378+
if v, found := annotations[annotationKey]; found && v == expectedValue {
1379+
logger.V(1).Info("Annotation found", "resource", resourceName, "key", annotationKey, "value", v)
1380+
return true
1381+
}
1382+
logger.V(1).Info("Annotation not yet present or value mismatch", "resource", resourceName, "key", annotationKey, "annotations", annotations)
1383+
return false
1384+
})
1385+
return nil
1386+
}
1387+
1388+
// ResourceHasLabel waits for a resource to have the given label key with the expected value.
1389+
func ResourceHasLabel(ctx context.Context, resourceName, labelKey, expectedValue string) error {
1390+
sc := scenarioCtx(ctx)
1391+
resourceName = substituteScenarioVars(resourceName, sc)
1392+
1393+
kind, name, ok := strings.Cut(resourceName, "/")
1394+
if !ok {
1395+
return fmt.Errorf("invalid resource name format: %q (expected kind/name)", resourceName)
1396+
}
1397+
1398+
waitFor(ctx, func() bool {
1399+
out, err := k8sClient("get", kind, name, "-n", sc.namespace, "-o", "json")
1400+
if err != nil {
1401+
return false
1402+
}
1403+
var obj unstructured.Unstructured
1404+
if err := json.Unmarshal([]byte(out), &obj); err != nil {
1405+
return false
1406+
}
1407+
labels := obj.GetLabels()
1408+
if v, found := labels[labelKey]; found && v == expectedValue {
1409+
logger.V(1).Info("Label found", "resource", resourceName, "key", labelKey, "value", v)
1410+
return true
1411+
}
1412+
logger.V(1).Info("Label not yet present or value mismatch", "resource", resourceName, "key", labelKey, "labels", labels)
1413+
return false
1414+
})
1415+
return nil
1416+
}
1417+
1418+
// UserPerformsRolloutRestart simulates a user running "kubectl rollout restart deployment/<name>".
1419+
// See: https://github.com/operator-framework/operator-lifecycle-manager/issues/3392
1420+
func UserPerformsRolloutRestart(ctx context.Context, resourceName string) error {
1421+
sc := scenarioCtx(ctx)
1422+
resourceName = substituteScenarioVars(resourceName, sc)
1423+
1424+
kind, deploymentName, ok := strings.Cut(resourceName, "/")
1425+
if !ok {
1426+
return fmt.Errorf("invalid resource name format: %q (expected kind/name)", resourceName)
1427+
}
1428+
1429+
if kind != "deployment" {
1430+
return fmt.Errorf("only deployment resources are supported for restart annotation, got: %q", kind)
1431+
}
1432+
1433+
// Run kubectl rollout restart to add the restart annotation.
1434+
// This is the real command users run, so we test actual user behavior.
1435+
out, err := k8sClient("rollout", "restart", resourceName, "-n", sc.namespace)
1436+
if err != nil {
1437+
return fmt.Errorf("failed to rollout restart %s: %w; stderr: %s", resourceName, err, stderrOutput(err))
1438+
}
1439+
1440+
logger.V(1).Info("Rollout restart initiated", "deployment", deploymentName, "output", out)
1441+
1442+
return nil
1443+
}
1444+
1445+
// DeploymentHasRestartAnnotation waits for the deployment's pod template to have
1446+
// the kubectl.kubernetes.io/restartedAt annotation. Uses JSON parsing to avoid
1447+
// JSONPath issues with dots in annotation keys. Polls with timeout.
1448+
func DeploymentHasRestartAnnotation(ctx context.Context, deploymentName string) error {
1449+
sc := scenarioCtx(ctx)
1450+
deploymentName = substituteScenarioVars(deploymentName, sc)
1451+
1452+
restartAnnotationKey := "kubectl.kubernetes.io/restartedAt"
1453+
waitFor(ctx, func() bool {
1454+
out, err := k8sClient("get", "deployment", deploymentName, "-n", sc.namespace, "-o", "json")
1455+
if err != nil {
1456+
return false
1457+
}
1458+
var d appsv1.Deployment
1459+
if err := json.Unmarshal([]byte(out), &d); err != nil {
1460+
return false
1461+
}
1462+
if v, found := d.Spec.Template.Annotations[restartAnnotationKey]; found {
1463+
logger.V(1).Info("Restart annotation found", "deployment", deploymentName, "restartedAt", v)
1464+
return true
1465+
}
1466+
logger.V(1).Info("Restart annotation not yet present", "deployment", deploymentName, "annotations", d.Spec.Template.Annotations)
1467+
return false
1468+
})
1469+
return nil
1470+
}
1471+
1472+
// TriggerClusterExtensionReconciliation patches the ClusterExtension spec to bump
1473+
// its metadata generation, forcing the controller to run a full reconciliation loop.
1474+
// Use with "ClusterExtension has been reconciled the latest generation" to confirm
1475+
// the controller processed the change before asserting on the cluster state.
1476+
//
1477+
// We set install.preflight.crdUpgradeSafety.enforcement to "None" because it is
1478+
// a real spec field that the API server will persist (unlike unknown fields, which
1479+
// are pruned by structural schemas). Any persisted spec change bumps
1480+
// .metadata.generation, giving us a reliable synchronization signal.
1481+
func TriggerClusterExtensionReconciliation(ctx context.Context) error {
1482+
sc := scenarioCtx(ctx)
1483+
payload := `{"spec":{"install":{"preflight":{"crdUpgradeSafety":{"enforcement":"None"}}}}}`
1484+
_, err := k8sClient("patch", "clusterextension", sc.clusterExtensionName,
1485+
"--type=merge",
1486+
"-p", payload)
1487+
if err != nil {
1488+
return fmt.Errorf("failed to trigger reconciliation for ClusterExtension %s: %w; stderr: %s", sc.clusterExtensionName, err, stderrOutput(err))
1489+
}
1490+
return nil
1491+
}
1492+
1493+
// DeploymentRolloutIsComplete verifies that a deployment rollout has completed successfully.
1494+
// This ensures the new ReplicaSet is fully scaled up and the old one is scaled down.
1495+
func DeploymentRolloutIsComplete(ctx context.Context, deploymentName string) error {
1496+
sc := scenarioCtx(ctx)
1497+
deploymentName = substituteScenarioVars(deploymentName, sc)
1498+
1499+
waitFor(ctx, func() bool {
1500+
out, err := k8sClient("rollout", "status", "deployment/"+deploymentName, "-n", sc.namespace, "--watch=false")
1501+
if err != nil {
1502+
logger.V(1).Info("Failed to get rollout status", "deployment", deploymentName, "error", err)
1503+
return false
1504+
}
1505+
// Successful rollout shows "successfully rolled out"
1506+
if strings.Contains(out, "successfully rolled out") {
1507+
logger.V(1).Info("Rollout completed successfully", "deployment", deploymentName)
1508+
return true
1509+
}
1510+
logger.V(1).Info("Rollout not yet complete", "deployment", deploymentName, "status", out)
1511+
return false
1512+
})
1513+
return nil
1514+
}
1515+
1516+
// DeploymentHasReplicaSets verifies that a deployment has the expected number of ReplicaSets
1517+
// and that the latest one is active with pods running.
1518+
func DeploymentHasReplicaSets(ctx context.Context, deploymentName string, expectedCountStr string) error {
1519+
sc := scenarioCtx(ctx)
1520+
deploymentName = substituteScenarioVars(deploymentName, sc)
1521+
1522+
expectedCount := 2 // Default to 2 (original + restarted)
1523+
if n, err := fmt.Sscanf(expectedCountStr, "%d", &expectedCount); err != nil || n != 1 {
1524+
logger.V(1).Info("Failed to parse expected count, using default", "input", expectedCountStr, "default", 2)
1525+
expectedCount = 2
1526+
}
1527+
1528+
waitFor(ctx, func() bool {
1529+
// First, get the deployment to find its selector labels
1530+
deploymentOut, err := k8sClient("get", "deployment", deploymentName, "-n", sc.namespace, "-o", "json")
1531+
if err != nil {
1532+
logger.V(1).Info("Failed to get deployment", "deployment", deploymentName, "error", err)
1533+
return false
1534+
}
1535+
1536+
var deployment appsv1.Deployment
1537+
if err := json.Unmarshal([]byte(deploymentOut), &deployment); err != nil {
1538+
logger.V(1).Info("Failed to parse deployment", "error", err)
1539+
return false
1540+
}
1541+
1542+
// Get all ReplicaSets owned by this deployment using ownerReferences
1543+
out, err := k8sClient("get", "rs", "-n", sc.namespace, "-o", "json")
1544+
if err != nil {
1545+
logger.V(1).Info("Failed to get ReplicaSets", "deployment", deploymentName, "error", err)
1546+
return false
1547+
}
1548+
1549+
var allRsList struct {
1550+
Items []appsv1.ReplicaSet `json:"items"`
1551+
}
1552+
if err := json.Unmarshal([]byte(out), &allRsList); err != nil {
1553+
logger.V(1).Info("Failed to parse ReplicaSets", "error", err)
1554+
return false
1555+
}
1556+
1557+
// Filter ReplicaSets owned by this deployment
1558+
var rsList []appsv1.ReplicaSet
1559+
for _, rs := range allRsList.Items {
1560+
for _, owner := range rs.OwnerReferences {
1561+
if owner.Kind == "Deployment" && owner.Name == deploymentName {
1562+
rsList = append(rsList, rs)
1563+
break
1564+
}
1565+
}
1566+
}
1567+
1568+
if len(rsList) < expectedCount {
1569+
logger.V(1).Info("Not enough ReplicaSets yet", "deployment", deploymentName, "current", len(rsList), "expected", expectedCount)
1570+
return false
1571+
}
1572+
1573+
// Verify at least one ReplicaSet has active replicas
1574+
hasActiveRS := false
1575+
for _, rs := range rsList {
1576+
if rs.Status.Replicas > 0 && rs.Status.ReadyReplicas > 0 {
1577+
hasActiveRS = true
1578+
logger.V(1).Info("Found active ReplicaSet", "name", rs.Name, "replicas", rs.Status.Replicas, "ready", rs.Status.ReadyReplicas)
1579+
}
1580+
}
1581+
1582+
if !hasActiveRS {
1583+
logger.V(1).Info("No active ReplicaSet found yet", "deployment", deploymentName)
1584+
return false
1585+
}
1586+
1587+
logger.V(1).Info("ReplicaSet verification passed", "deployment", deploymentName, "count", len(rsList))
1588+
return true
1589+
})
1590+
return nil
1591+
}

0 commit comments

Comments
 (0)