From 8f92bafbb5f4e2342c87b89d5b217f1c1e16e81e Mon Sep 17 00:00:00 2001 From: michaelhtm <98621731+michaelhtm@users.noreply.github.com> Date: Tue, 5 May 2026 17:14:19 -0700 Subject: [PATCH 1/2] init --- pkg/config/field_group.go | 51 ++ pkg/config/resource.go | 56 +++ pkg/config/validate.go | 71 +++ pkg/config/validate_test.go | 195 ++++++++ pkg/generate/ack/controller.go | 20 + pkg/generate/code/compare_field_group.go | 57 +++ pkg/generate/code/field_group_test.go | 148 ++++++ pkg/generate/code/set_resource_field_group.go | 202 ++++++++ pkg/generate/code/set_sdk_field_group.go | 158 ++++++ pkg/generate/code/set_sdk_sync.go | 219 ++++++++ pkg/model/crd.go | 6 + pkg/model/field_group.go | 467 ++++++++++++++++++ pkg/model/field_group_test.go | 154 ++++++ pkg/model/model.go | 9 + .../generator-with-field-groups.yaml | 19 + templates/pkg/resource/manager.go.tpl | 6 + templates/pkg/resource/sdk.go.tpl | 12 +- .../pkg/resource/sdk_find_field_groups.go.tpl | 65 +++ .../resource/sdk_update_field_groups.go.tpl | 170 +++++++ 19 files changed, 2084 insertions(+), 1 deletion(-) create mode 100644 pkg/config/field_group.go create mode 100644 pkg/generate/code/compare_field_group.go create mode 100644 pkg/generate/code/field_group_test.go create mode 100644 pkg/generate/code/set_resource_field_group.go create mode 100644 pkg/generate/code/set_sdk_field_group.go create mode 100644 pkg/generate/code/set_sdk_sync.go create mode 100644 pkg/model/field_group.go create mode 100644 pkg/model/field_group_test.go create mode 100644 pkg/testdata/models/apis/ecr/0000-00-00/generator-with-field-groups.yaml create mode 100644 templates/pkg/resource/sdk_find_field_groups.go.tpl create mode 100644 templates/pkg/resource/sdk_update_field_groups.go.tpl diff --git a/pkg/config/field_group.go b/pkg/config/field_group.go new file mode 100644 index 00000000..6d3ddd5f --- /dev/null +++ b/pkg/config/field_group.go @@ -0,0 +1,51 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package config + +// FieldGroupOperationConfig defines an API operation that manages a subset of +// a resource's fields. Used in both update_operations and read_operations to +// specify per-field-group API calls. +// +// When the generator encounters these configs, it auto-detects which fields +// belong to each operation by inspecting the operation's Input/Output shape +// and cross-referencing with the resource's identifier fields (those shared +// with ReadOne/Delete operations). Explicitly listing fields in the Fields +// slice overrides this auto-detection. +// +// Example generator.yaml: +// +// resources: +// Repository: +// update_operations: +// - operation_id: PutImageScanningConfiguration +// requeue_on_success: true +// - operation_id: PutImageTagMutability +// read_operations: +// - operation_id: GetLifecyclePolicy +// - operation_id: GetRepositoryPolicy +type FieldGroupOperationConfig struct { + // OperationID is the SDK operation's exported name + // (e.g., "PutImageScanningConfiguration"). + OperationID string `json:"operation_id"` + // Fields optionally overrides auto-detection of payload fields. When + // empty, payload fields are auto-detected from the operation's Input + // shape by excluding identifier fields. When set, only the listed CRD + // field names are treated as payload fields for this group. + Fields []string `json:"fields,omitempty"` + // RequeueOnSuccess, when true, causes the reconciler to requeue after + // this operation succeeds. This is useful for operations whose response + // does not contain the updated field values, requiring a subsequent + // ReadOne to refresh state. + RequeueOnSuccess bool `json:"requeue_on_success,omitempty"` +} diff --git a/pkg/config/resource.go b/pkg/config/resource.go index b71929bb..e3db2eea 100644 --- a/pkg/config/resource.go +++ b/pkg/config/resource.go @@ -135,6 +135,18 @@ type ResourceConfig struct { // SDK implementation details that are auto-filled by the SDK middleware // when nil and should not be exposed in the CRD. IgnoreIdempotencyToken bool `json:"ignore_idempotency_token,omitempty"` + // UpdateOperations defines per-field-group update API calls. When + // present, the generated sdkUpdate becomes an orchestrator that calls + // each operation only for its changed fields. When absent, the existing + // single-update behavior is preserved. + // + // Cannot be used together with update_operation.custom_method_name. + UpdateOperations []FieldGroupOperationConfig `json:"update_operations,omitempty"` + // ReadOperations defines per-field-group read API calls that supplement + // the primary ReadOne/ReadMany/GetAttributes operation. Each entry + // specifies an additional API call whose output populates a subset of + // the resource's fields. + ReadOperations []FieldGroupOperationConfig `json:"read_operations,omitempty"` } // TagConfig instructs the code generator on how to generate functions that @@ -874,3 +886,47 @@ func (rc *ResourceConfig) GetFieldConfig(subject string) *FieldConfig { } return nil } + +// GetUpdateFieldGroupOperations returns the list of per-field-group update +// operation configs for a given resource, or nil if none are configured. +func (c *Config) GetUpdateFieldGroupOperations(resourceName string) []FieldGroupOperationConfig { + if c == nil { + return nil + } + rConfig, found := c.Resources[resourceName] + if !found { + return nil + } + if len(rConfig.UpdateOperations) == 0 { + return nil + } + return rConfig.UpdateOperations +} + +// GetReadFieldGroupOperations returns the list of per-field-group read +// operation configs for a given resource, or nil if none are configured. +func (c *Config) GetReadFieldGroupOperations(resourceName string) []FieldGroupOperationConfig { + if c == nil { + return nil + } + rConfig, found := c.Resources[resourceName] + if !found { + return nil + } + if len(rConfig.ReadOperations) == 0 { + return nil + } + return rConfig.ReadOperations +} + +// HasFieldGroupUpdates returns true if the resource has per-field-group +// update operations configured. +func (c *Config) HasFieldGroupUpdates(resourceName string) bool { + return len(c.GetUpdateFieldGroupOperations(resourceName)) > 0 +} + +// HasFieldGroupReads returns true if the resource has per-field-group +// read operations configured. +func (c *Config) HasFieldGroupReads(resourceName string) bool { + return len(c.GetReadFieldGroupOperations(resourceName)) > 0 +} diff --git a/pkg/config/validate.go b/pkg/config/validate.go index 285ec13c..f4b87be9 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -44,6 +44,7 @@ func ValidateConfig( errs = append(errs, validateRenameOperations(cfg, sdkOperations)...) errs = append(errs, validateIgnoredOperations(cfg, sdkOperations)...) + errs = append(errs, validateFieldGroupOperations(cfg, sdkOperations)...) return errs } @@ -99,6 +100,76 @@ func sortedKeys(m map[string]struct{}) []string { return keys } +// validateFieldGroupOperations checks that field group operation configs +// (update_operations and read_operations) reference valid SDK operations and +// are not misconfigured. +func validateFieldGroupOperations( + cfg *Config, + sdkOperations map[string]struct{}, +) []error { + var errs []error + for resName, resCfg := range cfg.Resources { + // Check update_operations + seen := make(map[string]bool) + for i, fgCfg := range resCfg.UpdateOperations { + if fgCfg.OperationID == "" { + errs = append(errs, fmt.Errorf( + "resources.%s.update_operations[%d]: operation_id must not be empty", + resName, i, + )) + continue + } + if _, ok := sdkOperations[fgCfg.OperationID]; !ok { + errs = append(errs, fmt.Errorf( + "resources.%s.update_operations[%d]: operation %q not found in SDK. available: %s", + resName, i, fgCfg.OperationID, formatAvailableTruncated(sortedKeys(sdkOperations), 10), + )) + } + if seen[fgCfg.OperationID] { + errs = append(errs, fmt.Errorf( + "resources.%s.update_operations[%d]: duplicate operation_id %q", + resName, i, fgCfg.OperationID, + )) + } + seen[fgCfg.OperationID] = true + } + + // Mutual exclusivity: custom_method_name + update_operations + if len(resCfg.UpdateOperations) > 0 && resCfg.UpdateOperation != nil && resCfg.UpdateOperation.CustomMethodName != "" { + errs = append(errs, fmt.Errorf( + "resources.%s: update_operations cannot be used together with update_operation.custom_method_name", + resName, + )) + } + + // Check read_operations + seen = make(map[string]bool) + for i, fgCfg := range resCfg.ReadOperations { + if fgCfg.OperationID == "" { + errs = append(errs, fmt.Errorf( + "resources.%s.read_operations[%d]: operation_id must not be empty", + resName, i, + )) + continue + } + if _, ok := sdkOperations[fgCfg.OperationID]; !ok { + errs = append(errs, fmt.Errorf( + "resources.%s.read_operations[%d]: operation %q not found in SDK. available: %s", + resName, i, fgCfg.OperationID, formatAvailableTruncated(sortedKeys(sdkOperations), 10), + )) + } + if seen[fgCfg.OperationID] { + errs = append(errs, fmt.Errorf( + "resources.%s.read_operations[%d]: duplicate operation_id %q", + resName, i, fgCfg.OperationID, + )) + } + seen[fgCfg.OperationID] = true + } + } + return errs +} + // formatAvailableTruncated formats a sorted slice, showing at most maxItems // entries with an ellipsis if truncated. func formatAvailableTruncated(items []string, maxItems int) string { diff --git a/pkg/config/validate_test.go b/pkg/config/validate_test.go index 686862b2..ef95d341 100644 --- a/pkg/config/validate_test.go +++ b/pkg/config/validate_test.go @@ -238,6 +238,201 @@ func TestValidateConfig_ErrorMessageIncludesAvailable(t *testing.T) { } } +func TestValidateFieldGroupOperations_ValidUpdateOps(t *testing.T) { + sdkOps := map[string]struct{}{ + "CreateRepository": {}, + "DeleteRepository": {}, + "PutImageScanningConfiguration": {}, + "PutImageTagMutability": {}, + } + + cfg := &Config{ + Resources: map[string]ResourceConfig{ + "Repository": { + UpdateOperations: []FieldGroupOperationConfig{ + {OperationID: "PutImageScanningConfiguration"}, + {OperationID: "PutImageTagMutability"}, + }, + }, + }, + } + + errs := validateFieldGroupOperations(cfg, sdkOps) + if len(errs) != 0 { + t.Errorf("expected 0 errors, got %d: %v", len(errs), errs) + } +} + +func TestValidateFieldGroupOperations_InvalidOperationID(t *testing.T) { + sdkOps := map[string]struct{}{ + "CreateRepository": {}, + } + + cfg := &Config{ + Resources: map[string]ResourceConfig{ + "Repository": { + UpdateOperations: []FieldGroupOperationConfig{ + {OperationID: "NonExistentOperation"}, + }, + }, + }, + } + + errs := validateFieldGroupOperations(cfg, sdkOps) + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d: %v", len(errs), errs) + } + if !strings.Contains(errs[0].Error(), "NonExistentOperation") { + t.Errorf("error should reference the bad operation, got: %s", errs[0].Error()) + } +} + +func TestValidateFieldGroupOperations_EmptyOperationID(t *testing.T) { + sdkOps := map[string]struct{}{} + + cfg := &Config{ + Resources: map[string]ResourceConfig{ + "Repository": { + UpdateOperations: []FieldGroupOperationConfig{ + {OperationID: ""}, + }, + }, + }, + } + + errs := validateFieldGroupOperations(cfg, sdkOps) + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d: %v", len(errs), errs) + } + if !strings.Contains(errs[0].Error(), "must not be empty") { + t.Errorf("error should mention empty operation_id, got: %s", errs[0].Error()) + } +} + +func TestValidateFieldGroupOperations_DuplicateOperationID(t *testing.T) { + sdkOps := map[string]struct{}{ + "PutImageScanningConfiguration": {}, + } + + cfg := &Config{ + Resources: map[string]ResourceConfig{ + "Repository": { + UpdateOperations: []FieldGroupOperationConfig{ + {OperationID: "PutImageScanningConfiguration"}, + {OperationID: "PutImageScanningConfiguration"}, + }, + }, + }, + } + + errs := validateFieldGroupOperations(cfg, sdkOps) + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d: %v", len(errs), errs) + } + if !strings.Contains(errs[0].Error(), "duplicate") { + t.Errorf("error should mention duplicate, got: %s", errs[0].Error()) + } +} + +func TestValidateFieldGroupOperations_CustomMethodConflict(t *testing.T) { + sdkOps := map[string]struct{}{ + "PutImageScanningConfiguration": {}, + } + + cfg := &Config{ + Resources: map[string]ResourceConfig{ + "Repository": { + UpdateOperation: &UpdateOperationConfig{ + CustomMethodName: "customUpdate", + }, + UpdateOperations: []FieldGroupOperationConfig{ + {OperationID: "PutImageScanningConfiguration"}, + }, + }, + }, + } + + errs := validateFieldGroupOperations(cfg, sdkOps) + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d: %v", len(errs), errs) + } + if !strings.Contains(errs[0].Error(), "custom_method_name") { + t.Errorf("error should mention custom_method_name conflict, got: %s", errs[0].Error()) + } +} + +func TestValidateFieldGroupOperations_UpdateOpWithoutCustomMethodAllowed(t *testing.T) { + sdkOps := map[string]struct{}{ + "PutImageScanningConfiguration": {}, + } + + cfg := &Config{ + Resources: map[string]ResourceConfig{ + "Repository": { + UpdateOperation: &UpdateOperationConfig{ + OmitUnchangedFields: true, + }, + UpdateOperations: []FieldGroupOperationConfig{ + {OperationID: "PutImageScanningConfiguration"}, + }, + }, + }, + } + + errs := validateFieldGroupOperations(cfg, sdkOps) + if len(errs) != 0 { + t.Errorf("expected 0 errors when custom_method_name is empty, got %d: %v", len(errs), errs) + } +} + +func TestValidateFieldGroupOperations_ReadOps(t *testing.T) { + sdkOps := map[string]struct{}{ + "GetLifecyclePolicy": {}, + "GetRepositoryPolicy": {}, + } + + cfg := &Config{ + Resources: map[string]ResourceConfig{ + "Repository": { + ReadOperations: []FieldGroupOperationConfig{ + {OperationID: "GetLifecyclePolicy"}, + {OperationID: "GetRepositoryPolicy"}, + }, + }, + }, + } + + errs := validateFieldGroupOperations(cfg, sdkOps) + if len(errs) != 0 { + t.Errorf("expected 0 errors, got %d: %v", len(errs), errs) + } +} + +func TestValidateFieldGroupOperations_ReadOpsDuplicate(t *testing.T) { + sdkOps := map[string]struct{}{ + "GetLifecyclePolicy": {}, + } + + cfg := &Config{ + Resources: map[string]ResourceConfig{ + "Repository": { + ReadOperations: []FieldGroupOperationConfig{ + {OperationID: "GetLifecyclePolicy"}, + {OperationID: "GetLifecyclePolicy"}, + }, + }, + }, + } + + errs := validateFieldGroupOperations(cfg, sdkOps) + if len(errs) != 1 { + t.Fatalf("expected 1 error, got %d: %v", len(errs), errs) + } + if !strings.Contains(errs[0].Error(), "duplicate") { + t.Errorf("error should mention duplicate, got: %s", errs[0].Error()) + } +} + func TestFormatAvailableTruncated(t *testing.T) { items := []string{"A", "B", "C", "D", "E"} got := formatAvailableTruncated(items, 3) diff --git a/pkg/generate/ack/controller.go b/pkg/generate/ack/controller.go index 5beab739..c0426be1 100644 --- a/pkg/generate/ack/controller.go +++ b/pkg/generate/ack/controller.go @@ -50,12 +50,14 @@ var ( "pkg/resource/references_read_referenced_resource.go.tpl", "pkg/resource/sdk_delete_custom.go.tpl", "pkg/resource/sdk_find_custom.go.tpl", + "pkg/resource/sdk_find_field_groups.go.tpl", "pkg/resource/sdk_find_read_one.go.tpl", "pkg/resource/sdk_find_get_attributes.go.tpl", "pkg/resource/sdk_find_read_many.go.tpl", "pkg/resource/sdk_find_not_implemented.go.tpl", "pkg/resource/sdk_update.go.tpl", "pkg/resource/sdk_update_custom.go.tpl", + "pkg/resource/sdk_update_field_groups.go.tpl", "pkg/resource/sdk_update_set_attributes.go.tpl", "pkg/resource/sdk_update_not_implemented.go.tpl", } @@ -154,6 +156,24 @@ var ( "HasPreDeleteSync": func(r *ackmodel.CRD) bool { return code.HasPreDeleteSync(r.Config(), r) }, + "GoCodeSetFieldGroupInput": func(r *ackmodel.CRD, fg *ackmodel.FieldGroupOperation, sourceVarName string, targetVarName string, indentLevel int) (string, error) { + return code.SetSDKFieldGroup(r.Config(), r, fg, sourceVarName, targetVarName, indentLevel) + }, + "GoCodeSetFieldGroupOutput": func(r *ackmodel.CRD, fg *ackmodel.FieldGroupOperation, sourceVarName string, targetVarName string, indentLevel int) (string, error) { + return code.SetResourceFieldGroup(r.Config(), r, fg, sourceVarName, targetVarName, indentLevel) + }, + "GoCodeFieldGroupDeltaCheck": func(r *ackmodel.CRD, fg *ackmodel.FieldGroupOperation, deltaVarName string) string { + return code.FieldGroupDeltaCheck(r.Config(), r, fg, deltaVarName) + }, + "GoCodeSetSyncInput": func(r *ackmodel.CRD, fg *ackmodel.FieldGroupOperation, resourceVarName string, targetVarName string, itemVarName string, indentLevel int) string { + return code.SetSyncInput(r.Config(), r, fg, resourceVarName, targetVarName, itemVarName, indentLevel) + }, + "GoCodeSetSyncMapAddInput": func(r *ackmodel.CRD, fg *ackmodel.FieldGroupOperation, resourceVarName string, targetVarName string, keyVarName string, valVarName string, indentLevel int) string { + return code.SetSyncMapAddInput(r.Config(), r, fg, resourceVarName, targetVarName, keyVarName, valVarName, indentLevel) + }, + "GoCodeSetSyncMapRemoveInput": func(r *ackmodel.CRD, fg *ackmodel.FieldGroupOperation, resourceVarName string, targetVarName string, keyVarName string, indentLevel int) string { + return code.SetSyncMapRemoveInput(r.Config(), r, fg, resourceVarName, targetVarName, keyVarName, indentLevel) + }, "GoCodeIsSynced": func(r *ackmodel.CRD, resVarName string, indentLevel int) (string, error) { return code.ResourceIsSynced(r.Config(), r, resVarName, indentLevel) }, diff --git a/pkg/generate/code/compare_field_group.go b/pkg/generate/code/compare_field_group.go new file mode 100644 index 00000000..79297ebe --- /dev/null +++ b/pkg/generate/code/compare_field_group.go @@ -0,0 +1,57 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package code + +import ( + "fmt" + "strings" + + ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/config" + "github.com/aws-controllers-k8s/code-generator/pkg/model" +) + +// FieldGroupDeltaCheck returns a Go boolean expression string that evaluates +// to true if any of the field group's payload fields have changed according +// to the delta. Used by update orchestrator templates to decide whether to +// call a specific field-group update operation. +// +// The generated expression looks like: +// +// delta.DifferentAt("Spec.ImageScanningConfiguration") +// +// For multiple payload fields: +// +// delta.DifferentAt("Spec.ImageScanningConfiguration") || delta.DifferentAt("Spec.ImageTagMutability") +func FieldGroupDeltaCheck( + cfg *ackgenconfig.Config, + r *model.CRD, + fg *model.FieldGroupOperation, + deltaVarName string, +) string { + if len(fg.PayloadFields) == 0 { + return "false" + } + + parts := make([]string, 0, len(fg.PayloadFields)) + for _, f := range fg.PayloadFields { + fieldPath := fmt.Sprintf("%s.%s", + strings.TrimPrefix(cfg.PrefixConfig.SpecField, "."), + f.Names.Camel, + ) + parts = append(parts, fmt.Sprintf( + "%s.DifferentAt(%q)", deltaVarName, fieldPath, + )) + } + return strings.Join(parts, " || ") +} diff --git a/pkg/generate/code/field_group_test.go b/pkg/generate/code/field_group_test.go new file mode 100644 index 00000000..e0b380b2 --- /dev/null +++ b/pkg/generate/code/field_group_test.go @@ -0,0 +1,148 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package code_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aws-controllers-k8s/code-generator/pkg/generate/code" + "github.com/aws-controllers-k8s/code-generator/pkg/testutil" +) + +func TestSetSDKFieldGroup_ECR_PutImageScanningConfiguration(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{ + GeneratorConfigFile: "generator-with-field-groups.yaml", + }) + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + require.True(crd.HasFieldGroupUpdates()) + require.Len(crd.UpdateFieldGroups, 2) + + fg := crd.UpdateFieldGroups[0] + require.Equal("PutImageScanningConfiguration", fg.OperationID) + + got, err := code.SetSDKFieldGroup(crd.Config(), crd, fg, "r.ko", "res", 1) + require.NoError(err) + + // Should set identifier fields (RegistryId, RepositoryName) and + // payload field (ImageScanningConfiguration) + assert.Contains(got, "r.ko.Spec.ImageScanningConfiguration") + assert.Contains(got, "r.ko.Spec.RegistryID") + assert.Contains(got, "r.ko.Spec.RepositoryName") + assert.Contains(got, "res.") + + // Should NOT contain Tags (not part of this field group) + assert.NotContains(got, "Tags") + // Should NOT contain ImageTagMutability (belongs to different field group) + assert.NotContains(got, "ImageTagMutability") +} + +func TestSetSDKFieldGroup_ECR_PutImageTagMutability(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{ + GeneratorConfigFile: "generator-with-field-groups.yaml", + }) + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + + fg := crd.UpdateFieldGroups[1] + require.Equal("PutImageTagMutability", fg.OperationID) + + got, err := code.SetSDKFieldGroup(crd.Config(), crd, fg, "r.ko", "res", 1) + require.NoError(err) + + assert.Contains(got, "r.ko.Spec.ImageTagMutability") + assert.Contains(got, "r.ko.Spec.RegistryID") + assert.Contains(got, "r.ko.Spec.RepositoryName") + // Should NOT contain ImageScanningConfiguration + assert.NotContains(got, "ImageScanningConfiguration") +} + +func TestSetResourceFieldGroup_ECR_PutImageScanningConfiguration(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{ + GeneratorConfigFile: "generator-with-field-groups.yaml", + }) + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + + fg := crd.UpdateFieldGroups[0] + require.Equal("PutImageScanningConfiguration", fg.OperationID) + + got, err := code.SetResourceFieldGroup(crd.Config(), crd, fg, "resp", "ko", 1) + require.NoError(err) + + // Should set the payload field from output + assert.Contains(got, "resp.ImageScanningConfiguration") + assert.Contains(got, "ko.Spec.ImageScanningConfiguration") + + // Should NOT contain identifier fields in output (they're not payload) + // RegistryId and RepositoryName are identifiers, not payload + assert.NotContains(got, "ko.Spec.RegistryID") + assert.NotContains(got, "ko.Spec.RepositoryName") +} + +func TestFieldGroupDeltaCheck_SingleField(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{ + GeneratorConfigFile: "generator-with-field-groups.yaml", + }) + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + + // PutImageScanningConfiguration has one payload field + fg := crd.UpdateFieldGroups[0] + got := code.FieldGroupDeltaCheck(crd.Config(), crd, fg, "delta") + + expected := `delta.DifferentAt("Spec.ImageScanningConfiguration")` + assert.Equal( + strings.TrimSpace(expected), + strings.TrimSpace(got), + ) +} + +func TestFieldGroupDeltaCheck_EmptyPayload(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{ + GeneratorConfigFile: "generator-with-field-groups.yaml", + }) + + crd := testutil.GetCRDByName(t, g, "Repository") + require.NotNil(crd) + + // Read operation has no payload fields (PolicyText isn't a CRD field) + fg := crd.ReadFieldGroups[0] + got := code.FieldGroupDeltaCheck(crd.Config(), crd, fg, "delta") + + assert.Equal("false", got) +} diff --git a/pkg/generate/code/set_resource_field_group.go b/pkg/generate/code/set_resource_field_group.go new file mode 100644 index 00000000..144f140d --- /dev/null +++ b/pkg/generate/code/set_resource_field_group.go @@ -0,0 +1,202 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package code + +import ( + "fmt" + "strings" + + ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/config" + "github.com/aws-controllers-k8s/code-generator/pkg/fieldpath" + "github.com/aws-controllers-k8s/code-generator/pkg/model" +) + +// SetResourceFieldGroup returns the Go code that sets a CRD's field values +// from the Output shape of a field-group operation. Only members that +// correspond to the field group's payload fields are set. +// +// This is the field-group analog of SetResource. It generates code like: +// +// if resp.ImageScanningConfiguration != nil { +// f0 := &svcapitypes.ImageScanningConfiguration{} +// ... +// ko.Spec.ImageScanningConfiguration = f0 +// } +func SetResourceFieldGroup( + cfg *ackgenconfig.Config, + r *model.CRD, + fg *model.FieldGroupOperation, + // String representing the name of the variable that we will grab the + // Output values from (typically "resp") + sourceVarName string, + // String representing the name of the variable that we will be setting + // (typically "ko") + targetVarName string, + // Number of levels of indentation to use + indentLevel int, +) (string, error) { + op := fg.Operation + if op == nil { + return "", nil + } + outputShape := op.OutputRef.Shape + if outputShape == nil { + return "", nil + } + + // For field-group operations, the payload fields are what we read back. + // Build a set of their CRD field names for filtering. + payloadFieldNames := make(map[string]bool, len(fg.PayloadFields)) + for _, f := range fg.PayloadFields { + payloadFieldNames[f.Names.Camel] = true + } + + out := "\n" + indent := strings.Repeat("\t", indentLevel) + + // Determine the opType for setter config lookup + opType := model.OpTypeUpdate + if fg.OpType == model.FieldGroupOpTypeRead { + opType = model.OpTypeGet + } + + for memberIndex, memberName := range outputShape.MemberNames() { + sourceAdaptedVarName := sourceVarName + "." + memberName + + // Resolve the CRD field name + fieldName := cfg.GetResourceFieldName( + r.Names.Original, op.ExportedName, memberName, + ) + + // Look up the field in Spec or Status + var f *model.Field + targetAdaptedVarName := targetVarName + inSpec, inStatus := r.HasMember(fieldName, op.ExportedName) + if inSpec { + f = r.SpecFields[fieldName] + targetAdaptedVarName += cfg.PrefixConfig.SpecField + } else if inStatus { + f = r.StatusFields[fieldName] + targetAdaptedVarName += cfg.PrefixConfig.StatusField + } else { + continue + } + + // Only process fields that are in this field group's payload + if !payloadFieldNames[f.Names.Camel] { + continue + } + + targetMemberShapeRef := f.ShapeRef + setCfg := f.GetSetterConfig(opType) + if setCfg != nil && setCfg.IgnoreResourceSetter() { + continue + } + + sourceMemberShapeRef := outputShape.MemberRefs[memberName] + if sourceMemberShapeRef.Shape == nil { + if setCfg != nil && setCfg.From != nil { + fp := fieldpath.FromString(*setCfg.From) + sourceMemberShapeRef = fp.ShapeRef(sourceMemberShapeRef) + } + if sourceMemberShapeRef == nil || sourceMemberShapeRef.Shape == nil { + return "", fmt.Errorf( + "resource %q, field %q: expected .Shape to not be nil for ShapeRef", + r.Names.Original, memberName, + ) + } + } + + if sourceMemberShapeRef.Shape.RealType == "union" { + sourceMemberShapeRef.Shape.Type = "union" + } + + targetMemberShape := targetMemberShapeRef.Shape + + if sourceMemberShapeRef.Shape.IsEnum() { + out += fmt.Sprintf( + "%sif %s != \"\" {\n", indent, sourceAdaptedVarName, + ) + } else if !sourceMemberShapeRef.HasDefaultValue() { + out += fmt.Sprintf( + "%sif %s != nil {\n", indent, sourceAdaptedVarName, + ) + } else { + indentLevel -= 1 + } + + qualifiedTargetVar := fmt.Sprintf( + "%s.%s", targetAdaptedVarName, f.Names.Camel, + ) + + switch targetMemberShape.Type { + case "list", "map", "structure", "union": + adaption := setResourceAdaptPrimitiveCollection( + sourceMemberShapeRef.Shape, qualifiedTargetVar, + sourceAdaptedVarName, indent, r.IsSecretField(memberName), + ) + out += adaption + if adaption != "" { + break + } + { + memberVarName := fmt.Sprintf("f%d", memberIndex) + out += varEmptyConstructorK8sType( + cfg, r, memberVarName, + targetMemberShapeRef.Shape, indentLevel+1, + ) + containerOut, err := setResourceForContainer( + cfg, r, f.Names.Camel, memberVarName, + targetMemberShapeRef, setCfg, + sourceAdaptedVarName, sourceMemberShapeRef, + f.Names.Camel, false, opType, indentLevel+1, + ) + if err != nil { + return "", err + } + out += containerOut + out += setResourceForScalar( + qualifiedTargetVar, memberVarName, + sourceMemberShapeRef, indentLevel+1, false, false, + ) + } + default: + if setCfg != nil && setCfg.From != nil { + sourceAdaptedVarName = sourceVarName + "." + *setCfg.From + } + out += setResourceForScalar( + qualifiedTargetVar, sourceAdaptedVarName, + sourceMemberShapeRef, indentLevel+1, false, false, + ) + } + if sourceMemberShapeRef.Shape.RealType == "union" { + sourceMemberShapeRef.Shape.Type = "structure" + } + if sourceMemberShapeRef.Shape.IsEnum() || !sourceMemberShapeRef.HasDefaultValue() { + out += fmt.Sprintf( + "%s} else {\n", indent, + ) + out += fmt.Sprintf( + "%s%s%s.%s = nil\n", indent, indent, + targetAdaptedVarName, f.Names.Camel, + ) + out += fmt.Sprintf( + "%s}\n", indent, + ) + } else { + indentLevel += 1 + } + } + return out, nil +} diff --git a/pkg/generate/code/set_sdk_field_group.go b/pkg/generate/code/set_sdk_field_group.go new file mode 100644 index 00000000..862efa44 --- /dev/null +++ b/pkg/generate/code/set_sdk_field_group.go @@ -0,0 +1,158 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package code + +import ( + "fmt" + "strings" + + ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/config" + "github.com/aws-controllers-k8s/code-generator/pkg/model" +) + +// SetSDKFieldGroup returns the Go code that sets an SDK Input shape's member +// fields from a CRD's fields for a specific field-group operation. Only +// members that are identifier or payload fields of the field group are set. +// +// This is the field-group analog of SetSDK. It generates code like: +// +// res.RegistryId = r.ko.Spec.RegistryID +// res.RepositoryName = r.ko.Spec.RepositoryName +// if r.ko.Spec.ImageScanningConfiguration != nil { +// f0 := &svcsdk.ImageScanningConfiguration{} +// ... +// res.ImageScanningConfiguration = f0 +// } +func SetSDKFieldGroup( + cfg *ackgenconfig.Config, + r *model.CRD, + fg *model.FieldGroupOperation, + // String representing the name of the variable that we will grab the + // Input values from (typically "r.ko") + sourceVarName string, + // String representing the name of the variable that we will be setting + // (typically "res") + targetVarName string, + // Number of levels of indentation to use + indentLevel int, +) (string, error) { + op := fg.Operation + if op == nil || op.InputRef.Shape == nil { + return "", nil + } + + inputShape := op.InputRef.Shape + + // Build a set of field names that belong to this field group (both + // identifiers and payload) for fast membership checks. + fgFieldNames := fieldGroupFieldNameSet(fg) + + out := "\n" + indent := strings.Repeat("\t", indentLevel) + + for memberIndex, memberName := range inputShape.MemberNames() { + // Resolve the CRD field name for this Input member + fieldName := cfg.GetResourceFieldName( + r.Names.Original, op.ExportedName, memberName, + ) + + // Check if this member's field is part of the field group + f, inSpec := r.SpecFields[fieldName] + if !inSpec || !fgFieldNames[f.Names.Camel] { + continue + } + + sourceAdaptedVarName := sourceVarName + cfg.PrefixConfig.SpecField + "." + f.Names.Camel + sourceFieldPath := f.Names.Camel + + setCfg := f.GetSetterConfig(model.OpTypeUpdate) + if setCfg != nil && setCfg.IgnoreSDKSetter() { + continue + } + + memberShapeRef := inputShape.MemberRefs[memberName] + memberShape := memberShapeRef.Shape + if memberShape.RealType == "union" { + memberShape.Type = "union" + } + + out += fmt.Sprintf( + "%sif %s != nil {\n", indent, sourceAdaptedVarName, + ) + + switch memberShape.Type { + case "list", "structure", "map", "union": + adaptiveCollection := setSDKAdaptiveResourceCollection( + memberShape, targetVarName, memberName, + sourceAdaptedVarName, indent, r.IsSecretField(memberName), + ) + out += adaptiveCollection + if adaptiveCollection != "" { + break + } + { + memberVarName := fmt.Sprintf("f%d", memberIndex) + out += varEmptyConstructorSDKType( + cfg, r, memberVarName, memberShape, indentLevel+1, + ) + containerOut, err := setSDKForContainer( + cfg, r, memberName, memberVarName, + sourceFieldPath, sourceAdaptedVarName, + memberShapeRef, false, model.OpTypeUpdate, indentLevel+1, + ) + if err != nil { + return "", err + } + out += containerOut + out += setSDKForScalar( + memberName, targetVarName, inputShape.Type, + sourceFieldPath, memberVarName, false, + memberShapeRef, indentLevel+1, + ) + } + default: + if r.IsSecretField(memberName) { + out += setSDKForSecret( + cfg, r, memberName, targetVarName, + sourceAdaptedVarName, indentLevel, + ) + } else { + out += setSDKForScalar( + memberName, targetVarName, inputShape.Type, + sourceFieldPath, sourceAdaptedVarName, false, + memberShapeRef, indentLevel+1, + ) + } + } + if memberShape.RealType == "union" { + memberShape.Type = "structure" + } + out += fmt.Sprintf("%s}\n", indent) + } + + return out, nil +} + +// fieldGroupFieldNameSet returns a set of CRD field names (Camel) that belong +// to the given field group operation (identifiers + payload). +func fieldGroupFieldNameSet(fg *model.FieldGroupOperation) map[string]bool { + s := make(map[string]bool, len(fg.IdentifierFields)+len(fg.PayloadFields)) + for _, f := range fg.IdentifierFields { + s[f.Names.Camel] = true + } + for _, f := range fg.PayloadFields { + s[f.Names.Camel] = true + } + return s +} diff --git a/pkg/generate/code/set_sdk_sync.go b/pkg/generate/code/set_sdk_sync.go new file mode 100644 index 00000000..17311fcc --- /dev/null +++ b/pkg/generate/code/set_sdk_sync.go @@ -0,0 +1,219 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package code + +import ( + "fmt" + "strings" + + ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/config" + "github.com/aws-controllers-k8s/code-generator/pkg/model" +) + +// SetSyncInput generates Go code that sets the SDK Input struct fields for a +// sync-list operation's add or remove call. It sets identifier fields from the +// resource and the per-item scalar field from the loop variable. +// +// Generated code looks like: +// +// input.RoleName = desired.ko.Spec.Name +// input.PolicyArn = item +func SetSyncInput( + cfg *ackgenconfig.Config, + r *model.CRD, + fg *model.FieldGroupOperation, + // Name of the resource variable (e.g., "desired") + resourceVarName string, + // Name of the SDK input variable (e.g., "input") + targetVarName string, + // Name of the loop item variable (e.g., "item") + itemVarName string, + indentLevel int, +) string { + op := fg.Operation + if op == nil || op.InputRef.Shape == nil { + return "" + } + inputShape := op.InputRef.Shape + identifierMemberNames := buildIdentifierMemberNamesForOp(cfg, r, fg, op.ExportedName) + + out := "" + indent := strings.Repeat("\t", indentLevel) + + for _, memberName := range inputShape.MemberNames() { + if identifierMemberNames[memberName] { + // Identifier field — set from the resource + fieldName := cfg.GetResourceFieldName( + r.Names.Original, op.ExportedName, memberName, + ) + f, ok := r.SpecFields[fieldName] + if !ok { + continue + } + out += fmt.Sprintf( + "%s%s.%s = %s.ko.Spec.%s\n", + indent, targetVarName, memberName, + resourceVarName, f.Names.Camel, + ) + } else { + // Per-item field — set from the loop variable + out += fmt.Sprintf( + "%s%s.%s = %s\n", + indent, targetVarName, memberName, itemVarName, + ) + } + } + return out +} + +// SetSyncMapAddInput generates Go code that sets the SDK Input struct fields +// for a sync-map operation's add/update call. It sets identifier fields from +// the resource, the map key field, and the map value field from loop variables. +// +// Generated code looks like: +// +// input.RoleName = desired.ko.Spec.Name +// input.PolicyName = &key +// input.PolicyDocument = val +func SetSyncMapAddInput( + cfg *ackgenconfig.Config, + r *model.CRD, + fg *model.FieldGroupOperation, + resourceVarName string, + targetVarName string, + keyVarName string, + valVarName string, + indentLevel int, +) string { + op := fg.Operation + if op == nil || op.InputRef.Shape == nil { + return "" + } + inputShape := op.InputRef.Shape + identifierMemberNames := buildIdentifierMemberNamesForOp(cfg, r, fg, op.ExportedName) + + out := "" + indent := strings.Repeat("\t", indentLevel) + + // Collect non-identifier members — these are the key and value fields. + // By convention, the first non-identifier string field is the key and + // the second is the value. + var nonIDMembers []string + for _, memberName := range inputShape.MemberNames() { + if !identifierMemberNames[memberName] { + nonIDMembers = append(nonIDMembers, memberName) + } + } + + for _, memberName := range inputShape.MemberNames() { + if identifierMemberNames[memberName] { + fieldName := cfg.GetResourceFieldName( + r.Names.Original, op.ExportedName, memberName, + ) + f, ok := r.SpecFields[fieldName] + if !ok { + continue + } + out += fmt.Sprintf( + "%s%s.%s = %s.ko.Spec.%s\n", + indent, targetVarName, memberName, + resourceVarName, f.Names.Camel, + ) + } else { + // Determine if this is the key or value field + if len(nonIDMembers) > 0 && memberName == nonIDMembers[0] { + out += fmt.Sprintf( + "%s%s.%s = &%s\n", + indent, targetVarName, memberName, keyVarName, + ) + } else { + out += fmt.Sprintf( + "%s%s.%s = %s\n", + indent, targetVarName, memberName, valVarName, + ) + } + } + } + return out +} + +// SetSyncMapRemoveInput generates Go code that sets the SDK Input struct fields +// for a sync-map operation's remove call. It sets identifier fields from the +// resource and the map key field from the loop variable. +// +// Generated code looks like: +// +// input.RoleName = desired.ko.Spec.Name +// input.PolicyName = &key +func SetSyncMapRemoveInput( + cfg *ackgenconfig.Config, + r *model.CRD, + fg *model.FieldGroupOperation, + resourceVarName string, + targetVarName string, + keyVarName string, + indentLevel int, +) string { + removeOp := fg.RemoveOperation + if removeOp == nil || removeOp.InputRef.Shape == nil { + return "" + } + inputShape := removeOp.InputRef.Shape + identifierMemberNames := buildIdentifierMemberNamesForOp(cfg, r, fg, removeOp.ExportedName) + + out := "" + indent := strings.Repeat("\t", indentLevel) + + for _, memberName := range inputShape.MemberNames() { + if identifierMemberNames[memberName] { + fieldName := cfg.GetResourceFieldName( + r.Names.Original, removeOp.ExportedName, memberName, + ) + f, ok := r.SpecFields[fieldName] + if !ok { + continue + } + out += fmt.Sprintf( + "%s%s.%s = %s.ko.Spec.%s\n", + indent, targetVarName, memberName, + resourceVarName, f.Names.Camel, + ) + } else { + out += fmt.Sprintf( + "%s%s.%s = &%s\n", + indent, targetVarName, memberName, keyVarName, + ) + } + } + return out +} + +// buildIdentifierMemberNamesForOp returns a set of SDK Input shape member +// names that correspond to identifier fields for the given operation. It +// reverses the rename mapping to get the original SDK member name. +func buildIdentifierMemberNamesForOp( + cfg *ackgenconfig.Config, + r *model.CRD, + fg *model.FieldGroupOperation, + opID string, +) map[string]bool { + s := make(map[string]bool, len(fg.IdentifierFields)) + for _, f := range fg.IdentifierFields { + memberName := cfg.GetOriginalMemberName( + r.Names.Original, opID, f.Names.Camel, + ) + s[memberName] = true + } + return s +} diff --git a/pkg/model/crd.go b/pkg/model/crd.go index b8967681..d3598b87 100644 --- a/pkg/model/crd.go +++ b/pkg/model/crd.go @@ -91,6 +91,12 @@ type CRD struct { // ShortNames represent the CRD list of aliases. Short names allow shorter // strings to match a CR on the CLI. ShortNames []string + // UpdateFieldGroups holds resolved field-group update operations. Each + // entry maps an SDK operation to the subset of CRD fields it manages. + UpdateFieldGroups []*FieldGroupOperation + // ReadFieldGroups holds resolved field-group read operations (supplementary + // reads called after the primary ReadOne/ReadMany). + ReadFieldGroups []*FieldGroupOperation } // Config returns a pointer to the generator config diff --git a/pkg/model/field_group.go b/pkg/model/field_group.go new file mode 100644 index 00000000..76e1d58c --- /dev/null +++ b/pkg/model/field_group.go @@ -0,0 +1,467 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package model + +import ( + "fmt" + "sort" + "strings" + + awssdkmodel "github.com/aws-controllers-k8s/code-generator/pkg/api" + "github.com/aws-controllers-k8s/pkg/names" + + ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/config" +) + +// FieldGroupOpType indicates whether a field group operation is for update or +// read. +type FieldGroupOpType int + +const ( + FieldGroupOpTypeUpdate FieldGroupOpType = iota + FieldGroupOpTypeRead +) + +// FieldGroupKind indicates how the field group operation interacts with the +// CRD field. Direct operations set the field in one API call. Sync operations +// diff desired vs observed and call add/remove operations per item. +type FieldGroupKind int + +const ( + // FieldGroupKindDirect is a standard field-group operation: the + // operation's Input shape directly accepts the CRD field value. + FieldGroupKindDirect FieldGroupKind = iota + // FieldGroupKindSyncList is a sync operation for []*string fields: + // diff desired vs observed, call OperationID per addition and + // RemoveOperation per removal. + FieldGroupKindSyncList + // FieldGroupKindSyncMap is a sync operation for map[string]*string + // fields: diff desired vs observed, call OperationID per add/update + // and RemoveOperation per removal. + FieldGroupKindSyncMap +) + +// FieldGroupOperation represents a resolved field-group operation. It ties a +// specific SDK API operation to the subset of CRD fields it manages, with +// fields partitioned into identifiers (locators) and payload (data). +type FieldGroupOperation struct { + // OpType indicates whether this is an update or read field group. + OpType FieldGroupOpType + // Kind indicates whether this is a direct operation or a sync operation. + Kind FieldGroupKind + // OperationID is the SDK operation's exported name + // (e.g., "PutImageScanningConfiguration", "AttachRolePolicy"). + OperationID string + // Names holds the various casing variants of the OperationID for use in + // generated code and hook identifiers. + Names names.Names + // Operation is the resolved SDK operation pointer. + Operation *awssdkmodel.Operation + // RemoveOperationID is the SDK operation used for removals in sync + // operations (e.g., "DetachRolePolicy"). Empty for direct operations. + RemoveOperationID string + // RemoveOperation is the resolved remove SDK operation pointer. Nil for + // direct operations. + RemoveOperation *awssdkmodel.Operation + // IdentifierFields are CRD fields shared with ReadOne/Delete Input + // shapes — these locate the resource and are always set in the API call. + IdentifierFields []*Field + // PayloadFields are the remaining CRD Spec fields from the operation's + // Input shape — these are the data fields managed by this group. + PayloadFields []*Field + // Config holds the original generator.yaml configuration for this field + // group operation (RequeueOnSuccess, Fields override, etc.). + Config ackgenconfig.FieldGroupOperationConfig +} + +// resolveFieldGroupOperations resolves all configured field-group operations +// (update and read) for a CRD, looking up SDK operations and partitioning +// Input shape members into identifier and payload fields. +// +// This must be called AFTER all CRD fields have been discovered (i.e., after +// processFields in GetCRDs), because it references crd.SpecFields. +func (r *CRD) resolveFieldGroupOperations() error { + updateCfgs := r.cfg.GetUpdateFieldGroupOperations(r.Names.Original) + for _, fgCfg := range updateCfgs { + fg, err := r.resolveOneFieldGroupOperation(FieldGroupOpTypeUpdate, fgCfg) + if err != nil { + return fmt.Errorf( + "resource %s, update_operations: %w", + r.Names.Original, err, + ) + } + r.UpdateFieldGroups = append(r.UpdateFieldGroups, fg) + } + + readCfgs := r.cfg.GetReadFieldGroupOperations(r.Names.Original) + for _, fgCfg := range readCfgs { + fg, err := r.resolveOneFieldGroupOperation(FieldGroupOpTypeRead, fgCfg) + if err != nil { + return fmt.Errorf( + "resource %s, read_operations: %w", + r.Names.Original, err, + ) + } + r.ReadFieldGroups = append(r.ReadFieldGroups, fg) + } + return nil +} + +// resolveOneFieldGroupOperation resolves a single field-group operation config +// into a FieldGroupOperation with partitioned identifier and payload fields. +// +// For update operations, both identifier and payload fields come from the +// Input shape. For read operations, identifier fields come from the Input +// shape and payload fields come from the Output shape (the data we read back). +// +// After resolving fields, the method detects whether this is a sync operation +// by checking if the payload field's CRD type is a list or map while the SDK +// Input member is scalar. If so, it infers the remove operation from naming +// conventions (Attach→Detach, Put→Delete, Add→Remove, etc). +func (r *CRD) resolveOneFieldGroupOperation( + opType FieldGroupOpType, + fgCfg ackgenconfig.FieldGroupOperationConfig, +) (*FieldGroupOperation, error) { + opID := fgCfg.OperationID + + op, found := r.sdkAPI.API.Operations[opID] + if !found { + return nil, fmt.Errorf("operation %q not found in SDK", opID) + } + + inputShape := op.InputRef.Shape + if inputShape == nil { + return nil, fmt.Errorf("operation %q has nil Input shape", opID) + } + + // Build the set of identifier member names from ReadOne and Delete Input + // shapes. These are the fields that locate the resource. + identifierMemberNames := r.buildIdentifierMemberSet() + + var identifierFields []*Field + var payloadFields []*Field + + if len(fgCfg.Fields) > 0 { + // Explicit Fields override: use those as payload + identifierFields, payloadFields = r.resolveExplicitFields( + opID, inputShape, identifierMemberNames, fgCfg.Fields, + ) + } else if opType == FieldGroupOpTypeRead { + // Read operations: identifiers from Input, payload from Output + identifierFields = r.resolveIdentifierFieldsFromInput( + opID, inputShape, identifierMemberNames, + ) + outputShape := op.OutputRef.Shape + if outputShape != nil { + payloadFields = r.resolvePayloadFieldsFromShape( + opID, outputShape, identifierMemberNames, + ) + } + } else { + // Update operations: both from Input + identifierFields, payloadFields = r.resolveAutoDetectedFields( + opID, inputShape, identifierMemberNames, + ) + } + + // Detect sync operations and resolve the remove operation. + kind := FieldGroupKindDirect + var removeOpID string + var removeOp *awssdkmodel.Operation + + if opType == FieldGroupOpTypeUpdate { + kind = r.detectFieldGroupKind(payloadFields) + if kind != FieldGroupKindDirect { + removeOpID = inferRemoveOperationID(opID) + if removeOpID == "" { + return nil, fmt.Errorf( + "operation %q: unable to infer remove operation for sync field group", + opID, + ) + } + var ok bool + removeOp, ok = r.sdkAPI.API.Operations[removeOpID] + if !ok { + return nil, fmt.Errorf( + "operation %q: inferred remove operation %q not found in SDK", + opID, removeOpID, + ) + } + } + } + + return &FieldGroupOperation{ + OpType: opType, + Kind: kind, + OperationID: opID, + Names: names.New(opID), + Operation: op, + RemoveOperationID: removeOpID, + RemoveOperation: removeOp, + IdentifierFields: identifierFields, + PayloadFields: payloadFields, + Config: fgCfg, + }, nil +} + +// buildIdentifierMemberSet returns a set of SDK member names that appear in +// the Input shapes of the ReadOne and/or Delete operations. These represent +// the "identifier" fields used to locate the resource. +func (r *CRD) buildIdentifierMemberSet() map[string]bool { + idMembers := map[string]bool{} + + addFromOp := func(op *awssdkmodel.Operation) { + if op == nil || op.InputRef.Shape == nil { + return + } + for _, memberName := range op.InputRef.Shape.MemberNames() { + idMembers[memberName] = true + } + } + + addFromOp(r.Ops.ReadOne) + addFromOp(r.Ops.Delete) + + // If neither ReadOne nor Delete exist (unusual), fall back to ReadMany + if r.Ops.ReadOne == nil && r.Ops.Delete == nil { + addFromOp(r.Ops.ReadMany) + } + + return idMembers +} + +// resolveAutoDetectedFields partitions the field-group operation's Input shape +// members into identifier and payload fields using auto-detection. +func (r *CRD) resolveAutoDetectedFields( + opID string, + inputShape *awssdkmodel.Shape, + identifierMemberNames map[string]bool, +) ([]*Field, []*Field) { + var identifierFields []*Field + var payloadFields []*Field + + for _, memberName := range inputShape.MemberNames() { + fieldName := r.cfg.GetResourceFieldName( + r.Names.Original, opID, memberName, + ) + field, ok := r.SpecFields[fieldName] + if !ok { + // Not a CRD Spec field (e.g., request metadata). Skip. + continue + } + + if identifierMemberNames[memberName] { + identifierFields = append(identifierFields, field) + } else { + payloadFields = append(payloadFields, field) + } + } + return identifierFields, payloadFields +} + +// resolveIdentifierFieldsFromInput returns CRD fields from the Input shape +// that are identifiers (shared with ReadOne/Delete). +func (r *CRD) resolveIdentifierFieldsFromInput( + opID string, + inputShape *awssdkmodel.Shape, + identifierMemberNames map[string]bool, +) []*Field { + var identifierFields []*Field + for _, memberName := range inputShape.MemberNames() { + fieldName := r.cfg.GetResourceFieldName( + r.Names.Original, opID, memberName, + ) + field, ok := r.SpecFields[fieldName] + if !ok { + continue + } + if identifierMemberNames[memberName] { + identifierFields = append(identifierFields, field) + } + } + return identifierFields +} + +// resolvePayloadFieldsFromShape returns CRD Spec fields from a shape's +// members that are NOT identifiers. Used for read operations where the +// payload comes from the Output shape. +func (r *CRD) resolvePayloadFieldsFromShape( + opID string, + shape *awssdkmodel.Shape, + identifierMemberNames map[string]bool, +) []*Field { + var payloadFields []*Field + for _, memberName := range shape.MemberNames() { + if identifierMemberNames[memberName] { + continue + } + fieldName := r.cfg.GetResourceFieldName( + r.Names.Original, opID, memberName, + ) + // Check both Spec and Status fields for reads — the output may + // map to either. + if field, ok := r.SpecFields[fieldName]; ok { + payloadFields = append(payloadFields, field) + } else if field, ok := r.StatusFields[fieldName]; ok { + payloadFields = append(payloadFields, field) + } + } + return payloadFields +} + +// resolveExplicitFields uses the explicitly configured field names to build +// the payload field list, and identifies remaining Input members that are +// identifiers. +func (r *CRD) resolveExplicitFields( + opID string, + inputShape *awssdkmodel.Shape, + identifierMemberNames map[string]bool, + explicitFields []string, +) ([]*Field, []*Field) { + // Build a set of explicit payload field names for fast lookup + explicitSet := make(map[string]bool, len(explicitFields)) + for _, fn := range explicitFields { + explicitSet[fn] = true + } + + var identifierFields []*Field + var payloadFields []*Field + + // First, resolve identifier fields from the Input shape + for _, memberName := range inputShape.MemberNames() { + fieldName := r.cfg.GetResourceFieldName( + r.Names.Original, opID, memberName, + ) + field, ok := r.SpecFields[fieldName] + if !ok { + continue + } + if identifierMemberNames[memberName] && !explicitSet[field.Names.Camel] { + identifierFields = append(identifierFields, field) + } + } + + // Then, resolve payload fields from explicit config (in order) + for _, fn := range explicitFields { + if field, ok := r.SpecFields[fn]; ok { + payloadFields = append(payloadFields, field) + } + } + + return identifierFields, payloadFields +} + +// IsDirect returns true if this is a standard direct field-group operation. +func (fg *FieldGroupOperation) IsDirect() bool { + return fg.Kind == FieldGroupKindDirect +} + +// IsSyncList returns true if this is a sync operation for a []*string field. +func (fg *FieldGroupOperation) IsSyncList() bool { + return fg.Kind == FieldGroupKindSyncList +} + +// IsSyncMap returns true if this is a sync operation for a map[string]*string field. +func (fg *FieldGroupOperation) IsSyncMap() bool { + return fg.Kind == FieldGroupKindSyncMap +} + +// IsSync returns true if this is any sync operation (list or map). +func (fg *FieldGroupOperation) IsSync() bool { + return fg.Kind == FieldGroupKindSyncList || fg.Kind == FieldGroupKindSyncMap +} + +// detectFieldGroupKind examines the payload fields to determine whether this +// is a direct operation or a sync operation. A sync operation is detected when +// there is exactly one payload field whose CRD type is a list ([]*string) or +// map (map[string]*string). +func (r *CRD) detectFieldGroupKind(payloadFields []*Field) FieldGroupKind { + if len(payloadFields) != 1 { + return FieldGroupKindDirect + } + f := payloadFields[0] + switch { + case strings.HasPrefix(f.GoType, "[]*"): + return FieldGroupKindSyncList + case strings.HasPrefix(f.GoType, "map["): + return FieldGroupKindSyncMap + } + return FieldGroupKindDirect +} + +// inferRemoveOperationID applies common AWS API naming conventions to derive +// the remove/detach operation from an add/attach/put operation name. Returns +// empty string if no convention matches. +// +// Conventions: +// - Attach* → Detach* +// - Put* → Delete* +// - Add* → Remove* +// - Associate* → Disassociate* +// - Tag* → Untag* +// - Create* → Delete* +func inferRemoveOperationID(addOpID string) string { + prefixes := []struct { + add string + remove string + }{ + {"Attach", "Detach"}, + {"Put", "Delete"}, + {"Add", "Remove"}, + {"Associate", "Disassociate"}, + {"Tag", "Untag"}, + {"Create", "Delete"}, + } + for _, p := range prefixes { + if strings.HasPrefix(addOpID, p.add) { + return p.remove + addOpID[len(p.add):] + } + } + return "" +} + +// HasFieldGroupUpdates returns true if this CRD has field-group update +// operations configured and resolved. +func (r *CRD) HasFieldGroupUpdates() bool { + return len(r.UpdateFieldGroups) > 0 +} + +// HasFieldGroupReads returns true if this CRD has field-group read +// operations configured and resolved. +func (r *CRD) HasFieldGroupReads() bool { + return len(r.ReadFieldGroups) > 0 +} + +// FieldGroupPayloadFieldNames returns a sorted, deduplicated list of all +// payload field names across all field-group operations of the given type. +func (r *CRD) FieldGroupPayloadFieldNames(opType FieldGroupOpType) []string { + seen := map[string]bool{} + var groups []*FieldGroupOperation + if opType == FieldGroupOpTypeUpdate { + groups = r.UpdateFieldGroups + } else { + groups = r.ReadFieldGroups + } + for _, fg := range groups { + for _, f := range fg.PayloadFields { + seen[f.Names.Camel] = true + } + } + names := make([]string, 0, len(seen)) + for n := range seen { + names = append(names, n) + } + sort.Strings(names) + return names +} diff --git a/pkg/model/field_group_test.go b/pkg/model/field_group_test.go new file mode 100644 index 00000000..b8d62e33 --- /dev/null +++ b/pkg/model/field_group_test.go @@ -0,0 +1,154 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package model_test + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model" + "github.com/aws-controllers-k8s/code-generator/pkg/testutil" +) + +func fieldNames(fields []*ackmodel.Field) []string { + names := make([]string, len(fields)) + for i, f := range fields { + names[i] = f.Names.Camel + } + sort.Strings(names) + return names +} + +func TestFieldGroupOperations_UpdateAutoDetect(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{ + GeneratorConfigFile: "generator-with-field-groups.yaml", + }) + + crds, err := g.GetCRDs() + require.Nil(err) + + crd := getCRDByName("Repository", crds) + require.NotNil(crd) + + // Should have 2 update field groups + assert.True(crd.HasFieldGroupUpdates()) + require.Len(crd.UpdateFieldGroups, 2) + + // PutImageScanningConfiguration + fg0 := crd.UpdateFieldGroups[0] + assert.Equal("PutImageScanningConfiguration", fg0.OperationID) + assert.Equal(ackmodel.FieldGroupOpTypeUpdate, fg0.OpType) + assert.NotNil(fg0.Operation) + assert.True(fg0.Config.RequeueOnSuccess) + + // Identifier fields: RegistryId and RepositoryName (shared with Delete) + idNames0 := fieldNames(fg0.IdentifierFields) + assert.Equal([]string{"RegistryID", "RepositoryName"}, idNames0) + + // Payload field: ImageScanningConfiguration + payloadNames0 := fieldNames(fg0.PayloadFields) + assert.Equal([]string{"ImageScanningConfiguration"}, payloadNames0) + + // PutImageTagMutability + fg1 := crd.UpdateFieldGroups[1] + assert.Equal("PutImageTagMutability", fg1.OperationID) + assert.False(fg1.Config.RequeueOnSuccess) + + idNames1 := fieldNames(fg1.IdentifierFields) + assert.Equal([]string{"RegistryID", "RepositoryName"}, idNames1) + + payloadNames1 := fieldNames(fg1.PayloadFields) + assert.Equal([]string{"ImageTagMutability"}, payloadNames1) +} + +func TestFieldGroupOperations_ReadAutoDetect(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{ + GeneratorConfigFile: "generator-with-field-groups.yaml", + }) + + crds, err := g.GetCRDs() + require.Nil(err) + + crd := getCRDByName("Repository", crds) + require.NotNil(crd) + + // Should have 1 read field group + assert.True(crd.HasFieldGroupReads()) + require.Len(crd.ReadFieldGroups, 1) + + fg := crd.ReadFieldGroups[0] + assert.Equal("GetRepositoryPolicy", fg.OperationID) + assert.Equal(ackmodel.FieldGroupOpTypeRead, fg.OpType) + assert.NotNil(fg.Operation) + + // Identifier fields from Input: RegistryId and RepositoryName + idNames := fieldNames(fg.IdentifierFields) + assert.Equal([]string{"RegistryID", "RepositoryName"}, idNames) + + // Payload fields from Output: PolicyText is not a CRD field, so + // auto-detection yields no payload. In practice, a read operation + // like this would use explicit `fields` config or have matching CRD fields. + assert.Len(fg.PayloadFields, 0) +} + +func TestFieldGroupOperations_NoConfig(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + // Default ECR generator.yaml has no field group operations + g := testutil.NewModelForService(t, "ecr") + + crds, err := g.GetCRDs() + require.Nil(err) + + crd := getCRDByName("Repository", crds) + require.NotNil(crd) + + assert.False(crd.HasFieldGroupUpdates()) + assert.False(crd.HasFieldGroupReads()) + assert.Len(crd.UpdateFieldGroups, 0) + assert.Len(crd.ReadFieldGroups, 0) +} + +func TestFieldGroupOperations_PayloadFieldNames(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + g := testutil.NewModelForServiceWithOptions(t, "ecr", &testutil.TestingModelOptions{ + GeneratorConfigFile: "generator-with-field-groups.yaml", + }) + + crds, err := g.GetCRDs() + require.Nil(err) + + crd := getCRDByName("Repository", crds) + require.NotNil(crd) + + // All update payload field names across both groups + updatePayloadNames := crd.FieldGroupPayloadFieldNames(ackmodel.FieldGroupOpTypeUpdate) + assert.Equal([]string{"ImageScanningConfiguration", "ImageTagMutability"}, updatePayloadNames) + + // Read payload has no CRD-matching fields in this test + readPayloadNames := crd.FieldGroupPayloadFieldNames(ackmodel.FieldGroupOpTypeRead) + assert.Len(readPayloadNames, 0) +} diff --git a/pkg/model/model.go b/pkg/model/model.go index d703655c..5f0801c5 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -520,6 +520,15 @@ func (m *Model) GetCRDs() ([]*CRD, error) { return nil, fmt.Errorf("generator.yaml validation failed:\n %s", strings.Join(fieldErrs, "\n ")) } + // Resolve field-group operations (update_operations, read_operations). + // This must happen after processFields so that CRD.SpecFields is fully + // populated for identifier/payload partitioning. + for _, crd := range crds { + if err := crd.resolveFieldGroupOperations(); err != nil { + return nil, err + } + } + m.crds = crds return crds, nil } diff --git a/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-field-groups.yaml b/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-field-groups.yaml new file mode 100644 index 00000000..7bcf5ce4 --- /dev/null +++ b/pkg/testdata/models/apis/ecr/0000-00-00/generator-with-field-groups.yaml @@ -0,0 +1,19 @@ +resources: + Repository: + exceptions: + errors: + 404: + code: RepositoryNotFoundException + list_operation: + match_fields: + - RepositoryName + update_operations: + - operation_id: PutImageScanningConfiguration + requeue_on_success: true + - operation_id: PutImageTagMutability + read_operations: + - operation_id: GetRepositoryPolicy +ignore: + field_paths: + - CreateRepositoryOutput.Repository.EncryptionConfiguration + - CreateRepositoryInput.EncryptionConfiguration diff --git a/templates/pkg/resource/manager.go.tpl b/templates/pkg/resource/manager.go.tpl index 3e246e11..f9045546 100644 --- a/templates/pkg/resource/manager.go.tpl +++ b/templates/pkg/resource/manager.go.tpl @@ -98,6 +98,12 @@ func (rm *resourceManager) ReadOne( } return rm.onError(r, err) } +{{- if .CRD.HasFieldGroupReads }} + observed, err = rm.sdkFindFieldGroups(ctx, r, observed) + if err != nil { + return rm.onError(observed, err) + } +{{- end }} return rm.onSuccess(observed) } diff --git a/templates/pkg/resource/sdk.go.tpl b/templates/pkg/resource/sdk.go.tpl index 30929e8f..d5c6ea6d 100644 --- a/templates/pkg/resource/sdk.go.tpl +++ b/templates/pkg/resource/sdk.go.tpl @@ -111,7 +111,11 @@ func (rm *resourceManager) sdkCreate( {{- if $hookCode := Hook .CRD "sdk_create_post_set_output" }} {{ $hookCode }} {{- end }} +{{- if .CRD.HasFieldGroupUpdates }} + return &resource{ko}, ackrequeue.NeededAfter(nil, 0) +{{- else }} return &resource{ko}, nil +{{- end }} } // newCreateRequestPayload returns an SDK-specific struct for the HTTP request @@ -127,7 +131,9 @@ func (rm *resourceManager) newCreateRequestPayload( // sdkUpdate patches the supplied resource in the backend AWS service API and // returns a new resource with updated fields. -{{ if .CRD.CustomUpdateMethodName }} +{{ if .CRD.HasFieldGroupUpdates }} + {{- template "sdk_update_field_groups" . }} +{{- else if .CRD.CustomUpdateMethodName }} {{- template "sdk_update_custom" . }} {{- else if .CRD.Ops.Update }} {{- template "sdk_update" . }} @@ -336,6 +342,10 @@ func (rm *resourceManager) terminalAWSError(err error) bool { {{- end }} } +{{- if .CRD.HasFieldGroupReads }} +{{ template "sdk_find_field_groups" . }} +{{- end }} + {{- if $hookCode := Hook .CRD "sdk_file_end" }} {{ $hookCode }} {{- end }} diff --git a/templates/pkg/resource/sdk_find_field_groups.go.tpl b/templates/pkg/resource/sdk_find_field_groups.go.tpl new file mode 100644 index 00000000..5346ec1e --- /dev/null +++ b/templates/pkg/resource/sdk_find_field_groups.go.tpl @@ -0,0 +1,65 @@ +{{- define "sdk_find_field_groups" -}} +func (rm *resourceManager) sdkFindFieldGroups( + ctx context.Context, + r *resource, + observed *resource, +) (*resource, error) { + ko := observed.ko.DeepCopy() + var err error +{{ range $fg := .CRD.ReadFieldGroups }} + ko, err = rm.sdkFind{{ $fg.OperationID }}(ctx, r, ko) + if err != nil { + return &resource{ko}, err + } +{{ end }} + return &resource{ko}, nil +} +{{ range $fg := .CRD.ReadFieldGroups }} +func (rm *resourceManager) sdkFind{{ $fg.OperationID }}( + ctx context.Context, + r *resource, + ko *svcapitypes.{{ $.CRD.Names.Camel }}, +) (*svcapitypes.{{ $.CRD.Names.Camel }}, error) { +{{- if $hookCode := Hook $.CRD (print "sdk_read_" $fg.Names.Snake "_pre_build_request") }} +{{ $hookCode }} +{{- end }} + input, err := rm.new{{ $fg.OperationID }}Input(r) + if err != nil { + return ko, err + } +{{- if $hookCode := Hook $.CRD (print "sdk_read_" $fg.Names.Snake "_post_build_request") }} +{{ $hookCode }} +{{- end }} + + var resp {{ $.CRD.GetOutputShapeGoType $fg.Operation }}; _ = resp + resp, err = rm.sdkapi.{{ $fg.OperationID }}(ctx, input) +{{- if $hookCode := Hook $.CRD (print "sdk_read_" $fg.Names.Snake "_post_request") }} +{{ $hookCode }} +{{- end }} + rm.metrics.RecordAPICall("READ_ONE", "{{ $fg.OperationID }}", err) + if err != nil { + var awsErr smithy.APIError + if errors.As(err, &awsErr) && awsErr.ErrorCode() == "{{ ResourceExceptionCode $.CRD 404 }}" { + return ko, nil + } + return ko, err + } +{{- if $hookCode := Hook $.CRD (print "sdk_read_" $fg.Names.Snake "_pre_set_output") }} +{{ $hookCode }} +{{- end }} +{{ GoCodeSetFieldGroupOutput $.CRD $fg "resp" "ko" 1 }} +{{- if $hookCode := Hook $.CRD (print "sdk_read_" $fg.Names.Snake "_post_set_output") }} +{{ $hookCode }} +{{- end }} + return ko, nil +} + +func (rm *resourceManager) new{{ $fg.OperationID }}Input( + r *resource, +) (*svcsdk.{{ $fg.Operation.InputRef.Shape.ShapeName }}, error) { + res := &svcsdk.{{ $fg.Operation.InputRef.Shape.ShapeName }}{} +{{ GoCodeSetFieldGroupInput $.CRD $fg "r.ko" "res" 1 }} + return res, nil +} +{{ end }} +{{- end -}} diff --git a/templates/pkg/resource/sdk_update_field_groups.go.tpl b/templates/pkg/resource/sdk_update_field_groups.go.tpl new file mode 100644 index 00000000..9ca3361e --- /dev/null +++ b/templates/pkg/resource/sdk_update_field_groups.go.tpl @@ -0,0 +1,170 @@ +{{- define "sdk_update_field_groups" -}} +func (rm *resourceManager) sdkUpdate( + ctx context.Context, + desired *resource, + latest *resource, + delta *ackcompare.Delta, +) (updated *resource, err error) { + rlog := ackrtlog.FromContext(ctx) + exit := rlog.Trace("rm.sdkUpdate") + defer func() { + exit(err) + }() +{{- if $hookCode := Hook .CRD "sdk_update_pre_build_request" }} +{{ $hookCode }} +{{- end }} + + ko := desired.ko.DeepCopy() + rm.setStatusDefaults(ko) + + var requeueNeeded bool +{{ range $fg := .CRD.UpdateFieldGroups }} + if {{ GoCodeFieldGroupDeltaCheck $.CRD $fg "delta" }} { +{{- if $fg.IsSync }} + err = rm.sync{{ $fg.OperationID }}(ctx, desired, latest) +{{- else }} + ko, err = rm.sdkUpdate{{ $fg.OperationID }}(ctx, desired, ko) +{{- end }} + if err != nil { + return nil, err + } +{{- if $fg.Config.RequeueOnSuccess }} + requeueNeeded = true +{{- end }} + } +{{ end }} +{{- if $hookCode := Hook .CRD "sdk_update_post_set_output" }} +{{ $hookCode }} +{{- end }} + if requeueNeeded { + return &resource{ko}, ackrequeue.NeededAfter(nil, 0) + } + return &resource{ko}, nil +} +{{ range $fg := .CRD.UpdateFieldGroups }} +{{- if $fg.IsSyncList }} +// sync{{ $fg.OperationID }} examines the desired and latest values of the +// {{ (index $fg.PayloadFields 0).Names.Camel }} field and calls the {{ $fg.OperationID }} and +// {{ $fg.RemoveOperationID }} APIs to bring the observed state in line with desired. +func (rm *resourceManager) sync{{ $fg.OperationID }}( + ctx context.Context, + desired *resource, + latest *resource, +) (err error) { + rlog := ackrtlog.FromContext(ctx) + exit := rlog.Trace("rm.sync{{ $fg.OperationID }}") + defer func() { exit(err) }() + + toAdd, toRemove := ackcompare.SliceStringPDifference( + desired.ko.Spec.{{ (index $fg.PayloadFields 0).Names.Camel }}, + latest.ko.Spec.{{ (index $fg.PayloadFields 0).Names.Camel }}, + ) + + for _, item := range toAdd { + rlog.Debug("adding item via {{ $fg.OperationID }}", "item", *item) + input := &svcsdk.{{ $fg.Operation.InputRef.Shape.ShapeName }}{} +{{ GoCodeSetSyncInput $.CRD $fg "desired" "input" "item" 2 }} + _, err = rm.sdkapi.{{ $fg.OperationID }}(ctx, input) + rm.metrics.RecordAPICall("UPDATE", "{{ $fg.OperationID }}", err) + if err != nil { + return err + } + } + for _, item := range toRemove { + rlog.Debug("removing item via {{ $fg.RemoveOperationID }}", "item", *item) + input := &svcsdk.{{ $fg.RemoveOperation.InputRef.Shape.ShapeName }}{} +{{ GoCodeSetSyncInput $.CRD $fg "desired" "input" "item" 2 }} + _, err = rm.sdkapi.{{ $fg.RemoveOperationID }}(ctx, input) + rm.metrics.RecordAPICall("UPDATE", "{{ $fg.RemoveOperationID }}", err) + if err != nil { + return err + } + } + return nil +} +{{- else if $fg.IsSyncMap }} +// sync{{ $fg.OperationID }} examines the desired and latest values of the +// {{ (index $fg.PayloadFields 0).Names.Camel }} field and calls the {{ $fg.OperationID }} and +// {{ $fg.RemoveOperationID }} APIs to bring the observed state in line with desired. +func (rm *resourceManager) sync{{ $fg.OperationID }}( + ctx context.Context, + desired *resource, + latest *resource, +) (err error) { + rlog := ackrtlog.FromContext(ctx) + exit := rlog.Trace("rm.sync{{ $fg.OperationID }}") + defer func() { exit(err) }() + + toAddOrUpdate, toRemove := ackcompare.MapStringStringPDifference( + desired.ko.Spec.{{ (index $fg.PayloadFields 0).Names.Camel }}, + latest.ko.Spec.{{ (index $fg.PayloadFields 0).Names.Camel }}, + ) + + for key, val := range toAddOrUpdate { + rlog.Debug("adding/updating item via {{ $fg.OperationID }}", "key", key) + input := &svcsdk.{{ $fg.Operation.InputRef.Shape.ShapeName }}{} +{{ GoCodeSetSyncMapAddInput $.CRD $fg "desired" "input" "key" "val" 2 }} + _, err = rm.sdkapi.{{ $fg.OperationID }}(ctx, input) + rm.metrics.RecordAPICall("UPDATE", "{{ $fg.OperationID }}", err) + if err != nil { + return err + } + } + for _, key := range toRemove { + rlog.Debug("removing item via {{ $fg.RemoveOperationID }}", "key", key) + input := &svcsdk.{{ $fg.RemoveOperation.InputRef.Shape.ShapeName }}{} +{{ GoCodeSetSyncMapRemoveInput $.CRD $fg "desired" "input" "key" 2 }} + _, err = rm.sdkapi.{{ $fg.RemoveOperationID }}(ctx, input) + rm.metrics.RecordAPICall("UPDATE", "{{ $fg.RemoveOperationID }}", err) + if err != nil { + return err + } + } + return nil +} +{{- else }} +func (rm *resourceManager) sdkUpdate{{ $fg.OperationID }}( + ctx context.Context, + desired *resource, + ko *svcapitypes.{{ $.CRD.Names.Camel }}, +) (*svcapitypes.{{ $.CRD.Names.Camel }}, error) { +{{- if $hookCode := Hook $.CRD (print "sdk_update_" $fg.Names.Snake "_pre_build_request") }} +{{ $hookCode }} +{{- end }} + input, err := rm.new{{ $fg.OperationID }}Payload(desired) + if err != nil { + return ko, err + } +{{- if $hookCode := Hook $.CRD (print "sdk_update_" $fg.Names.Snake "_post_build_request") }} +{{ $hookCode }} +{{- end }} + + var resp {{ $.CRD.GetOutputShapeGoType $fg.Operation }}; _ = resp + resp, err = rm.sdkapi.{{ $fg.OperationID }}(ctx, input) +{{- if $hookCode := Hook $.CRD (print "sdk_update_" $fg.Names.Snake "_post_request") }} +{{ $hookCode }} +{{- end }} + rm.metrics.RecordAPICall("UPDATE", "{{ $fg.OperationID }}", err) + if err != nil { + return ko, err + } +{{- if $hookCode := Hook $.CRD (print "sdk_update_" $fg.Names.Snake "_pre_set_output") }} +{{ $hookCode }} +{{- end }} +{{ GoCodeSetFieldGroupOutput $.CRD $fg "resp" "ko" 1 }} +{{- if $hookCode := Hook $.CRD (print "sdk_update_" $fg.Names.Snake "_post_set_output") }} +{{ $hookCode }} +{{- end }} + return ko, nil +} + +func (rm *resourceManager) new{{ $fg.OperationID }}Payload( + r *resource, +) (*svcsdk.{{ $fg.Operation.InputRef.Shape.ShapeName }}, error) { + res := &svcsdk.{{ $fg.Operation.InputRef.Shape.ShapeName }}{} +{{ GoCodeSetFieldGroupInput $.CRD $fg "r.ko" "res" 1 }} + return res, nil +} +{{- end }} +{{ end }} +{{- end -}} From 76fc0a75579ce2d50e67e5265a4c4cd9cb1cd1bb Mon Sep 17 00:00:00 2001 From: michaelhtm <98621731+michaelhtm@users.noreply.github.com> Date: Fri, 22 May 2026 12:42:46 -0700 Subject: [PATCH 2/2] [WIP] generate e2e tests in go --- Makefile | 6 +- cmd/ack-generate/command/e2e_tests.go | 121 +++ pkg/api/api_test.go | 1 - pkg/api/docstring.go | 1 - pkg/api/docstring_test.go | 2 - pkg/api/endpoint_trait.go | 1 - pkg/api/eventstream.go | 1 - pkg/api/eventstream_tmpl.go | 2 - pkg/api/eventstream_tmpl_reader.go | 2 - pkg/api/eventstream_tmpl_readertests.go | 1 - pkg/api/eventstream_tmpl_tests.go | 2 - pkg/api/eventstream_tmpl_writer.go | 2 - pkg/api/eventstream_tmpl_writertests.go | 1 - pkg/api/example.go | 2 - pkg/api/example_test.go | 2 - pkg/api/examples_builder.go | 2 - pkg/api/examples_builder_customizations.go | 1 - pkg/api/exportable_name.go | 1 - pkg/api/legacy_io_suffix.go | 2 +- pkg/api/legacy_jsonvalue.go | 1 - pkg/api/logger.go | 1 - pkg/api/operation.go | 9 +- pkg/api/pagination.go | 1 - pkg/api/param_filler.go | 2 - pkg/api/passes.go | 1 - pkg/api/passes_test.go | 2 +- pkg/api/s3manger_input.go | 1 - pkg/api/service_name.go | 2 - pkg/api/shape_validation.go | 2 - pkg/api/shape_value_builder.go | 2 - pkg/api/smoke.go | 1 - pkg/api/waiters.go | 1 - pkg/config/field_group.go | 16 + pkg/config/testconfig.go | 99 ++ pkg/config/validate_test.go | 8 +- pkg/generate/ack/e2e_tests.go | 927 ++++++++++++++++++ pkg/generate/code/set_resource_field_group.go | 161 +++ pkg/generate/code/tags.go | 54 +- pkg/generate/code/tags_test.go | 2 +- pkg/model/crd.go | 5 + pkg/model/field_group.go | 13 + pkg/sdk/fetch.go | 2 +- pkg/sdk/repo.go | 2 +- scripts/build-e2e-tests.sh | 135 +++ .../pkg/resource/sdk_find_field_groups.go.tpl | 6 + templates/test/e2e/Makefile.tpl | 80 ++ templates/test/e2e/main_test.go.tpl | 74 ++ templates/test/e2e/resource_test.go.tpl | 142 +++ 48 files changed, 1822 insertions(+), 83 deletions(-) create mode 100644 cmd/ack-generate/command/e2e_tests.go create mode 100644 pkg/config/testconfig.go create mode 100644 pkg/generate/ack/e2e_tests.go create mode 100755 scripts/build-e2e-tests.sh create mode 100644 templates/test/e2e/Makefile.tpl create mode 100644 templates/test/e2e/main_test.go.tpl create mode 100644 templates/test/e2e/resource_test.go.tpl diff --git a/Makefile b/Makefile index fa4e3a17..1c44384d 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ GO_CMD_FLAGS=-tags codegen .PHONY: all build-ack-generate test \ build-controller build-controller-image \ local-build-controller-image lint-shell \ - check-crd-compatibility + check-crd-compatibility build-e2e-tests all: test @@ -51,6 +51,10 @@ CRD_PATHS ?= config/crd/bases,helm/crds check-crd-compatibility: build-ack-generate ## Check CRDs for breaking changes against BASE_REF @bin/ack-generate crd-compat-check --base-ref=$(BASE_REF) --crd-paths=$(CRD_PATHS) +build-e2e-tests: build-ack-generate ## Generate e2e test scaffolds for SERVICE + @echo "==== generating e2e tests for $(AWS_SERVICE)-controller ====" + @./scripts/build-e2e-tests.sh $(AWS_SERVICE) + test: ## Run code tests go test ${GO_CMD_FLAGS} ./... diff --git a/cmd/ack-generate/command/e2e_tests.go b/cmd/ack-generate/command/e2e_tests.go new file mode 100644 index 00000000..451b4284 --- /dev/null +++ b/cmd/ack-generate/command/e2e_tests.go @@ -0,0 +1,121 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package command + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "strings" + "time" + + "github.com/spf13/cobra" + + ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/config" + ackgenerate "github.com/aws-controllers-k8s/code-generator/pkg/generate/ack" + ackmetadata "github.com/aws-controllers-k8s/code-generator/pkg/metadata" + "github.com/aws-controllers-k8s/code-generator/pkg/sdk" + "github.com/aws-controllers-k8s/code-generator/pkg/util" +) + +var ( + optTestConfigPath string +) + +var e2eTestsCmd = &cobra.Command{ + Use: "e2e-tests ", + Short: "Generates Go e2e test files for an AWS service controller", + Long: `Generates Go e2e test scaffolds that exercise the CRUD lifecycle +for each resource defined in the controller's testconfig.yaml file. + +The generated tests use the shared test library from test-infra/pkg/e2e/ +and follow the create → wait-synced → update → delete pattern.`, + RunE: generateE2ETests, +} + +func init() { + e2eTestsCmd.PersistentFlags().StringVar( + &optTestConfigPath, "test-config", "", + "Path to testconfig.yaml file. Defaults to /testconfig.yaml", + ) + rootCmd.AddCommand(e2eTestsCmd) +} + +func generateE2ETests(cmd *cobra.Command, args []string) error { + cmdStart := time.Now() + if len(args) != 1 { + return fmt.Errorf("please specify the service alias for the AWS service API to generate") + } + svcAlias := strings.ToLower(args[0]) + if optOutputPath == "" { + optOutputPath = filepath.Join(optServicesDir, svcAlias) + } + + cfg, err := setupGenerator(svcAlias) + if err != nil { + return err + } + + // Load testconfig.yaml + testConfigPath := optTestConfigPath + if testConfigPath == "" { + testConfigPath = filepath.Join(optOutputPath, "testconfig.yaml") + } + testCfg, err := ackgenconfig.NewTestConfig(testConfigPath) + if err != nil { + return fmt.Errorf("loading test config: %w", err) + } + if testCfg == nil { + return fmt.Errorf("testconfig.yaml not found at %s — create it to define test values for resources", testConfigPath) + } + + // Load the AWS SDK model + metadata, err := ackmetadata.NewServiceMetadata(optMetadataConfigPath) + if err != nil { + return err + } + m, err := loadModelWithLatestAPIVersion(svcAlias, metadata, cfg) + if err != nil { + return err + } + + // Generate test templates + ts, err := ackgenerate.E2ETests(m, optTemplateDirs, testCfg) + if err != nil { + return err + } + + if err = ts.Execute(); err != nil { + return err + } + + for path, contents := range ts.Executed() { + if optDryRun { + fmt.Printf("============================= %s ======================================\n", path) + fmt.Println(strings.TrimSpace(contents.String())) + continue + } + outPath := filepath.Join(optOutputPath, path) + outDir := filepath.Dir(outPath) + if _, err := sdk.EnsureDir(outDir); err != nil { + return err + } + if err = ioutil.WriteFile(outPath, contents.Bytes(), 0666); err != nil { + return err + } + } + + util.Tracef("generateE2ETests total: %s\n", time.Since(cmdStart)) + return nil +} diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 87abbae2..cf6e434b 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -1,4 +1,3 @@ - package api import ( diff --git a/pkg/api/docstring.go b/pkg/api/docstring.go index ccabb7ec..4bcce272 100644 --- a/pkg/api/docstring.go +++ b/pkg/api/docstring.go @@ -1,4 +1,3 @@ - package api import ( diff --git a/pkg/api/docstring_test.go b/pkg/api/docstring_test.go index b80ecc5b..acd6972e 100644 --- a/pkg/api/docstring_test.go +++ b/pkg/api/docstring_test.go @@ -1,5 +1,3 @@ - - package api import ( diff --git a/pkg/api/endpoint_trait.go b/pkg/api/endpoint_trait.go index 97fe4729..8dd73755 100644 --- a/pkg/api/endpoint_trait.go +++ b/pkg/api/endpoint_trait.go @@ -1,4 +1,3 @@ - package api import ( diff --git a/pkg/api/eventstream.go b/pkg/api/eventstream.go index ff440afa..f2377d2b 100644 --- a/pkg/api/eventstream.go +++ b/pkg/api/eventstream.go @@ -1,4 +1,3 @@ - package api import ( diff --git a/pkg/api/eventstream_tmpl.go b/pkg/api/eventstream_tmpl.go index e49c7b2f..98e9f89b 100644 --- a/pkg/api/eventstream_tmpl.go +++ b/pkg/api/eventstream_tmpl.go @@ -1,5 +1,3 @@ - - package api import ( diff --git a/pkg/api/eventstream_tmpl_reader.go b/pkg/api/eventstream_tmpl_reader.go index 37cb1ee6..ef0f6e56 100644 --- a/pkg/api/eventstream_tmpl_reader.go +++ b/pkg/api/eventstream_tmpl_reader.go @@ -1,5 +1,3 @@ - - package api import "text/template" diff --git a/pkg/api/eventstream_tmpl_readertests.go b/pkg/api/eventstream_tmpl_readertests.go index 3a72852d..8b8e644d 100644 --- a/pkg/api/eventstream_tmpl_readertests.go +++ b/pkg/api/eventstream_tmpl_readertests.go @@ -1,4 +1,3 @@ - package api import ( diff --git a/pkg/api/eventstream_tmpl_tests.go b/pkg/api/eventstream_tmpl_tests.go index 87e80aa7..43456de1 100644 --- a/pkg/api/eventstream_tmpl_tests.go +++ b/pkg/api/eventstream_tmpl_tests.go @@ -1,5 +1,3 @@ - - package api import ( diff --git a/pkg/api/eventstream_tmpl_writer.go b/pkg/api/eventstream_tmpl_writer.go index 520ff2c3..d1267dc4 100644 --- a/pkg/api/eventstream_tmpl_writer.go +++ b/pkg/api/eventstream_tmpl_writer.go @@ -1,5 +1,3 @@ - - package api import "text/template" diff --git a/pkg/api/eventstream_tmpl_writertests.go b/pkg/api/eventstream_tmpl_writertests.go index ab5be283..53e9efe7 100644 --- a/pkg/api/eventstream_tmpl_writertests.go +++ b/pkg/api/eventstream_tmpl_writertests.go @@ -1,4 +1,3 @@ - package api import ( diff --git a/pkg/api/example.go b/pkg/api/example.go index 0bd25f76..bffcbba8 100644 --- a/pkg/api/example.go +++ b/pkg/api/example.go @@ -1,5 +1,3 @@ - - package api import ( diff --git a/pkg/api/example_test.go b/pkg/api/example_test.go index 061493cf..1aed1dd0 100644 --- a/pkg/api/example_test.go +++ b/pkg/api/example_test.go @@ -1,5 +1,3 @@ - - package api import ( diff --git a/pkg/api/examples_builder.go b/pkg/api/examples_builder.go index 4eee7b91..1f868a5c 100644 --- a/pkg/api/examples_builder.go +++ b/pkg/api/examples_builder.go @@ -1,5 +1,3 @@ - - package api import ( diff --git a/pkg/api/examples_builder_customizations.go b/pkg/api/examples_builder_customizations.go index 3142f891..28739bb1 100644 --- a/pkg/api/examples_builder_customizations.go +++ b/pkg/api/examples_builder_customizations.go @@ -1,4 +1,3 @@ - package api type wafregionalExamplesBuilder struct { diff --git a/pkg/api/exportable_name.go b/pkg/api/exportable_name.go index c69bc99a..efb1356d 100644 --- a/pkg/api/exportable_name.go +++ b/pkg/api/exportable_name.go @@ -1,4 +1,3 @@ - package api import "strings" diff --git a/pkg/api/legacy_io_suffix.go b/pkg/api/legacy_io_suffix.go index b123a501..6c288349 100644 --- a/pkg/api/legacy_io_suffix.go +++ b/pkg/api/legacy_io_suffix.go @@ -187,7 +187,7 @@ var legacyIOSuffixed = IoSuffix{ "DatabaseInput": struct{}{}, "PartitionInput": struct{}{}, "ConnectionInput": struct{}{}, - "DQTransformOutput": struct{}{}, + "DQTransformOutput": struct{}{}, }, "Glacier": { diff --git a/pkg/api/legacy_jsonvalue.go b/pkg/api/legacy_jsonvalue.go index 7f85f9af..ae5a442e 100644 --- a/pkg/api/legacy_jsonvalue.go +++ b/pkg/api/legacy_jsonvalue.go @@ -1,4 +1,3 @@ - package api type legacyJSONValues struct { diff --git a/pkg/api/logger.go b/pkg/api/logger.go index 2f219543..c0e9c429 100644 --- a/pkg/api/logger.go +++ b/pkg/api/logger.go @@ -1,4 +1,3 @@ - package api import ( diff --git a/pkg/api/operation.go b/pkg/api/operation.go index 15515c4f..9a3afaa4 100644 --- a/pkg/api/operation.go +++ b/pkg/api/operation.go @@ -1,4 +1,3 @@ - package api import ( @@ -12,10 +11,10 @@ import ( // An Operation defines a specific API Operation. type Operation struct { - API *API `json:"-"` - ExportedName string - Name string - Documentation string `json:"-"` + API *API `json:"-"` + ExportedName string + Name string + Documentation string `json:"-"` // HTTP HTTPInfo Host string `json:"host"` InputRef ShapeRef `json:"input"` diff --git a/pkg/api/pagination.go b/pkg/api/pagination.go index 0f13d835..cd55c7a7 100644 --- a/pkg/api/pagination.go +++ b/pkg/api/pagination.go @@ -1,4 +1,3 @@ - package api import ( diff --git a/pkg/api/param_filler.go b/pkg/api/param_filler.go index 8fb122ef..7134b03d 100644 --- a/pkg/api/param_filler.go +++ b/pkg/api/param_filler.go @@ -1,5 +1,3 @@ - - package api import ( diff --git a/pkg/api/passes.go b/pkg/api/passes.go index 2a1c435c..d76939bf 100644 --- a/pkg/api/passes.go +++ b/pkg/api/passes.go @@ -538,7 +538,6 @@ func createAPIParamShape(a *API, opName string, ref *ShapeRef, shapeName string, return } - if s, ok := a.Shapes[shapeName]; ok { panic(fmt.Sprintf( "attempting to create duplicate API parameter shape, %v, %v, %v, %v\n", diff --git a/pkg/api/passes_test.go b/pkg/api/passes_test.go index 0fdfd79a..0bbeffda 100644 --- a/pkg/api/passes_test.go +++ b/pkg/api/passes_test.go @@ -1061,4 +1061,4 @@ func TestRenamedUnionShapePreservesOriginalName(t *testing.T) { if unionShape.OriginalShapeName != "MemoryStrategyInput" { t.Fatalf("expected OriginalShapeName %q after rename, got %q", "MemoryStrategyInput", unionShape.OriginalShapeName) } -} \ No newline at end of file +} diff --git a/pkg/api/s3manger_input.go b/pkg/api/s3manger_input.go index 64295b84..69e0e7f5 100644 --- a/pkg/api/s3manger_input.go +++ b/pkg/api/s3manger_input.go @@ -1,4 +1,3 @@ - package api import ( diff --git a/pkg/api/service_name.go b/pkg/api/service_name.go index 40653916..69fd3040 100644 --- a/pkg/api/service_name.go +++ b/pkg/api/service_name.go @@ -1,5 +1,3 @@ - - package api // ServiceName returns the SDK's naming of the service. Has diff --git a/pkg/api/shape_validation.go b/pkg/api/shape_validation.go index 929b41c3..c1a2a02c 100644 --- a/pkg/api/shape_validation.go +++ b/pkg/api/shape_validation.go @@ -1,5 +1,3 @@ - - package api import ( diff --git a/pkg/api/shape_value_builder.go b/pkg/api/shape_value_builder.go index 8f33a34d..904c1758 100644 --- a/pkg/api/shape_value_builder.go +++ b/pkg/api/shape_value_builder.go @@ -1,5 +1,3 @@ - - package api import ( diff --git a/pkg/api/smoke.go b/pkg/api/smoke.go index a801b514..6965064a 100644 --- a/pkg/api/smoke.go +++ b/pkg/api/smoke.go @@ -1,4 +1,3 @@ - package api import ( diff --git a/pkg/api/waiters.go b/pkg/api/waiters.go index b3e9b453..538b2ddd 100644 --- a/pkg/api/waiters.go +++ b/pkg/api/waiters.go @@ -1,4 +1,3 @@ - package api import ( diff --git a/pkg/config/field_group.go b/pkg/config/field_group.go index 6d3ddd5f..6481112c 100644 --- a/pkg/config/field_group.go +++ b/pkg/config/field_group.go @@ -48,4 +48,20 @@ type FieldGroupOperationConfig struct { // does not contain the updated field values, requiring a subsequent // ReadOne to refresh state. RequeueOnSuccess bool `json:"requeue_on_success,omitempty"` + // Exceptions configures per-operation error handling. Currently + // supports a 404 error code mapping, identical in meaning to the + // resource-level exceptions.errors.404.code: when the operation + // returns this error code, treat it as "no data" rather than a + // failure (set the field to nil and continue). + Exceptions *FieldGroupExceptionsConfig `json:"exceptions,omitempty"` +} + +// FieldGroupExceptionsConfig holds per-operation exception mappings. +type FieldGroupExceptionsConfig struct { + Errors map[int]FieldGroupErrorConfig `json:"errors,omitempty"` +} + +// FieldGroupErrorConfig maps an HTTP status to an AWS error code. +type FieldGroupErrorConfig struct { + Code string `json:"code"` } diff --git a/pkg/config/testconfig.go b/pkg/config/testconfig.go new file mode 100644 index 00000000..aaf82c17 --- /dev/null +++ b/pkg/config/testconfig.go @@ -0,0 +1,99 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package config + +import ( + "fmt" + "os" + + "sigs.k8s.io/yaml" +) + +// TestConfig represents the testconfig.yaml file that provides domain-specific +// test values for e2e test generation. +type TestConfig struct { + // Service is the ACK service alias (e.g., "rds", "s3"). + Service string `json:"service"` + // Resources maps CRD Kind names to a map of named test configurations. + // The outer key is the CRD Kind (e.g., "Bucket"), the inner key is the + // test name (e.g., "versioning_lifecycle") which becomes the test function + // suffix. + Resources map[string]map[string]TestResourceConfig `json:"resources"` + // Bootstrap lists shared AWS resources to create before tests and destroy after. + Bootstrap []BootstrapResourceConfig `json:"bootstrap,omitempty"` +} + +// TestResourceConfig provides test values and configuration for a single CRD resource. +type TestResourceConfig struct { + // CreateValues maps Go struct field names (PascalCase) to values used when + // creating the resource in tests. + CreateValues map[string]interface{} `json:"create_values,omitempty"` + // UpdateValues maps Go struct field names to new values used in update tests. + UpdateValues map[string]interface{} `json:"update_values,omitempty"` + // CreateWait is the timeout in seconds to wait for the resource to become synced + // after creation. Defaults to 60. + CreateWait int `json:"create_wait,omitempty"` + // DeleteWait is the timeout in seconds to wait for deletion. Defaults to 60. + DeleteWait int `json:"delete_wait,omitempty"` + // Skip when true, no test is generated for this resource. + Skip bool `json:"skip,omitempty"` + // BootstrapFields maps Go struct field names to bootstrap resource field paths + // (e.g., "SharedVPC.SecurityGroupID"). At runtime, the value is retrieved from + // the named bootstrap resource. + BootstrapFields map[string]string `json:"bootstrap_fields,omitempty"` +} + +// BootstrapResourceConfig describes a shared AWS resource to be bootstrapped. +type BootstrapResourceConfig struct { + // Type is the bootstrap resource type (e.g., "VPC", "IAMRole", "S3Bucket"). + Type string `json:"type"` + // Name is the logical name used to reference this resource in bootstrap_fields. + Name string `json:"name"` + // DependsOn lists logical names of other bootstrap resources that must be + // created before this one. + DependsOn []string `json:"depends_on,omitempty"` +} + +// GetCreateWait returns the create wait timeout in seconds, defaulting to 60. +func (r TestResourceConfig) GetCreateWait() int { + if r.CreateWait > 0 { + return r.CreateWait + } + return 60 +} + +// GetDeleteWait returns the delete wait timeout in seconds, defaulting to 60. +func (r TestResourceConfig) GetDeleteWait() int { + if r.DeleteWait > 0 { + return r.DeleteWait + } + return 60 +} + +// NewTestConfig reads and parses a testconfig.yaml file at the given path. +// Returns nil (not an error) if the file does not exist. +func NewTestConfig(path string) (*TestConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("reading test config %s: %w", path, err) + } + tc := &TestConfig{} + if err := yaml.Unmarshal(data, tc); err != nil { + return nil, fmt.Errorf("parsing test config %s: %w", path, err) + } + return tc, nil +} diff --git a/pkg/config/validate_test.go b/pkg/config/validate_test.go index ef95d341..98f491ae 100644 --- a/pkg/config/validate_test.go +++ b/pkg/config/validate_test.go @@ -240,10 +240,10 @@ func TestValidateConfig_ErrorMessageIncludesAvailable(t *testing.T) { func TestValidateFieldGroupOperations_ValidUpdateOps(t *testing.T) { sdkOps := map[string]struct{}{ - "CreateRepository": {}, - "DeleteRepository": {}, - "PutImageScanningConfiguration": {}, - "PutImageTagMutability": {}, + "CreateRepository": {}, + "DeleteRepository": {}, + "PutImageScanningConfiguration": {}, + "PutImageTagMutability": {}, } cfg := &Config{ diff --git a/pkg/generate/ack/e2e_tests.go b/pkg/generate/ack/e2e_tests.go new file mode 100644 index 00000000..73530395 --- /dev/null +++ b/pkg/generate/ack/e2e_tests.go @@ -0,0 +1,927 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package ack + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + ttpl "text/template" + + "github.com/aws-controllers-k8s/pkg/names" + + awssdkmodel "github.com/aws-controllers-k8s/code-generator/pkg/api" + ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/config" + "github.com/aws-controllers-k8s/code-generator/pkg/generate/templateset" + ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model" +) + +var ( + e2eIncludePaths = []string{} + e2eCopyPaths = []string{} + e2eFuncMap = ttpl.FuncMap{ + "ToLower": strings.ToLower, + "TruncateTo": func(maxLen int, s string) string { + if len(s) > maxLen { + return s[:maxLen] + } + return s + }, + } +) + +// VerifyCall represents a single AWS SDK verification call to make after +// create or update. +type VerifyCall struct { + // OperationName is the SDK operation (e.g., "GetBucketVersioning") + OperationName string + // InputTypeName is the SDK input struct (e.g., "GetBucketVersioningInput") + InputTypeName string + // IdentifierAssignments maps SDK input struct field names to Go expressions + // (e.g., "Bucket": "aws.String(name)") + IdentifierAssignments map[string]string + // Assertions are the field-level assertions on the output + Assertions []VerifyAssertion +} + +// VerifyAssertion represents a single assertion on a verification response. +type VerifyAssertion struct { + // ResponsePath is the Go access path on the output (e.g., "Status") + ResponsePath string + // Expected is the Go expression for the expected value + Expected string + // Message is the failure message + Message string + // DerefFunc is an optional dereference function to wrap the response access + // (e.g., "aws.ToBool" for *bool fields) + DerefFunc string +} + +// templateE2EMainVars holds template variables for the main_test.go template. +type templateE2EMainVars struct { + templateset.MetaVars + TestConfig *ackgenconfig.TestConfig +} + +// templateE2EFileVars holds template variables for the per-resource test file. +type templateE2EFileVars struct { + templateset.MetaVars + CRD *ackmodel.CRD + Tests []templateE2ETestVars +} + +// templateE2ETestVars holds template variables for a single named test within +// a resource test file. +type templateE2ETestVars struct { + TestName string // snake_case test name from testconfig key + TestNameCamel string // CamelCase for function name + FieldAssignments map[string]string + UpdateAssignments map[string]string + HasUpdate bool + CreateWaitDuration string + HasVerification bool + CreateVerifyCalls []VerifyCall + UpdateVerifyCalls []VerifyCall +} + +// E2ETests returns a pointer to a TemplateSet containing all templates for +// generating Go e2e test files for an ACK service controller. +func E2ETests( + m *ackmodel.Model, + templateBasePaths []string, + testCfg *ackgenconfig.TestConfig, +) (*templateset.TemplateSet, error) { + crds, err := m.GetCRDs() + if err != nil { + return nil, err + } + + metaVars := m.MetaVars() + + ts := templateset.New( + templateBasePaths, + e2eIncludePaths, + e2eCopyPaths, + e2eFuncMap, + ) + + // Add service-level templates + mainVars := &templateE2EMainVars{ + MetaVars: metaVars, + TestConfig: testCfg, + } + + serviceTemplates := map[string]string{ + "test/e2e-go/main_test.go": "test/e2e/main_test.go.tpl", + "Makefile": "test/e2e/Makefile.tpl", + } + for outPath, tplPath := range serviceTemplates { + if err := ts.Add(outPath, tplPath, mainVars); err != nil { + return nil, err + } + } + + // Add per-CRD test templates + for _, crd := range crds { + testsMap, ok := testCfg.Resources[crd.Names.Original] + if !ok { + continue + } + + // Sort test names for deterministic output + testNames := make([]string, 0, len(testsMap)) + for name := range testsMap { + testNames = append(testNames, name) + } + sort.Strings(testNames) + + var testEntries []templateE2ETestVars + for _, testName := range testNames { + resourceCfg := testsMap[testName] + if resourceCfg.Skip { + continue + } + + fieldAssignments := resolveFieldAssignments(crd, resourceCfg, testCfg) + updateAssignments := resolveUpdateAssignments(crd, resourceCfg) + hasUpdate := len(updateAssignments) > 0 + + createWait := resourceCfg.GetCreateWait() + waitDuration := fmt.Sprintf("%d*time.Second", createWait) + + // Resolve verification calls + createVerifyCalls := resolveVerifyCalls(crd, resourceCfg.CreateValues) + updateVerifyCalls := resolveVerifyCalls(crd, resourceCfg.UpdateValues) + hasVerification := len(createVerifyCalls) > 0 + + testEntries = append(testEntries, templateE2ETestVars{ + TestName: testName, + TestNameCamel: snakeToCamel(testName), + FieldAssignments: fieldAssignments, + UpdateAssignments: updateAssignments, + HasUpdate: hasUpdate, + CreateWaitDuration: waitDuration, + HasVerification: hasVerification, + CreateVerifyCalls: createVerifyCalls, + UpdateVerifyCalls: updateVerifyCalls, + }) + } + + if len(testEntries) == 0 { + continue + } + + fileVars := &templateE2EFileVars{ + MetaVars: metaVars, + CRD: crd, + Tests: testEntries, + } + + outPath := filepath.Join("test/e2e-go", fmt.Sprintf("resource_%s_test.go", crd.Names.Snake)) + tplPath := "test/e2e/resource_test.go.tpl" + if err := ts.Add(outPath, tplPath, fileVars); err != nil { + return nil, err + } + } + + return ts, nil +} + +// resolveVerifyCalls builds verification calls for the given field values by +// finding the appropriate read operation for each field. +func resolveVerifyCalls(crd *ackmodel.CRD, fieldValues map[string]interface{}) []VerifyCall { + if len(fieldValues) == 0 { + return nil + } + + // Group fields by their read operation + type opGroup struct { + fg *ackmodel.FieldGroupOperation + fields map[string]interface{} + } + groups := make(map[string]*opGroup) + + for fieldName, val := range fieldValues { + fg := findReadOpForField(crd, fieldName) + if fg == nil { + continue + } + key := fg.OperationID + if _, ok := groups[key]; !ok { + groups[key] = &opGroup{fg: fg, fields: make(map[string]interface{})} + } + groups[key].fields[fieldName] = val + } + + // Sort by operation name for deterministic output + opNames := make([]string, 0, len(groups)) + for name := range groups { + opNames = append(opNames, name) + } + sort.Strings(opNames) + + var calls []VerifyCall + for _, opName := range opNames { + group := groups[opName] + fg := group.fg + + identAssignments := buildIdentifierAssignments(fg) + assertions := buildAssertionsFromShape(fg, group.fields) + + if len(assertions) == 0 { + continue + } + + calls = append(calls, VerifyCall{ + OperationName: fg.OperationID, + InputTypeName: fg.OperationID + "Input", + IdentifierAssignments: identAssignments, + Assertions: assertions, + }) + } + + return calls +} + +// findReadOpForField finds the FieldGroupOperation that reads the given field. +// Search order: +// 1. ReadFieldGroups (explicitly configured read operations) +// 2. Infer Get* from UpdateFieldGroups Put* operations +// 3. Infer Get* from field's From.Operation (Put*/Update* → Get*/Describe*) +func findReadOpForField(crd *ackmodel.CRD, fieldName string) *ackmodel.FieldGroupOperation { + // 1. Check ReadFieldGroups + for _, fg := range crd.ReadFieldGroups { + for _, f := range fg.PayloadFields { + if f.Names.Camel == fieldName { + return fg + } + } + } + + // 2. Infer from UpdateFieldGroups: Put* → Get* + for _, fg := range crd.UpdateFieldGroups { + for _, f := range fg.PayloadFields { + if f.Names.Camel != fieldName { + continue + } + getOpID := inferGetFromPut(fg.OperationID) + if getOpID == "" { + continue + } + op, found := crd.GetSDKAPI().API.Operations[getOpID] + if !found { + continue + } + return &ackmodel.FieldGroupOperation{ + OpType: ackmodel.FieldGroupOpTypeRead, + Kind: ackmodel.FieldGroupKindDirect, + OperationID: getOpID, + Names: names.New(getOpID), + Operation: op, + IdentifierFields: fg.IdentifierFields, + PayloadFields: fg.PayloadFields, + Config: fg.Config, + } + } + } + + // 3. Fallback: use the field's From.Operation config to infer the Get operation. + // This handles controllers that use custom update methods but still have + // fields configured with from.operation (e.g., S3's committed generator.yaml). + field, found := crd.Fields[fieldName] + if !found { + return nil + } + if field.FieldConfig == nil || field.FieldConfig.From == nil { + return nil + } + putOpID := field.FieldConfig.From.Operation + getOpID := inferGetFromPut(putOpID) + if getOpID == "" { + return nil + } + op, found := crd.GetSDKAPI().API.Operations[getOpID] + if !found { + return nil + } + // Build identifier fields from the primary key + var identifierFields []*ackmodel.Field + pkField, err := crd.GetPrimaryKeyField() + if err == nil && pkField != nil { + identifierFields = []*ackmodel.Field{pkField} + } + return &ackmodel.FieldGroupOperation{ + OpType: ackmodel.FieldGroupOpTypeRead, + Kind: ackmodel.FieldGroupKindDirect, + OperationID: getOpID, + Names: names.New(getOpID), + Operation: op, + IdentifierFields: identifierFields, + PayloadFields: []*ackmodel.Field{field}, + } +} + +// inferGetFromPut derives the Get operation name from a Put operation name. +func inferGetFromPut(putOpID string) string { + if strings.HasPrefix(putOpID, "Put") { + return "Get" + putOpID[3:] + } + if strings.HasPrefix(putOpID, "Update") { + return "Describe" + putOpID[6:] + } + return "" +} + +// buildIdentifierAssignments generates Go expressions for each identifier field +// in the read operation's input. +func buildIdentifierAssignments(fg *ackmodel.FieldGroupOperation) map[string]string { + assignments := make(map[string]string) + if fg.Operation == nil || fg.Operation.InputRef.Shape == nil { + return assignments + } + + inputShape := fg.Operation.InputRef.Shape + for _, memberName := range inputShape.MemberNames() { + // Only include identifier fields (skip optional fields like ExpectedBucketOwner) + isIdentifier := false + for _, idf := range fg.IdentifierFields { + if idf.Names.Camel == memberName || idf.Names.Original == memberName { + isIdentifier = true + break + } + } + if !isIdentifier { + // Check if this member is required (it might be the bucket name with a different name) + memberRef := inputShape.MemberRefs[memberName] + if memberRef == nil || !inputShape.IsRequired(memberName) { + continue + } + } + + memberRef := inputShape.MemberRefs[memberName] + if memberRef == nil || memberRef.Shape == nil { + continue + } + switch memberRef.Shape.Type { + case "string": + assignments[memberName] = "aws.String(name)" + default: + assignments[memberName] = "aws.String(name)" + } + } + + return assignments +} + +// buildAssertionsFromShape generates assertions by matching field values against +// the SDK output shape members. +func buildAssertionsFromShape(fg *ackmodel.FieldGroupOperation, fieldValues map[string]interface{}) []VerifyAssertion { + if fg.Operation == nil || fg.Operation.OutputRef.Shape == nil { + return nil + } + + outputShape := fg.Operation.OutputRef.Shape + var assertions []VerifyAssertion + + for fieldName, val := range fieldValues { + // The field value from testconfig may be a nested struct (e.g., {Status: "Enabled"}) + // or a scalar. We need to match it against the output shape. + mapVal, isMap := normalizeToMap(val) + if isMap { + // Nested struct: each key in the map corresponds to an output shape member + subAssertions := buildNestedAssertions(outputShape, "", mapVal, fieldName) + assertions = append(assertions, subAssertions...) + } else { + // Scalar: find the output member that corresponds to this field + member := findOutputMemberForField(outputShape, fieldName) + if member != "" { + memberRef := outputShape.MemberRefs[member] + expr := goAssertionExpr(memberRef, val) + assertions = append(assertions, VerifyAssertion{ + ResponsePath: member, + Expected: expr, + DerefFunc: derefFuncForShape(memberRef), + Message: fmt.Sprintf("%s mismatch", fieldName), + }) + } + } + } + + sort.Slice(assertions, func(i, j int) bool { + return assertions[i].ResponsePath < assertions[j].ResponsePath + }) + return assertions +} + +// buildNestedAssertions handles map values from testconfig (e.g., {Status: "Enabled"}) +// and generates assertions for each leaf value. +func buildNestedAssertions(shape *awssdkmodel.Shape, pathPrefix string, vals map[string]interface{}, contextFieldName string) []VerifyAssertion { + var assertions []VerifyAssertion + + // If none of the vals keys match the shape's direct members, check if the + // shape has a single struct wrapper member we should descend into. + // E.g., GetPublicAccessBlockOutput has only "PublicAccessBlockConfiguration" + // but the test values have "BlockPublicACLs" etc. + if !anyMemberMatches(shape, vals) { + if wrapperRef, wrapperName := findSingleStructMember(shape); wrapperRef != nil { + wrapperPath := wrapperName + if pathPrefix != "" { + wrapperPath = pathPrefix + "." + wrapperName + } + return buildNestedAssertions(wrapperRef.Shape, wrapperPath, vals, contextFieldName) + } + } + + for key, val := range vals { + memberRef, memberName := findMemberByName(shape, key) + if memberRef == nil { + continue + } + + fullPath := memberName + if pathPrefix != "" { + fullPath = pathPrefix + "." + memberName + } + + subMap, isSubMap := normalizeToMap(val) + if isSubMap && memberRef.Shape != nil && memberRef.Shape.Type == "structure" { + nested := buildNestedAssertions(memberRef.Shape, fullPath, subMap, contextFieldName+"."+key) + assertions = append(assertions, nested...) + } else { + expr := goAssertionExpr(memberRef, val) + assertions = append(assertions, VerifyAssertion{ + ResponsePath: fullPath, + Expected: expr, + DerefFunc: derefFuncForShape(memberRef), + Message: fmt.Sprintf("%s mismatch", contextFieldName+"."+key), + }) + } + } + + return assertions +} + +// anyMemberMatches returns true if any key in vals matches a member of the shape. +func anyMemberMatches(shape *awssdkmodel.Shape, vals map[string]interface{}) bool { + for key := range vals { + if ref, _ := findMemberByName(shape, key); ref != nil { + return true + } + } + return false +} + +// findSingleStructMember returns the sole struct-type member of a shape, if +// there is exactly one (ignoring metadata/result fields). Returns nil otherwise. +func findSingleStructMember(shape *awssdkmodel.Shape) (*awssdkmodel.ShapeRef, string) { + if shape == nil { + return nil, "" + } + var found *awssdkmodel.ShapeRef + var foundName string + for name, ref := range shape.MemberRefs { + if name == "ResultMetadata" { + continue + } + if ref.Shape != nil && ref.Shape.Type == "structure" { + if found != nil { + return nil, "" + } + found = ref + foundName = name + } + } + return found, foundName +} + +// findMemberByName looks up a shape member by camelCase or original name. +func findMemberByName(shape *awssdkmodel.Shape, name string) (*awssdkmodel.ShapeRef, string) { + if shape == nil { + return nil, "" + } + // Direct match + if ref, ok := shape.MemberRefs[name]; ok { + return ref, name + } + // Case-insensitive match + lower := strings.ToLower(name) + for memberName, ref := range shape.MemberRefs { + if strings.ToLower(memberName) == lower { + return ref, memberName + } + } + return nil, "" +} + +// findOutputMemberForField finds the output shape member that corresponds to +// a CRD field name. +func findOutputMemberForField(outputShape *awssdkmodel.Shape, fieldName string) string { + if outputShape == nil { + return "" + } + // Direct match + if _, ok := outputShape.MemberRefs[fieldName]; ok { + return fieldName + } + // Case-insensitive match + lower := strings.ToLower(fieldName) + for memberName := range outputShape.MemberRefs { + if strings.ToLower(memberName) == lower { + return memberName + } + } + return "" +} + +// goAssertionExpr generates a Go expression for an expected value based on the +// SDK shape reference type. +func goAssertionExpr(ref *awssdkmodel.ShapeRef, val interface{}) string { + if ref == nil || ref.Shape == nil { + return fmt.Sprintf("%q", fmt.Sprintf("%v", val)) + } + shape := ref.Shape + + switch shape.Type { + case "string", "character": + s := fmt.Sprintf("%v", val) + // If the shape has an enum, use the typed enum + if len(shape.Enum) > 0 { + typeName := shape.ShapeName + return fmt.Sprintf("svcsdktypes.%s(%q)", typeName, s) + } + return fmt.Sprintf("%q", s) + case "boolean", "primitiveBoolean": + return fmt.Sprintf("%t", val) + case "byte", "short", "integer", "primitiveInteger": + switch v := val.(type) { + case float64: + return fmt.Sprintf("int32(%d)", int32(v)) + default: + return fmt.Sprintf("int32(%v)", v) + } + case "long": + switch v := val.(type) { + case float64: + return fmt.Sprintf("int64(%d)", int64(v)) + default: + return fmt.Sprintf("int64(%v)", v) + } + case "float", "double": + return fmt.Sprintf("%v", val) + default: + return fmt.Sprintf("%q", fmt.Sprintf("%v", val)) + } +} + +// derefFuncForShape returns the aws.ToX function name needed to dereference +// pointer-typed SDK struct members, or "" if no dereference is needed. +func derefFuncForShape(ref *awssdkmodel.ShapeRef) string { + if ref == nil || ref.Shape == nil { + return "" + } + switch ref.Shape.Type { + case "boolean", "primitiveBoolean": + return "aws.ToBool" + case "string", "character": + if len(ref.Shape.Enum) > 0 { + return "" + } + return "aws.ToString" + case "byte", "short", "integer", "primitiveInteger": + return "aws.ToInt32" + case "long": + return "aws.ToInt64" + case "float": + return "aws.ToFloat32" + case "double": + return "aws.ToFloat64" + default: + return "" + } +} + +// normalizeToMap converts a value to map[string]interface{} if it's a map type. +func normalizeToMap(val interface{}) (map[string]interface{}, bool) { + switch v := val.(type) { + case map[string]interface{}: + return v, true + case map[interface{}]interface{}: + m := make(map[string]interface{}) + for k, vv := range v { + m[fmt.Sprintf("%v", k)] = vv + } + return m, true + } + return nil, false +} + +// snakeToCamel converts a snake_case string to CamelCase. +func snakeToCamel(s string) string { + parts := strings.Split(s, "_") + for i, p := range parts { + if len(p) > 0 { + parts[i] = strings.ToUpper(p[:1]) + p[1:] + } + } + return strings.Join(parts, "") +} + +// resolveFieldAssignments builds the map of Go code expressions for each field +// in the CRD Spec, following the value resolution order: +// 1. bootstrap_fields → bootstrapResources.X.Y +// 2. create_values → literal value +// 3. auto-derived identifier → name variable reference +// 4. auto-derived shape defaults (only for required fields) +// 5. omit optional fields with no explicit value +func resolveFieldAssignments( + crd *ackmodel.CRD, + resourceCfg ackgenconfig.TestResourceConfig, + testCfg *ackgenconfig.TestConfig, +) map[string]string { + assignments := make(map[string]string) + + for fieldName, field := range crd.SpecFields { + goFieldName := field.Names.Camel + + // 1. Bootstrap fields (always included if configured) + if path, ok := resourceCfg.BootstrapFields[goFieldName]; ok { + assignments[goFieldName] = bootstrapFieldExpr(field, path) + continue + } + + // 2. Explicit create_values (always included if configured) + if val, ok := resourceCfg.CreateValues[goFieldName]; ok { + assignments[goFieldName] = goLiteralForValue(field, val) + continue + } + + // 3. Auto-derived: primary identifier fields get the random name + if isPrimaryIdentifierField(crd, fieldName) { + assignments[goFieldName] = identifierExpr(field) + continue + } + + // For optional fields with no explicit config, skip them + if !field.IsRequired() { + continue + } + + // 4. Auto-derived from shape (required fields only) + if expr := shapeDefaultExpr(field); expr != "" { + assignments[goFieldName] = expr + continue + } + + // 5. Required field with no derivable value — placeholder + assignments[goFieldName] = fmt.Sprintf(`new("TODO_%s")`, goFieldName) + } + + return assignments +} + +// resolveUpdateAssignments builds Go code expressions for update test fields. +func resolveUpdateAssignments( + crd *ackmodel.CRD, + resourceCfg ackgenconfig.TestResourceConfig, +) map[string]string { + assignments := make(map[string]string) + for goFieldName, val := range resourceCfg.UpdateValues { + field := findFieldByCamelName(crd, goFieldName) + if field == nil { + continue + } + assignments[goFieldName] = goLiteralForValue(field, val) + } + return assignments +} + +// findFieldByCamelName looks up a spec field by its Go struct name (Names.Camel). +func findFieldByCamelName(crd *ackmodel.CRD, camelName string) *ackmodel.Field { + for _, field := range crd.SpecFields { + if field.Names.Camel == camelName { + return field + } + } + return nil +} + +// bootstrapFieldExpr generates a Go expression that reads from the bootstrapResources +// global variable. path is like "SharedVPC.SecurityGroupID". +func bootstrapFieldExpr(field *ackmodel.Field, path string) string { + expr := "bootstrapResources." + path + goType := field.GoType + if strings.HasPrefix(goType, "*") { + return fmt.Sprintf("new(%s)", expr) + } + if strings.HasPrefix(goType, "[]*string") { + return fmt.Sprintf("[]*string{new(%s)}", expr) + } + return expr +} + +// identifierExpr generates a Go expression for an identifier field that uses +// the test's random name variable. +func identifierExpr(field *ackmodel.Field) string { + goType := field.GoType + if strings.HasPrefix(goType, "*") { + return "&name" + } + return "name" +} + +// isPrimaryIdentifierField returns true if the field is the resource's primary +// identifier — either marked as is_primary_key in config, or matching strict +// naming patterns for common identifier fields. +func isPrimaryIdentifierField(crd *ackmodel.CRD, fieldName string) bool { + // Check if field is explicitly marked as primary key + field, ok := crd.SpecFields[fieldName] + if ok && field.FieldConfig != nil && field.FieldConfig.IsPrimaryKey { + return true + } + + lower := strings.ToLower(fieldName) + resourceLower := strings.ToLower(crd.Names.Original) + + // Strict matches: the field name IS the resource name + identifier suffix + identifierPatterns := []string{ + resourceLower + "name", + resourceLower + "id", + resourceLower + "identifier", + } + for _, p := range identifierPatterns { + if lower == p { + return true + } + } + + // Exact match for the generic "Name" field (very common pattern) + if lower == "name" { + return true + } + + return false +} + +// shapeDefaultExpr generates a Go expression using shape metadata (enums, etc.) +func shapeDefaultExpr(field *ackmodel.Field) string { + if field.ShapeRef == nil || field.ShapeRef.Shape == nil { + return "" + } + shape := field.ShapeRef.Shape + + // Enum: use first value + if len(shape.Enum) > 0 { + goType := field.GoType + if strings.HasPrefix(goType, "*") { + return fmt.Sprintf(`new("%s")`, shape.Enum[0]) + } + return fmt.Sprintf(`"%s"`, shape.Enum[0]) + } + + return "" +} + +// goLiteralForValue converts a testconfig.yaml value to a Go code expression. +func goLiteralForValue(field *ackmodel.Field, val interface{}) string { + goType := field.GoType + + switch v := val.(type) { + case string: + if strings.HasPrefix(goType, "*string") { + return fmt.Sprintf("new(%q)", v) + } + return fmt.Sprintf("%q", v) + case int: + if strings.Contains(goType, "int64") { + if strings.HasPrefix(goType, "*") { + return fmt.Sprintf("new(%d)", v) + } + return fmt.Sprintf("int64(%d)", v) + } + if strings.HasPrefix(goType, "*") { + return fmt.Sprintf("new(%d)", v) + } + return fmt.Sprintf("%d", v) + case float64: + // YAML numbers often decode as float64 + intVal := int64(v) + if v == float64(intVal) { + if strings.HasPrefix(goType, "*") { + return fmt.Sprintf("new(%d)", intVal) + } + return fmt.Sprintf("int64(%d)", intVal) + } + if strings.HasPrefix(goType, "*") { + return fmt.Sprintf("new(%g)", v) + } + return fmt.Sprintf("%g", v) + case bool: + if strings.HasPrefix(goType, "*") { + return fmt.Sprintf("new(%t)", v) + } + return fmt.Sprintf("%t", v) + case map[string]interface{}: + return goLiteralForStruct(field, v) + case map[interface{}]interface{}: + normalized := make(map[string]interface{}) + for k, val := range v { + normalized[fmt.Sprintf("%v", k)] = val + } + return goLiteralForStruct(field, normalized) + default: + return fmt.Sprintf("%#v", v) + } +} + +// goLiteralForStruct generates a Go struct literal from a map of field values. +func goLiteralForStruct(field *ackmodel.Field, vals map[string]interface{}) string { + if field.ShapeRef == nil || field.ShapeRef.Shape == nil { + return fmt.Sprintf("%#v", vals) + } + return goLiteralForShapeStruct(field.ShapeRef, vals) +} + +// goLiteralForShapeStruct generates a struct literal for a given shape reference. +func goLiteralForShapeStruct(ref *awssdkmodel.ShapeRef, vals map[string]interface{}) string { + shape := ref.Shape + typeName := shape.ShapeName + + members := make([]string, 0) + for memberName, memberRef := range shape.MemberRefs { + goMemberName := names.New(memberName).Camel + val, ok := vals[goMemberName] + if !ok { + val, ok = vals[memberName] + } + if !ok { + continue + } + memberExpr := goLiteralForShapeRef(memberRef, val) + members = append(members, fmt.Sprintf("%s: %s", goMemberName, memberExpr)) + } + + if len(members) == 1 { + return fmt.Sprintf("&svcapitypes.%s{%s}", typeName, members[0]) + } + + var sb strings.Builder + sb.WriteString(fmt.Sprintf("&svcapitypes.%s{", typeName)) + for _, m := range members { + sb.WriteString("\n\t" + m + ",") + } + sb.WriteString("\n}") + return sb.String() +} + +// goLiteralForShapeRef converts a value to a Go expression based on the AWS SDK shape. +func goLiteralForShapeRef(ref *awssdkmodel.ShapeRef, val interface{}) string { + if ref == nil || ref.Shape == nil { + return fmt.Sprintf("%#v", val) + } + shape := ref.Shape + + switch shape.Type { + case "string", "character": + s := fmt.Sprintf("%v", val) + return fmt.Sprintf("new(%q)", s) + case "boolean", "primitiveBoolean": + return fmt.Sprintf("new(bool(%t))", val) + case "byte", "short", "integer", "long", "primitiveInteger": + switch v := val.(type) { + case float64: + return fmt.Sprintf("new(%d)", int64(v)) + default: + return fmt.Sprintf("new(%v)", v) + } + case "float", "double": + return fmt.Sprintf("new(%v)", val) + case "structure", "union": + mapVal, ok := val.(map[string]interface{}) + if !ok { + if m, ok2 := val.(map[interface{}]interface{}); ok2 { + mapVal = make(map[string]interface{}) + for k, v := range m { + mapVal[fmt.Sprintf("%v", k)] = v + } + } + } + if mapVal == nil { + return fmt.Sprintf("%#v", val) + } + return goLiteralForShapeStruct(ref, mapVal) + default: + return fmt.Sprintf("%#v", val) + } +} diff --git a/pkg/generate/code/set_resource_field_group.go b/pkg/generate/code/set_resource_field_group.go index 144f140d..92e81ab1 100644 --- a/pkg/generate/code/set_resource_field_group.go +++ b/pkg/generate/code/set_resource_field_group.go @@ -17,6 +17,7 @@ import ( "fmt" "strings" + awssdkmodel "github.com/aws-controllers-k8s/code-generator/pkg/api" ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/config" "github.com/aws-controllers-k8s/code-generator/pkg/fieldpath" "github.com/aws-controllers-k8s/code-generator/pkg/model" @@ -71,6 +72,11 @@ func SetResourceFieldGroup( opType = model.OpTypeGet } + // Track which payload fields were successfully matched from the output + // shape members. Any unmatched fields with a from.path config will be + // handled by the unwrapped-output fallback below. + matchedFields := make(map[string]bool) + for memberIndex, memberName := range outputShape.MemberNames() { sourceAdaptedVarName := sourceVarName + "." + memberName @@ -197,6 +203,161 @@ func SetResourceFieldGroup( } else { indentLevel += 1 } + matchedFields[f.Names.Camel] = true + } + + // Fallback: for payload fields that weren't matched by any output member, + // check if the field has a from.path config. If so, the output response + // members are the *unwrapped* content of that shape, and we generate code + // that constructs the wrapper struct from the flat output members. + for _, f := range fg.PayloadFields { + if matchedFields[f.Names.Camel] { + continue + } + if f.FieldConfig == nil || f.FieldConfig.From == nil { + continue + } + wrapperOut, err := setResourceFieldGroupWrapped( + cfg, r, f, fg, outputShape, sourceVarName, targetVarName, + indentLevel, + ) + if err != nil { + return "", err + } + out += wrapperOut + } + + return out, nil +} + +// setResourceFieldGroupWrapped generates code for a payload field whose Get +// output is the unwrapped content of the field's from.path shape. It +// constructs the wrapper struct and maps output members into it. +// +// For example, if the CRD field is Accelerate (*AccelerateConfiguration) with +// from.path = "AccelerateConfiguration", and the Get output has member +// "Status" (which is a member of AccelerateConfiguration), we generate: +// +// ko.Spec.Accelerate = &svcapitypes.AccelerateConfiguration{} +// ko.Spec.Accelerate.Status = aws.String(string(resp.Status)) +func setResourceFieldGroupWrapped( + cfg *ackgenconfig.Config, + r *model.CRD, + f *model.Field, + fg *model.FieldGroupOperation, + outputShape *awssdkmodel.Shape, + sourceVarName string, + targetVarName string, + indentLevel int, +) (string, error) { + fromCfg := f.FieldConfig.From + // Look up the wrapper shape from the from.operation's Input shape + fromOpID := fromCfg.Operation + fromOp, ok := r.GetSDKAPI().API.Operations[fromOpID] + if !ok { + return "", fmt.Errorf( + "field %q: from.operation %q not found in SDK", + f.Names.Camel, fromOpID, + ) + } + fromInputShape := fromOp.InputRef.Shape + if fromInputShape == nil { + return "", fmt.Errorf( + "field %q: from.operation %q has nil Input shape", + f.Names.Camel, fromOpID, + ) + } + + // Navigate from.path to find the wrapper shape + wrapperShapeRef, ok := fromInputShape.MemberRefs[fromCfg.Path] + if !ok { + return "", fmt.Errorf( + "field %q: from.path %q not found in %s Input shape", + f.Names.Camel, fromCfg.Path, fromOpID, + ) } + wrapperShape := wrapperShapeRef.Shape + if wrapperShape == nil { + return "", fmt.Errorf( + "field %q: from.path %q has nil shape", + f.Names.Camel, fromCfg.Path, + ) + } + + indent := strings.Repeat("\t", indentLevel) + targetPrefix := targetVarName + cfg.PrefixConfig.SpecField + "." + f.Names.Camel + + out := "" + // Generate the wrapper struct construction using the existing + // setResourceForContainer machinery. The output response serves as the + // source, and the wrapper shape defines both source and target member + // structure. + out += fmt.Sprintf( + "%s%s = &svcapitypes.%s{}\n", + indent, targetPrefix, wrapperShape.ShapeName, + ) + + // Map each wrapper shape member from the output response + for _, wrapperMemberName := range wrapperShape.MemberNames() { + // Check if the output shape has this member + outputMemberRef, outputHasMember := outputShape.MemberRefs[wrapperMemberName] + if !outputHasMember || outputMemberRef.Shape == nil { + continue + } + sourceMemberVar := sourceVarName + "." + wrapperMemberName + targetMemberVar := targetPrefix + "." + wrapperMemberName + + outputMemberShape := outputMemberRef.Shape + switch outputMemberShape.Type { + case "string": + if outputMemberShape.IsEnum() { + out += fmt.Sprintf( + "%sif %s != \"\" {\n", indent, sourceMemberVar, + ) + out += fmt.Sprintf( + "%s\t%s = aws.String(string(%s))\n", + indent, targetMemberVar, sourceMemberVar, + ) + out += fmt.Sprintf("%s}\n", indent) + } else { + out += fmt.Sprintf( + "%sif %s != nil {\n", indent, sourceMemberVar, + ) + out += fmt.Sprintf( + "%s\t%s = %s\n", + indent, targetMemberVar, sourceMemberVar, + ) + out += fmt.Sprintf("%s}\n", indent) + } + case "integer", "long": + out += fmt.Sprintf( + "%sif %s != nil {\n", indent, sourceMemberVar, + ) + out += fmt.Sprintf( + "%s\t%s = %s\n", + indent, targetMemberVar, sourceMemberVar, + ) + out += fmt.Sprintf("%s}\n", indent) + case "boolean": + out += fmt.Sprintf( + "%sif %s != nil {\n", indent, sourceMemberVar, + ) + out += fmt.Sprintf( + "%s\t%s = %s\n", + indent, targetMemberVar, sourceMemberVar, + ) + out += fmt.Sprintf("%s}\n", indent) + case "list", "map", "structure": + out += fmt.Sprintf( + "%sif %s != nil {\n", indent, sourceMemberVar, + ) + out += fmt.Sprintf( + "%s\t%s = %s\n", + indent, targetMemberVar, sourceMemberVar, + ) + out += fmt.Sprintf("%s}\n", indent) + } + } + return out, nil } diff --git a/pkg/generate/code/tags.go b/pkg/generate/code/tags.go index 02a0bd23..98d68689 100644 --- a/pkg/generate/code/tags.go +++ b/pkg/generate/code/tags.go @@ -23,23 +23,21 @@ import ( // GoCodeToACKTags returns Go code that converts Resource Tags // to ACK Tags. If Resource Tags field is of type list, we // also maintain and return the order of the list as a []string -// -// // // Sample output: // -// for _, k := range keyOrder { -// v, ok := tags[k] -// if ok { -// tag := svcapitypes.Tag{Key: &k, Value: &v} -// result = append(result, &tag) -// delete(tags, k) -// } -// } -// for k, v := range tags { -// tag := svcapitypes.Tag{Key: &k, Value: &v} -// result = append(result, &tag) -// } +// for _, k := range keyOrder { +// v, ok := tags[k] +// if ok { +// tag := svcapitypes.Tag{Key: &k, Value: &v} +// result = append(result, &tag) +// delete(tags, k) +// } +// } +// for k, v := range tags { +// tag := svcapitypes.Tag{Key: &k, Value: &v} +// result = append(result, &tag) +// } func GoCodeConvertToACKTags(r *model.CRD, sourceVarName string, targetVarName string, keyOrderVarName string, indentLevel int) (string, error) { out := "\n" @@ -92,25 +90,23 @@ func GoCodeConvertToACKTags(r *model.CRD, sourceVarName string, targetVarName st // GoCodeFromACKTags returns Go code that converts ACKTags // to the Resource Tag shape type. Tag fields can only be // maps or lists of Tag Go type. If Tag field is a list, -// when converting from ACK Tags, we try to preserve the +// when converting from ACK Tags, we try to preserve the // original order -// -// // // Sample output: // -// for _, k := range keyOrder { -// v, ok := tags[k] -// if ok { -// tag := svcapitypes.Tag{Key: &k, Value: &v} -// result = append(result, &tag) -// delete(tags, k) -// } -// } -// for k, v := range tags { -// tag := svcapitypes.Tag{Key: &k, Value: &v} -// result = append(result, &tag) -// } +// for _, k := range keyOrder { +// v, ok := tags[k] +// if ok { +// tag := svcapitypes.Tag{Key: &k, Value: &v} +// result = append(result, &tag) +// delete(tags, k) +// } +// } +// for k, v := range tags { +// tag := svcapitypes.Tag{Key: &k, Value: &v} +// result = append(result, &tag) +// } func GoCodeFromACKTags(r *model.CRD, tagsSourceVarName string, orderVarName string, targetVarName string, indentLevel int) (string, error) { out := "\n" indent := strings.Repeat("\t", indentLevel) diff --git a/pkg/generate/code/tags_test.go b/pkg/generate/code/tags_test.go index 95231bf6..dd2ec2b4 100644 --- a/pkg/generate/code/tags_test.go +++ b/pkg/generate/code/tags_test.go @@ -132,4 +132,4 @@ func TestFromACKTagsForMapShape(t *testing.T) { ) require.NoError(err) assert.Equal(expectedSyncedConditions, got) -} \ No newline at end of file +} diff --git a/pkg/model/crd.go b/pkg/model/crd.go index d3598b87..e13b545a 100644 --- a/pkg/model/crd.go +++ b/pkg/model/crd.go @@ -122,6 +122,11 @@ func (r *CRD) GetStorageVersion(defaultVersion string) (string, error) { r.Names.Original) } +// GetSDKAPI returns the SDKAPI pointer for this CRD's service. +func (r *CRD) GetSDKAPI() *SDKAPI { + return r.sdkAPI +} + // SDKAPIPackageName returns the aws-sdk-go package name used for this // resource's API func (r *CRD) SDKAPIPackageName() string { diff --git a/pkg/model/field_group.go b/pkg/model/field_group.go index 76e1d58c..a4ce3ce6 100644 --- a/pkg/model/field_group.go +++ b/pkg/model/field_group.go @@ -362,6 +362,19 @@ func (r *CRD) resolveExplicitFields( return identifierFields, payloadFields } +// ExceptionCode404 returns the AWS error code to treat as "not found" (nil +// field) for this specific operation. Returns empty string if not configured, +// in which case the template falls back to the resource-level 404 code. +func (fg *FieldGroupOperation) ExceptionCode404() string { + if fg.Config.Exceptions == nil { + return "" + } + if errCfg, ok := fg.Config.Exceptions.Errors[404]; ok { + return errCfg.Code + } + return "" +} + // IsDirect returns true if this is a standard direct field-group operation. func (fg *FieldGroupOperation) IsDirect() bool { return fg.Kind == FieldGroupKindDirect diff --git a/pkg/sdk/fetch.go b/pkg/sdk/fetch.go index 4c99b804..b993c909 100644 --- a/pkg/sdk/fetch.go +++ b/pkg/sdk/fetch.go @@ -26,7 +26,7 @@ import ( ) const ( - sdkModelURLTemplate = "https://raw.githubusercontent.com/aws/aws-sdk-go-v2/%s/codegen/sdk-codegen/aws-models/%s.json" + sdkModelURLTemplate = "https://raw.githubusercontent.com/aws/aws-sdk-go-v2/%s/codegen/sdk-codegen/aws-models/%s.json" defaultHTTPFetchTimeout = 60 * time.Second ) diff --git a/pkg/sdk/repo.go b/pkg/sdk/repo.go index 73494ef5..3935e194 100644 --- a/pkg/sdk/repo.go +++ b/pkg/sdk/repo.go @@ -75,7 +75,7 @@ func EnsureDir(fp string) (bool, error) { // isDirWriteable returns true if the supplied directory path is writeable, // false otherwise func isDirWriteable(fp string) bool { - testPath := filepath.Join(fp, "test") + testPath := filepath.Join(fp, ".ack-generate-write-check") f, err := os.Create(testPath) if err != nil { return false diff --git a/scripts/build-e2e-tests.sh b/scripts/build-e2e-tests.sh new file mode 100755 index 00000000..2a75f9f9 --- /dev/null +++ b/scripts/build-e2e-tests.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash + +# A script that generates Go e2e test scaffolds for an ACK service controller + +set -eo pipefail + +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$SCRIPTS_DIR/.." + +source "$SCRIPTS_DIR/lib/common.sh" + +ACK_GENERATE_CACHE_DIR=${ACK_GENERATE_CACHE_DIR:-"$HOME/.cache/aws-controllers-k8s"} +DEFAULT_ACK_GENERATE_BIN_PATH="$ROOT_DIR/bin/ack-generate" +ACK_GENERATE_BIN_PATH=${ACK_GENERATE_BIN_PATH:-$DEFAULT_ACK_GENERATE_BIN_PATH} +ACK_GENERATE_CONFIG_PATH=${ACK_GENERATE_CONFIG_PATH:-""} +ACK_METADATA_CONFIG_PATH=${ACK_METADATA_CONFIG_PATH:-""} +AWS_SDK_GO_VERSION=${AWS_SDK_GO_VERSION:-""} + +USAGE=" +Usage: + $(basename "$0") + + should be an AWS service API alias that you wish to generate e2e tests +for -- e.g. 's3' 'sns' or 'sqs' + +Environment variables: + ACK_GENERATE_CACHE_DIR: Overrides the directory used for caching AWS API + models used by the ack-generate tool. + Default: $ACK_GENERATE_CACHE_DIR + ACK_GENERATE_BIN_PATH: Overrides the path to the ack-generate binary. + Default: $ACK_GENERATE_BIN_PATH + ACK_GENERATE_CONFIG_PATH: Specify a path to the generator config YAML file. + Default: generator.yaml in controller source path + ACK_METADATA_CONFIG_PATH: Specify a path to the metadata config YAML file. + Default: metadata.yaml in controller source path + AWS_SDK_GO_VERSION: Overrides the version of github.com/aws/aws-sdk-go used + by 'ack-generate' to fetch the service API Specifications. + Default: Version of aws/aws-sdk-go in service go.mod + SERVICE_CONTROLLER_SOURCE_PATH: Path to the service controller source directory. + Default: ../-controller + TEMPLATE_DIRS: Overrides the list of directories containing ack-generate + templates. + TEST_CONFIG_PATH: Path to testconfig.yaml. + Default: testconfig.yaml in controller source path +" + +if [ $# -ne 1 ]; then + echo "ERROR: $(basename "$0") only accepts a single parameter" 1>&2 + echo "$USAGE" + exit 1 +fi + +if [ ! -f "$ACK_GENERATE_BIN_PATH" ]; then + if is_installed "ack-generate"; then + ACK_GENERATE_BIN_PATH=$(which "ack-generate") + else + echo "ERROR: Unable to find an ack-generate binary. +Either set the ACK_GENERATE_BIN_PATH to a valid location or +run: + + make build-ack-generate + +from the root directory or install ack-generate using: + + go get -u -tags codegen github.com/aws-controllers-k8s/code-generator/cmd/ack-generate" 1>&2 + exit 1; + fi +fi +SERVICE=$(echo "$1" | tr '[:upper:]' '[:lower:]') + +DEFAULT_SERVICE_CONTROLLER_SOURCE_PATH="$ROOT_DIR/../$SERVICE-controller" +SERVICE_CONTROLLER_SOURCE_PATH=${SERVICE_CONTROLLER_SOURCE_PATH:-$DEFAULT_SERVICE_CONTROLLER_SOURCE_PATH} + +if [[ ! -d $SERVICE_CONTROLLER_SOURCE_PATH ]]; then + echo "Error evaluating SERVICE_CONTROLLER_SOURCE_PATH environment variable:" 1>&2 + echo "$SERVICE_CONTROLLER_SOURCE_PATH is not a directory." 1>&2 + echo "${USAGE}" + exit 1 +fi + +DEFAULT_TEMPLATE_DIRS="$ROOT_DIR/templates" +if [[ -d "$SERVICE_CONTROLLER_SOURCE_PATH/templates" ]]; then + DEFAULT_TEMPLATE_DIRS="$SERVICE_CONTROLLER_SOURCE_PATH/templates,$DEFAULT_TEMPLATE_DIRS" +fi +TEMPLATE_DIRS=${TEMPLATE_DIRS:-$DEFAULT_TEMPLATE_DIRS} + +# Determine the API version from the controller source +ACK_GENERATE_API_VERSION=${ACK_GENERATE_API_VERSION:-"v1alpha1"} +if [[ -d "$SERVICE_CONTROLLER_SOURCE_PATH/apis" ]]; then + LATEST_API_VERSION=$(ls "$SERVICE_CONTROLLER_SOURCE_PATH/apis/" | sort -V | tail -1) + if [[ -n "$LATEST_API_VERSION" ]]; then + ACK_GENERATE_API_VERSION="$LATEST_API_VERSION" + fi +fi + +# Build args for ack-generate e2e-tests +ag_args=("e2e-tests" "$SERVICE" -o "$SERVICE_CONTROLLER_SOURCE_PATH" --template-dirs "$TEMPLATE_DIRS") + +if [ -n "$ACK_GENERATE_CACHE_DIR" ]; then + ag_args=("${ag_args[@]}" --cache-dir "$ACK_GENERATE_CACHE_DIR") +fi + +if [ -n "$ACK_GENERATE_CONFIG_PATH" ]; then + ag_args=("${ag_args[@]}" --generator-config-path "$ACK_GENERATE_CONFIG_PATH") +elif [ -f "$SERVICE_CONTROLLER_SOURCE_PATH/generator.yaml" ]; then + ag_args=("${ag_args[@]}" --generator-config-path "$SERVICE_CONTROLLER_SOURCE_PATH/generator.yaml") +fi + +if [ -n "$ACK_METADATA_CONFIG_PATH" ]; then + ag_args=("${ag_args[@]}" --metadata-config-path "$ACK_METADATA_CONFIG_PATH") +elif [ -f "$SERVICE_CONTROLLER_SOURCE_PATH/metadata.yaml" ]; then + ag_args=("${ag_args[@]}" --metadata-config-path "$SERVICE_CONTROLLER_SOURCE_PATH/metadata.yaml") +fi + +if [ -n "$AWS_SDK_GO_VERSION" ]; then + ag_args=("${ag_args[@]}" --aws-sdk-go-version "$AWS_SDK_GO_VERSION") +fi + +TEST_CONFIG_PATH=${TEST_CONFIG_PATH:-"$SERVICE_CONTROLLER_SOURCE_PATH/testconfig.yaml"} +if [ -f "$TEST_CONFIG_PATH" ]; then + ag_args=("${ag_args[@]}" --test-config "$TEST_CONFIG_PATH") +else + echo "ERROR: testconfig.yaml not found at $TEST_CONFIG_PATH" 1>&2 + echo "Create a testconfig.yaml to define test values for resources." 1>&2 + exit 1 +fi + +echo "Generating e2e tests for $SERVICE" +$ACK_GENERATE_BIN_PATH "${ag_args[@]}" + +pushd "$SERVICE_CONTROLLER_SOURCE_PATH" 1>/dev/null +gofmt -w . +popd 1>/dev/null + +echo "e2e test generation complete." diff --git a/templates/pkg/resource/sdk_find_field_groups.go.tpl b/templates/pkg/resource/sdk_find_field_groups.go.tpl index 5346ec1e..683f257a 100644 --- a/templates/pkg/resource/sdk_find_field_groups.go.tpl +++ b/templates/pkg/resource/sdk_find_field_groups.go.tpl @@ -39,9 +39,15 @@ func (rm *resourceManager) sdkFind{{ $fg.OperationID }}( rm.metrics.RecordAPICall("READ_ONE", "{{ $fg.OperationID }}", err) if err != nil { var awsErr smithy.APIError +{{- if $fg.ExceptionCode404 }} + if errors.As(err, &awsErr) && awsErr.ErrorCode() == "{{ $fg.ExceptionCode404 }}" { + return ko, nil + } +{{- else }} if errors.As(err, &awsErr) && awsErr.ErrorCode() == "{{ ResourceExceptionCode $.CRD 404 }}" { return ko, nil } +{{- end }} return ko, err } {{- if $hookCode := Hook $.CRD (print "sdk_read_" $fg.Names.Snake "_pre_set_output") }} diff --git a/templates/test/e2e/Makefile.tpl b/templates/test/e2e/Makefile.tpl new file mode 100644 index 00000000..4e7378f3 --- /dev/null +++ b/templates/test/e2e/Makefile.tpl @@ -0,0 +1,80 @@ +# Code generated by ack-generate. DO NOT EDIT. +# This file is regenerated on each run of `ack-generate e2e-tests`. + +SHELL := /bin/bash # Use bash syntax + +# Set up variables +GO111MODULE=on + +# Build ldflags +VERSION ?= "v0.0.0" +GITCOMMIT=$(shell git rev-parse HEAD) +BUILDDATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') +GO_LDFLAGS=-ldflags "-X main.version=$(VERSION) \ + -X main.buildHash=$(GITCOMMIT) \ + -X main.buildDate=$(BUILDDATE)" + +AWS_REGION ?= us-west-2 +E2E_TEST_NAMESPACE ?= default +CONTROLLER_HEALTHZ_ADDR ?= 0.0.0.0:8081 +CONTROLLER_LOG_FILE ?= controller.log + +.PHONY: all test local-test build-controller install-crds run-e2e + +all: test + +test: ## Run code tests + go test -v ./... + +local-test: ## Run code tests using go.local.mod file + go test -modfile=go.local.mod -v ./... + +build-controller: ## Build the controller binary + @echo "building controller binary..." + @go build ${GO_LDFLAGS} -o bin/controller cmd/controller/main.go + +install-crds: ## Install CRDs into the current cluster + @echo "installing CRDs..." + @kubectl apply -f config/crd/bases/ + @kubectl apply -f config/crd/common/bases/ 2>/dev/null || true + +run-e2e: build-controller install-crds ## Run generated Go e2e tests with local controller + @echo "starting controller locally (logs: $(CONTROLLER_LOG_FILE))..." + @./bin/controller \ + --aws-region $(AWS_REGION) \ + --watch-namespace $(E2E_TEST_NAMESPACE) \ + --enable-development-logging=true \ + --log-level debug \ + --enable-leader-election=false \ + --healthz-addr $(CONTROLLER_HEALTHZ_ADDR) > $(CONTROLLER_LOG_FILE) 2>&1 & \ + CONTROLLER_PID=$$!; \ + echo "controller started (PID: $$CONTROLLER_PID)"; \ + echo "waiting for controller to become ready..."; \ + for i in $$(seq 1 30); do \ + if curl -sf http://$(CONTROLLER_HEALTHZ_ADDR)/readyz > /dev/null 2>&1; then \ + echo "controller is ready"; \ + break; \ + fi; \ + if ! kill -0 $$CONTROLLER_PID 2>/dev/null; then \ + echo "ERROR: controller process exited unexpectedly"; \ + exit 1; \ + fi; \ + sleep 1; \ + done; \ + if ! curl -sf http://$(CONTROLLER_HEALTHZ_ADDR)/readyz > /dev/null 2>&1; then \ + echo "ERROR: controller did not become ready within 30s"; \ + kill $$CONTROLLER_PID 2>/dev/null; \ + exit 1; \ + fi; \ + echo "running e2e tests..."; \ + ACK_TEST_NAMESPACE=$(E2E_TEST_NAMESPACE) AWS_REGION=$(AWS_REGION) \ + go test -v -timeout 30m ./test/e2e-go/...; \ + TEST_EXIT=$$?; \ + echo "stopping controller (PID: $$CONTROLLER_PID)..."; \ + kill $$CONTROLLER_PID 2>/dev/null; \ + wait $$CONTROLLER_PID 2>/dev/null; \ + exit $$TEST_EXIT + +help: ## Show this help. + @grep -F -h "##" $(MAKEFILE_LIST) | grep -F -v grep | sed -e 's/\\$$//' \ + | awk -F'[:#]' '{print $$1 = sprintf("%-30s", $$1), $$4}' diff --git a/templates/test/e2e/main_test.go.tpl b/templates/test/e2e/main_test.go.tpl new file mode 100644 index 00000000..fd080913 --- /dev/null +++ b/templates/test/e2e/main_test.go.tpl @@ -0,0 +1,74 @@ +// Code generated by ack-generate. DO NOT EDIT. +// This file is regenerated on each run of `ack-generate e2e-tests`. + +package e2e + +import ( + "context" + "log" + "os" + "testing" + + "github.com/aws/aws-sdk-go-v2/config" + svcsdk "github.com/aws/aws-sdk-go-v2/service/{{ .ServicePackageName }}" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + + svcapitypes "github.com/aws-controllers-k8s/{{ .ControllerName }}-controller/apis/{{ .APIVersion }}" + acktest "github.com/aws-controllers-k8s/test-infra/pkg/e2e" +{{- if .TestConfig.Bootstrap }} + "github.com/aws-controllers-k8s/test-infra/pkg/e2e/bootstrap" +{{- end }} +) +{{ if .TestConfig.Bootstrap }} +var bootstrapResources *bootstrapState + +type bootstrapState struct { +{{- range .TestConfig.Bootstrap }} + {{ .Name }} *bootstrap.{{ .Type }}Resource +{{- end }} +} +{{ end }} +var awsClient *svcsdk.Client + +func TestMain(m *testing.M) { + scheme := runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + _ = svcapitypes.AddToScheme(scheme) + + acktest.SetScheme(scheme) + acktest.Setup() + + ctx := context.Background() + awsCfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + log.Fatalf("failed to load AWS config: %v", err) + } + awsClient = svcsdk.NewFromConfig(awsCfg) +{{ if .TestConfig.Bootstrap }} + clients, err := bootstrap.NewClients(ctx) + if err != nil { + log.Fatalf("failed to create bootstrap clients: %v", err) + } + + bootstrapResources = &bootstrapState{} +{{- range .TestConfig.Bootstrap }} + bootstrapResources.{{ .Name }} = bootstrap.New{{ .Type }}("{{ .Name }}", clients) +{{- end }} + + resources := []bootstrap.Resource{ +{{- range .TestConfig.Bootstrap }} + bootstrapResources.{{ .Name }}, +{{- end }} + } + if err := bootstrap.SetupAll(ctx, resources); err != nil { + log.Fatalf("bootstrap setup failed: %v", err) + } +{{ end }} + code := m.Run() +{{ if .TestConfig.Bootstrap }} + bootstrap.TeardownAll(ctx, resources) +{{ end }} + acktest.Teardown() + os.Exit(code) +} diff --git a/templates/test/e2e/resource_test.go.tpl b/templates/test/e2e/resource_test.go.tpl new file mode 100644 index 00000000..735537a1 --- /dev/null +++ b/templates/test/e2e/resource_test.go.tpl @@ -0,0 +1,142 @@ +// Code generated by ack-generate. DO NOT EDIT. +// This file is regenerated on each run of `ack-generate e2e-tests`. + +package e2e + +import ( + "context" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + svcsdk "github.com/aws/aws-sdk-go-v2/service/{{ .ServicePackageName }}" + svcsdktypes "github.com/aws/aws-sdk-go-v2/service/{{ .ServicePackageName }}/types" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + svcapitypes "github.com/aws-controllers-k8s/{{ .ControllerName }}-controller/apis/{{ .APIVersion }}" + acktest "github.com/aws-controllers-k8s/test-infra/pkg/e2e" +) + +// Ensure imports are used. +var ( + _ = aws.String + _ = svcsdk.New + _ = svcsdktypes.NoSuchBucket{} +) +{{ range $idx, $test := .Tests }} +func Test{{ $.CRD.Kind }}_{{ $test.TestNameCamel }}(t *testing.T) { + // t.Parallel() + ctx := context.Background() + name := acktest.RandomName("{{ $.CRD.Names.Snake | TruncateTo 12 }}", 40) + + t.Logf("test starting for {{ $.CRD.Kind }} with name %s", name) + + resource := &svcapitypes.{{ $.CRD.Kind }}{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: acktest.Namespace(), + }, + Spec: svcapitypes.{{ $.CRD.Kind }}Spec{ +{{- range $fieldName, $val := $test.FieldAssignments }} + {{ $fieldName }}: {{ $val }}, +{{- end }} + }, + } + + // Create +{{- range $fieldName, $val := $test.FieldAssignments }} + t.Logf(" {{ $fieldName }}: %s", acktest.FormatVal(resource.Spec.{{ $fieldName }})) +{{- end }} + t.Logf("creating {{ $.CRD.Kind }} %s in namespace %s", name, acktest.Namespace()) + createStart := time.Now() + err := acktest.CreateResource(ctx, resource) + require.NoError(t, err, "failed to create {{ $.CRD.Kind }}") + t.Logf("{{ $.CRD.Kind }} %s created successfully (took %s)", name, time.Since(createStart)) + t.Cleanup(func() { + t.Logf("deleting {{ $.CRD.Kind }} %s", name) + deleteStart := time.Now() + if err := acktest.DeleteResource(context.Background(), resource); err != nil { + t.Logf("warning: failed to delete {{ $.CRD.Kind }} %s: %v", name, err) + } else { + t.Logf("{{ $.CRD.Kind }} %s deleted (took %s)", name, time.Since(deleteStart)) + } + }) + + // Wait for sync + t.Logf("waiting for {{ $.CRD.Kind }} %s to become synced (timeout: {{ $test.CreateWaitDuration }})", name) + syncStart := time.Now() + err = acktest.WaitForCondition(ctx, resource, "ACK.ResourceSynced", "True", {{ $test.CreateWaitDuration }}) + require.NoError(t, err, "{{ $.CRD.Kind }} did not become synced") + t.Logf("{{ $.CRD.Kind }} %s synced (took %s)", name, time.Since(syncStart)) +{{ if $test.HasVerification }} + // Verify in AWS after create + t.Logf("verifying {{ $.CRD.Kind }} %s in AWS...", name) +{{- range $test.CreateVerifyCalls }} + { + resp, err := awsClient.{{ .OperationName }}(ctx, &svcsdk.{{ .InputTypeName }}{ +{{- range $member, $expr := .IdentifierAssignments }} + {{ $member }}: {{ $expr }}, +{{- end }} + }) + require.NoError(t, err, "{{ .OperationName }} failed") +{{- range .Assertions }} +{{- if .DerefFunc }} + require.Equal(t, {{ .Expected }}, {{ .DerefFunc }}(resp.{{ .ResponsePath }}), "{{ .Message }}") +{{- else }} + require.Equal(t, {{ .Expected }}, resp.{{ .ResponsePath }}, "{{ .Message }}") +{{- end }} +{{- end }} + } +{{- end }} + t.Logf("AWS verification passed for {{ $.CRD.Kind }} %s", name) +{{ end }} +{{ if $test.HasUpdate }} + // Update + err = acktest.GetResource(ctx, resource) + require.NoError(t, err, "failed to get {{ $.CRD.Kind }} before update") + updated := resource.DeepCopy() +{{- range $fieldName, $val := $test.UpdateAssignments }} + updated.Spec.{{ $fieldName }} = {{ $val }} +{{- end }} + t.Logf("updating {{ $.CRD.Kind }} %s:", name) +{{- range $fieldName, $val := $test.UpdateAssignments }} + t.Logf(" {{ $fieldName }}: %s", acktest.FormatVal(updated.Spec.{{ $fieldName }})) +{{- end }} + updateStart := time.Now() + err = acktest.UpdateResource(ctx, updated) + require.NoError(t, err, "failed to update {{ $.CRD.Kind }}") + t.Logf("{{ $.CRD.Kind }} %s update applied (took %s)", name, time.Since(updateStart)) + + time.Sleep(5 * time.Second) + + t.Logf("waiting for {{ $.CRD.Kind }} %s to re-sync after update (timeout: {{ $test.CreateWaitDuration }})", name) + resyncStart := time.Now() + err = acktest.WaitForCondition(ctx, updated, "ACK.ResourceSynced", "True", {{ $test.CreateWaitDuration }}) + require.NoError(t, err, "{{ $.CRD.Kind }} did not re-sync after update") + t.Logf("{{ $.CRD.Kind }} %s re-synced after update (took %s)", name, time.Since(resyncStart)) +{{ if gt (len $test.UpdateVerifyCalls) 0 }} + // Verify in AWS after update + t.Logf("verifying {{ $.CRD.Kind }} %s in AWS after update...", name) +{{- range $test.UpdateVerifyCalls }} + { + resp, err := awsClient.{{ .OperationName }}(ctx, &svcsdk.{{ .InputTypeName }}{ +{{- range $member, $expr := .IdentifierAssignments }} + {{ $member }}: {{ $expr }}, +{{- end }} + }) + require.NoError(t, err, "{{ .OperationName }} failed after update") +{{- range .Assertions }} +{{- if .DerefFunc }} + require.Equal(t, {{ .Expected }}, {{ .DerefFunc }}(resp.{{ .ResponsePath }}), "{{ .Message }}") +{{- else }} + require.Equal(t, {{ .Expected }}, resp.{{ .ResponsePath }}, "{{ .Message }}") +{{- end }} +{{- end }} + } +{{- end }} + t.Logf("AWS verification after update passed for {{ $.CRD.Kind }} %s", name) +{{ end }} +{{ end -}} +} +{{ end }}