diff --git a/.gitignore b/.gitignore index faaedadc..460311e2 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ go.work.sum .DS_Store *.log .vscode/settings.json +coverage.txt diff --git a/config_direct_field_tracking_test.go b/config_direct_field_tracking_test.go new file mode 100644 index 00000000..33743033 --- /dev/null +++ b/config_direct_field_tracking_test.go @@ -0,0 +1,157 @@ +package modular + +import ( + "strings" + "testing" + + "github.com/GoCodeAlone/modular/feeders" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// TestDirectFeederFieldTracking tests field tracking when calling feeder.Feed() directly +func TestDirectFeederFieldTracking(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + }{ + { + name: "basic environment variable tracking", + envVars: map[string]string{ + "APP_NAME": "Test App", + "APP_DEBUG": "true", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set up environment variables + for key, value := range tt.envVars { + t.Setenv(key, value) + } + + // Create logger that captures debug output + mockLogger := new(MockLogger) + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + // Create field tracker + tracker := NewDefaultFieldTracker() + tracker.SetLogger(mockLogger) + + // Create test configuration struct + type TestConfig struct { + App struct { + Name string `env:"APP_NAME"` + Debug bool `env:"APP_DEBUG"` + } + } + + config := &TestConfig{} + + // Create environment feeder with field tracking + envFeeder := feeders.NewEnvFeeder() + envFeeder.SetVerboseDebug(true, mockLogger) + + // Set up field tracking bridge + bridge := NewFieldTrackingBridge(tracker) + envFeeder.SetFieldTracker(bridge) + + // Feed configuration directly + err := envFeeder.Feed(config) + require.NoError(t, err) + + // Verify that config values were actually set + assert.Equal(t, "Test App", config.App.Name) + assert.True(t, config.App.Debug) + + // Verify that field populations were tracked + assert.NotEmpty(t, tracker.FieldPopulations, "Should have tracked field populations") + + // Print tracked populations for debugging + t.Logf("Tracked %d field populations:", len(tracker.FieldPopulations)) + for i, fp := range tracker.FieldPopulations { + t.Logf(" %d: %s -> %v (from %s:%s)", i, fp.FieldPath, fp.Value, fp.SourceType, fp.SourceKey) + } + }) + } +} + +// TestInstanceAwareDirectFieldTracking tests instance-aware field tracking with direct feeding +func TestInstanceAwareDirectFieldTracking(t *testing.T) { + // Set up environment variables for instance-aware tracking + envVars := map[string]string{ + "DB_PRIMARY_DRIVER": "postgres", + "DB_PRIMARY_DSN": "postgres://localhost/primary", + "DB_SECONDARY_DRIVER": "mysql", + "DB_SECONDARY_DSN": "mysql://localhost/secondary", + } + + for key, value := range envVars { + t.Setenv(key, value) + } + + // Create logger that captures debug output + mockLogger := new(MockLogger) + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + // Create field tracker + tracker := NewDefaultFieldTracker() + tracker.SetLogger(mockLogger) + + // Create test configuration structures + type ConnectionConfig struct { + Driver string `env:"DRIVER"` + DSN string `env:"DSN"` + } + + // Test the primary connection first + primaryConfig := &ConnectionConfig{} + + // Create instance-aware environment feeder + instanceAwareFeeder := feeders.NewInstanceAwareEnvFeeder(func(instanceKey string) string { + return "DB_" + strings.ToUpper(instanceKey) + "_" + }) + instanceAwareFeeder.SetVerboseDebug(true, mockLogger) + + // Set up field tracking bridge + bridge := NewFieldTrackingBridge(tracker) + instanceAwareFeeder.SetFieldTracker(bridge) + + // Feed primary configuration + err := instanceAwareFeeder.FeedKey("primary", primaryConfig) + require.NoError(t, err) + + // Verify that config values were actually set + assert.Equal(t, "postgres", primaryConfig.Driver) + assert.Equal(t, "postgres://localhost/primary", primaryConfig.DSN) + + // Test secondary connection + secondaryConfig := &ConnectionConfig{} + err = instanceAwareFeeder.FeedKey("secondary", secondaryConfig) + require.NoError(t, err) + + // Verify that config values were actually set + assert.Equal(t, "mysql", secondaryConfig.Driver) + assert.Equal(t, "mysql://localhost/secondary", secondaryConfig.DSN) + + // Verify that field populations were tracked + assert.NotEmpty(t, tracker.FieldPopulations, "Should have tracked field populations") + + // Print tracked populations for debugging + t.Logf("Tracked %d field populations:", len(tracker.FieldPopulations)) + for i, fp := range tracker.FieldPopulations { + t.Logf(" %d: %s -> %v (from %s:%s, instance:%s)", i, fp.FieldPath, fp.Value, fp.SourceType, fp.SourceKey, fp.InstanceKey) + } + + // Verify specific field populations + primaryDriverPop := tracker.GetFieldPopulation("Driver") + if primaryDriverPop != nil { + assert.Equal(t, "Driver", primaryDriverPop.FieldName) + assert.Equal(t, "env", primaryDriverPop.SourceType) + assert.Equal(t, "DB_PRIMARY_DRIVER", primaryDriverPop.SourceKey) + assert.Equal(t, "postgres", primaryDriverPop.Value) + assert.Equal(t, "primary", primaryDriverPop.InstanceKey) + } +} diff --git a/config_feeders.go b/config_feeders.go index 060b5c60..648f4963 100644 --- a/config_feeders.go +++ b/config_feeders.go @@ -1,19 +1,20 @@ package modular import ( - "github.com/golobby/config/v3" - "github.com/GoCodeAlone/modular/feeders" ) +// Feeder defines the interface for configuration feeders that provide configuration data. +type Feeder interface { + // Feed gets a struct and feeds it using configuration data. + Feed(structure interface{}) error +} + // ConfigFeeders provides a default set of configuration feeders for common use cases var ConfigFeeders = []Feeder{ feeders.NewEnvFeeder(), } -// Feeder aliases -type Feeder = config.Feeder - // ComplexFeeder extends the basic Feeder interface with additional functionality for complex configuration scenarios type ComplexFeeder interface { Feeder diff --git a/config_field_tracking.go b/config_field_tracking.go new file mode 100644 index 00000000..fc45db94 --- /dev/null +++ b/config_field_tracking.go @@ -0,0 +1,263 @@ +package modular + +import ( + "fmt" + "reflect" + "strings" +) + +// FieldTracker interface allows feeders to report which fields they populate +type FieldTracker interface { + // RecordFieldPopulation records that a field was populated by a feeder + RecordFieldPopulation(fp FieldPopulation) + + // SetLogger sets the logger for the tracker + SetLogger(logger Logger) +} + +// FieldPopulation represents a single field population event +type FieldPopulation struct { + FieldPath string // Full path to the field (e.g., "Connections.primary.DSN") + FieldName string // Name of the field + FieldType string // Type of the field + FeederType string // Type of feeder that populated it + SourceType string // Type of source (env, yaml, etc.) + SourceKey string // Source key that was used (e.g., "DB_PRIMARY_DSN") + Value interface{} // Value that was set + InstanceKey string // Instance key for instance-aware fields + SearchKeys []string // All keys that were searched for this field + FoundKey string // The key that was actually found +} + +// FieldTrackingFeeder interface allows feeders to support field tracking +type FieldTrackingFeeder interface { + // SetFieldTracker sets the field tracker for this feeder + SetFieldTracker(tracker FieldTracker) +} + +// DefaultFieldTracker is a basic implementation of FieldTracker +type DefaultFieldTracker struct { + FieldPopulations []FieldPopulation + logger Logger +} + +// NewDefaultFieldTracker creates a new default field tracker +func NewDefaultFieldTracker() *DefaultFieldTracker { + return &DefaultFieldTracker{ + FieldPopulations: make([]FieldPopulation, 0), + } +} + +// RecordFieldPopulation records a field population event +func (t *DefaultFieldTracker) RecordFieldPopulation(fp FieldPopulation) { + t.FieldPopulations = append(t.FieldPopulations, fp) + if t.logger != nil { + t.logger.Debug("Field populated", + "fieldPath", fp.FieldPath, + "fieldName", fp.FieldName, + "fieldType", fp.FieldType, + "feederType", fp.FeederType, + "sourceType", fp.SourceType, + "sourceKey", fp.SourceKey, + "value", fp.Value, + "instanceKey", fp.InstanceKey, + "searchKeys", strings.Join(fp.SearchKeys, ", "), + "foundKey", fp.FoundKey, + ) + } +} + +// SetLogger sets the logger for the tracker +func (t *DefaultFieldTracker) SetLogger(logger Logger) { + t.logger = logger +} + +// GetFieldPopulation returns the population info for a specific field path +func (t *DefaultFieldTracker) GetFieldPopulation(fieldPath string) *FieldPopulation { + for _, fp := range t.FieldPopulations { + if fp.FieldPath == fieldPath { + return &fp + } + } + return nil +} + +// GetPopulationsByFeeder returns all field populations by a specific feeder type +func (t *DefaultFieldTracker) GetPopulationsByFeeder(feederType string) []FieldPopulation { + var result []FieldPopulation + for _, fp := range t.FieldPopulations { + if fp.FeederType == feederType { + result = append(result, fp) + } + } + return result +} + +// GetPopulationsBySource returns all field populations by a specific source type +func (t *DefaultFieldTracker) GetPopulationsBySource(sourceType string) []FieldPopulation { + var result []FieldPopulation + for _, fp := range t.FieldPopulations { + if fp.SourceType == sourceType { + result = append(result, fp) + } + } + return result +} + +// StructStateDiffer captures before/after states to determine field changes +type StructStateDiffer struct { + beforeState map[string]interface{} + afterState map[string]interface{} + tracker FieldTracker + logger Logger +} + +// NewStructStateDiffer creates a new struct state differ +func NewStructStateDiffer(tracker FieldTracker, logger Logger) *StructStateDiffer { + return &StructStateDiffer{ + beforeState: make(map[string]interface{}), + afterState: make(map[string]interface{}), + tracker: tracker, + logger: logger, + } +} + +// CaptureBeforeState captures the state before feeder processing +func (d *StructStateDiffer) CaptureBeforeState(structure interface{}, prefix string) { + d.captureState(structure, prefix, d.beforeState) + if d.logger != nil { + d.logger.Debug("Captured before state", "prefix", prefix, "fieldCount", len(d.beforeState)) + } +} + +// CaptureAfterStateAndDiff captures the state after feeder processing and computes diffs +func (d *StructStateDiffer) CaptureAfterStateAndDiff(structure interface{}, prefix string, feederType, sourceType string) { + d.captureState(structure, prefix, d.afterState) + if d.logger != nil { + d.logger.Debug("Captured after state", "prefix", prefix, "fieldCount", len(d.afterState)) + } + + // Compute and record differences + d.computeAndRecordDiffs(feederType, sourceType, prefix) +} + +// captureState recursively captures all field values in a structure +func (d *StructStateDiffer) captureState(structure interface{}, prefix string, state map[string]interface{}) { + rv := reflect.ValueOf(structure) + if rv.Kind() == reflect.Ptr { + if rv.IsNil() { + return + } + rv = rv.Elem() + } + + if rv.Kind() != reflect.Struct { + return + } + + d.captureStructFields(rv, prefix, state) +} + +// captureStructFields recursively captures all field values in a struct +func (d *StructStateDiffer) captureStructFields(rv reflect.Value, prefix string, state map[string]interface{}) { + rt := rv.Type() + + for i := 0; i < rv.NumField(); i++ { + field := rv.Field(i) + fieldType := rt.Field(i) + + if !field.CanInterface() { + continue // Skip unexported fields + } + + fieldPath := fieldType.Name + if prefix != "" { + fieldPath = prefix + "." + fieldType.Name + } + + switch field.Kind() { + case reflect.Struct: + d.captureStructFields(field, fieldPath, state) + case reflect.Ptr: + if !field.IsNil() && field.Elem().Kind() == reflect.Struct { + d.captureStructFields(field.Elem(), fieldPath, state) + } else if !field.IsNil() { + state[fieldPath] = field.Elem().Interface() + } + case reflect.Map: + if !field.IsNil() { + for _, key := range field.MapKeys() { + mapValue := field.MapIndex(key) + mapFieldPath := fieldPath + "." + key.String() + if mapValue.Kind() == reflect.Struct { + d.captureStructFields(mapValue, mapFieldPath, state) + } else if mapValue.Kind() == reflect.Ptr && !mapValue.IsNil() && mapValue.Elem().Kind() == reflect.Struct { + d.captureStructFields(mapValue.Elem(), mapFieldPath, state) + } else { + state[mapFieldPath] = mapValue.Interface() + } + } + } + case reflect.Invalid, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128, reflect.Array, + reflect.Chan, reflect.Func, reflect.Interface, reflect.Slice, reflect.String, reflect.UnsafePointer: + state[fieldPath] = field.Interface() + } + } +} + +// computeAndRecordDiffs computes differences and records field populations +func (d *StructStateDiffer) computeAndRecordDiffs(feederType, sourceType, instanceKey string) { + for fieldPath, afterValue := range d.afterState { + beforeValue, existed := d.beforeState[fieldPath] + + // Check if field changed (either new or value changed) + if !existed || !reflect.DeepEqual(beforeValue, afterValue) { + // Parse field path to get field name and type + parts := strings.Split(fieldPath, ".") + fieldName := parts[len(parts)-1] + + // Determine field type + fieldTypeStr := fmt.Sprintf("%T", afterValue) + + // Create field population record + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldName, + FieldType: fieldTypeStr, + FeederType: feederType, + SourceType: sourceType, + SourceKey: "detected_by_diff", // Will be enhanced by feeders that support direct tracking + Value: afterValue, + InstanceKey: instanceKey, + SearchKeys: []string{}, // Will be enhanced by feeders that support direct tracking + FoundKey: "detected_by_diff", + } + + if d.tracker != nil { + d.tracker.RecordFieldPopulation(fp) + } + + if d.logger != nil { + d.logger.Debug("Detected field change via diff", + "fieldPath", fieldPath, + "fieldName", fieldName, + "fieldType", fieldTypeStr, + "feederType", feederType, + "sourceType", sourceType, + "value", afterValue, + "instanceKey", instanceKey, + "existed", existed, + "beforeValue", beforeValue, + ) + } + } + } +} + +// Reset clears the captured states for reuse +func (d *StructStateDiffer) Reset() { + d.beforeState = make(map[string]interface{}) + d.afterState = make(map[string]interface{}) +} diff --git a/config_field_tracking_implementation_test.go b/config_field_tracking_implementation_test.go new file mode 100644 index 00000000..3c58df46 --- /dev/null +++ b/config_field_tracking_implementation_test.go @@ -0,0 +1,276 @@ +package modular + +import ( + "strings" + "testing" + + "github.com/GoCodeAlone/modular/feeders" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// FieldTrackingBridge bridges between the main package FieldTracker interface +// and the feeders package FieldTracker interface to avoid circular imports +type FieldTrackingBridge struct { + mainTracker FieldTracker +} + +// NewFieldTrackingBridge creates a bridge between the interfaces +func NewFieldTrackingBridge(tracker FieldTracker) *FieldTrackingBridge { + return &FieldTrackingBridge{mainTracker: tracker} +} + +// RecordFieldPopulation bridges field population records +func (b *FieldTrackingBridge) RecordFieldPopulation(fp feeders.FieldPopulation) { + // Convert from feeders.FieldPopulation to main.FieldPopulation + mainFP := FieldPopulation{ + FieldPath: fp.FieldPath, + FieldName: fp.FieldName, + FieldType: fp.FieldType, + FeederType: fp.FeederType, + SourceType: fp.SourceType, + SourceKey: fp.SourceKey, + Value: fp.Value, + InstanceKey: fp.InstanceKey, + SearchKeys: fp.SearchKeys, + FoundKey: fp.FoundKey, + } + b.mainTracker.RecordFieldPopulation(mainFP) +} + +// SetupFieldTrackingForFeeders sets up field tracking for feeders that support it +func SetupFieldTrackingForFeeders(cfgFeeders []Feeder, tracker FieldTracker) { + bridge := NewFieldTrackingBridge(tracker) + + for _, feeder := range cfgFeeders { + // Use type assertion for the specific feeder types we know about + switch f := feeder.(type) { + case interface{ SetFieldTracker(feeders.FieldTracker) }: + f.SetFieldTracker(bridge) + } + } +} + +// TestEnhancedFieldTracking tests the enhanced field tracking functionality +func TestEnhancedFieldTracking(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + expected map[string]FieldPopulation + }{ + { + name: "basic environment variable tracking", + envVars: map[string]string{ + "APP_NAME": "Test App", + "APP_DEBUG": "true", + }, + expected: map[string]FieldPopulation{ + "AppName": { + FieldPath: "AppName", + FieldName: "AppName", + FieldType: "string", + FeederType: "*feeders.EnvFeeder", + SourceType: "env", + SourceKey: "APP_NAME", + Value: "Test App", + InstanceKey: "", + SearchKeys: []string{"APP_NAME"}, + FoundKey: "APP_NAME", + }, + "Debug": { + FieldPath: "Debug", + FieldName: "Debug", + FieldType: "bool", + FeederType: "*feeders.EnvFeeder", + SourceType: "env", + SourceKey: "APP_DEBUG", + Value: true, + InstanceKey: "", + SearchKeys: []string{"APP_DEBUG"}, + FoundKey: "APP_DEBUG", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set up environment variables + for key, value := range tt.envVars { + t.Setenv(key, value) + } + + // Create logger that captures debug output + mockLogger := new(MockLogger) + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + // Create field tracker + tracker := NewDefaultFieldTracker() + tracker.SetLogger(mockLogger) + + // Create test configuration struct + type TestConfig struct { + AppName string `env:"APP_NAME"` + Debug bool `env:"APP_DEBUG"` + } + + config := &TestConfig{} + + // Create configuration builder with field tracking + cfgBuilder := NewConfig() + cfgBuilder.SetVerboseDebug(true, mockLogger) + cfgBuilder.SetFieldTracker(tracker) + + // Add environment feeder - the AddFeeder method should set up field tracking automatically + envFeeder := feeders.NewEnvFeeder() + cfgBuilder.AddFeeder(envFeeder) + + // Add the configuration structure + cfgBuilder.AddStructKey("test", config) + + // Feed configuration + err := cfgBuilder.Feed() + require.NoError(t, err) + + // Verify the config was populated correctly + assert.Equal(t, "Test App", config.AppName) + assert.True(t, config.Debug) + + // Field tracking should now work with the bridge + // Verify field populations were tracked + assert.GreaterOrEqual(t, len(tracker.FieldPopulations), 2, "Expected at least 2 field populations, got %d", len(tracker.FieldPopulations)) + + // Check for specific field populations + appNamePop := tracker.GetFieldPopulation("AppName") + if assert.NotNil(t, appNamePop, "AppName field population should be tracked") { + assert.Equal(t, "Test App", appNamePop.Value) + assert.Equal(t, "env", appNamePop.SourceType) + assert.Equal(t, "APP_NAME", appNamePop.SourceKey) + } + + debugPop := tracker.GetFieldPopulation("Debug") + if assert.NotNil(t, debugPop, "Debug field population should be tracked") { + assert.Equal(t, true, debugPop.Value) + assert.Equal(t, "env", debugPop.SourceType) + assert.Equal(t, "APP_DEBUG", debugPop.SourceKey) + } + }) + } +} + +// TestInstanceAwareFieldTracking tests instance-aware field tracking +func TestInstanceAwareFieldTracking(t *testing.T) { + // Set up environment variables for instance-aware tracking + envVars := map[string]string{ + "DB_PRIMARY_DRIVER": "postgres", + "DB_PRIMARY_DSN": "postgres://localhost/primary", + "DB_SECONDARY_DRIVER": "mysql", + "DB_SECONDARY_DSN": "mysql://localhost/secondary", + } + + for key, value := range envVars { + t.Setenv(key, value) + } + + // Create logger that captures debug output + mockLogger := new(MockLogger) + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + // Create field tracker + tracker := NewDefaultFieldTracker() + tracker.SetLogger(mockLogger) + + // Create test configuration structures + type ConnectionConfig struct { + Driver string `env:"DRIVER"` + DSN string `env:"DSN"` + } + + type DBConfig struct { + Connections map[string]ConnectionConfig + } + + dbConfig := &DBConfig{ + Connections: map[string]ConnectionConfig{ + "primary": {}, + "secondary": {}, + }, + } + + // Create configuration builder with field tracking + cfgBuilder := NewConfig() + cfgBuilder.SetVerboseDebug(true, mockLogger) + cfgBuilder.SetFieldTracker(tracker) + + // Add instance-aware environment feeder + instanceAwareFeeder := feeders.NewInstanceAwareEnvFeeder(func(instanceKey string) string { + return "DB_" + strings.ToUpper(instanceKey) + "_" + }) + cfgBuilder.AddFeeder(instanceAwareFeeder) + + // Add the configuration structure + cfgBuilder.AddStructKey("db", dbConfig) + + // Feed configuration + err := cfgBuilder.Feed() + require.NoError(t, err) + + // Now use FeedInstances specifically for the connections map + err = instanceAwareFeeder.FeedInstances(dbConfig.Connections) + require.NoError(t, err) + + // Verify that config values were actually set + assert.Equal(t, "postgres", dbConfig.Connections["primary"].Driver) + assert.Equal(t, "postgres://localhost/primary", dbConfig.Connections["primary"].DSN) + assert.Equal(t, "mysql", dbConfig.Connections["secondary"].Driver) + assert.Equal(t, "mysql://localhost/secondary", dbConfig.Connections["secondary"].DSN) + + // Field tracking should now work - verify we have tracked populations + assert.GreaterOrEqual(t, len(tracker.FieldPopulations), 4, "Should track at least 4 field populations (2 fields × 2 instances)") + + // Check for specific field populations + var primaryDriverPop, primaryDSNPop, secondaryDriverPop, secondaryDSNPop *FieldPopulation + for i := range tracker.FieldPopulations { + pop := &tracker.FieldPopulations[i] + if pop.FieldName == "Driver" && pop.InstanceKey == "primary" { + primaryDriverPop = pop + } else if pop.FieldName == "DSN" && pop.InstanceKey == "primary" { + primaryDSNPop = pop + } else if pop.FieldName == "Driver" && pop.InstanceKey == "secondary" { + secondaryDriverPop = pop + } else if pop.FieldName == "DSN" && pop.InstanceKey == "secondary" { + secondaryDSNPop = pop + } + } + + // Verify primary instance tracking + if assert.NotNil(t, primaryDriverPop, "Primary driver field population should be tracked") { + assert.Equal(t, "postgres", primaryDriverPop.Value) + assert.Equal(t, "env", primaryDriverPop.SourceType) + assert.Equal(t, "DB_PRIMARY_DRIVER", primaryDriverPop.SourceKey) + assert.Equal(t, "primary", primaryDriverPop.InstanceKey) + } + + if assert.NotNil(t, primaryDSNPop, "Primary DSN field population should be tracked") { + assert.Equal(t, "postgres://localhost/primary", primaryDSNPop.Value) + assert.Equal(t, "env", primaryDSNPop.SourceType) + assert.Equal(t, "DB_PRIMARY_DSN", primaryDSNPop.SourceKey) + assert.Equal(t, "primary", primaryDSNPop.InstanceKey) + } + + // Verify secondary instance tracking + if assert.NotNil(t, secondaryDriverPop, "Secondary driver field population should be tracked") { + assert.Equal(t, "mysql", secondaryDriverPop.Value) + assert.Equal(t, "env", secondaryDriverPop.SourceType) + assert.Equal(t, "DB_SECONDARY_DRIVER", secondaryDriverPop.SourceKey) + assert.Equal(t, "secondary", secondaryDriverPop.InstanceKey) + } + + if assert.NotNil(t, secondaryDSNPop, "Secondary DSN field population should be tracked") { + assert.Equal(t, "mysql://localhost/secondary", secondaryDSNPop.Value) + assert.Equal(t, "env", secondaryDSNPop.SourceType) + assert.Equal(t, "DB_SECONDARY_DSN", secondaryDSNPop.SourceKey) + assert.Equal(t, "secondary", secondaryDSNPop.InstanceKey) + } +} diff --git a/config_field_tracking_test.go b/config_field_tracking_test.go new file mode 100644 index 00000000..5abfc585 --- /dev/null +++ b/config_field_tracking_test.go @@ -0,0 +1,586 @@ +package modular + +import ( + "os" + "reflect" + "testing" + + "github.com/GoCodeAlone/modular/feeders" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// ConfigFieldTracker tracks which fields are populated by which feeders +type ConfigFieldTracker struct { + FieldPopulations []FieldPopulation + logger Logger +} + +// NewConfigFieldTracker creates a new field tracker +func NewConfigFieldTracker(logger Logger) *ConfigFieldTracker { + return &ConfigFieldTracker{ + FieldPopulations: make([]FieldPopulation, 0), + logger: logger, + } +} + +// RecordFieldPopulation records that a field was populated +func (t *ConfigFieldTracker) RecordFieldPopulation(fp FieldPopulation) { + t.FieldPopulations = append(t.FieldPopulations, fp) + if t.logger != nil { + t.logger.Debug("Field populated", + "fieldPath", fp.FieldPath, + "fieldName", fp.FieldName, + "fieldType", fp.FieldType, + "feederType", fp.FeederType, + "sourceType", fp.SourceType, + "sourceKey", fp.SourceKey, + "value", fp.Value, + "instanceKey", fp.InstanceKey, + ) + } +} + +// GetFieldPopulation returns the population info for a specific field path +func (t *ConfigFieldTracker) GetFieldPopulation(fieldPath string) *FieldPopulation { + for _, fp := range t.FieldPopulations { + if fp.FieldPath == fieldPath { + return &fp + } + } + return nil +} + +// GetPopulationsByFeeder returns all field populations by a specific feeder type +func (t *ConfigFieldTracker) GetPopulationsByFeeder(feederType string) []FieldPopulation { + var result []FieldPopulation + for _, fp := range t.FieldPopulations { + if fp.FeederType == feederType { + result = append(result, fp) + } + } + return result +} + +// GetPopulationsBySource returns all field populations by a specific source type +func (t *ConfigFieldTracker) GetPopulationsBySource(sourceType string) []FieldPopulation { + var result []FieldPopulation + for _, fp := range t.FieldPopulations { + if fp.SourceType == sourceType { + result = append(result, fp) + } + } + return result +} + +// TestFieldLevelPopulationTracking tests that we can track exactly which fields +// are populated by which feeders with full visibility into the population process +func TestFieldLevelPopulationTracking(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + yamlData string + expected map[string]FieldPopulation // fieldPath -> expected population + }{ + { + name: "environment variable population tracking", + envVars: map[string]string{ + "APP_NAME": "Test App", + "APP_DEBUG": "true", + "DB_PRIMARY_DRIVER": "postgres", + "DB_PRIMARY_DSN": "postgres://localhost/primary", + "DB_SECONDARY_DRIVER": "mysql", + "DB_SECONDARY_DSN": "mysql://localhost/secondary", + }, + expected: map[string]FieldPopulation{ + "AppName": { + FieldPath: "AppName", + FieldName: "AppName", + FieldType: "string", + FeederType: "*feeders.EnvFeeder", + SourceType: "env", + SourceKey: "APP_NAME", + Value: "Test App", + InstanceKey: "", + }, + "Debug": { + FieldPath: "Debug", + FieldName: "Debug", + FieldType: "bool", + FeederType: "*feeders.EnvFeeder", + SourceType: "env", + SourceKey: "APP_DEBUG", + Value: true, + InstanceKey: "", + }, + "Connections.primary.Driver": { + FieldPath: "Connections.primary.Driver", + FieldName: "Driver", + FieldType: "string", + FeederType: "*feeders.InstanceAwareEnvFeeder", + SourceType: "env", + SourceKey: "DB_PRIMARY_DRIVER", + Value: "postgres", + InstanceKey: "primary", + }, + "Connections.primary.DSN": { + FieldPath: "Connections.primary.DSN", + FieldName: "DSN", + FieldType: "string", + FeederType: "*feeders.InstanceAwareEnvFeeder", + SourceType: "env", + SourceKey: "DB_PRIMARY_DSN", + Value: "postgres://localhost/primary", + InstanceKey: "primary", + }, + "Connections.secondary.Driver": { + FieldPath: "Connections.secondary.Driver", + FieldName: "Driver", + FieldType: "string", + FeederType: "*feeders.InstanceAwareEnvFeeder", + SourceType: "env", + SourceKey: "DB_SECONDARY_DRIVER", + Value: "mysql", + InstanceKey: "secondary", + }, + "Connections.secondary.DSN": { + FieldPath: "Connections.secondary.DSN", + FieldName: "DSN", + FieldType: "string", + FeederType: "*feeders.InstanceAwareEnvFeeder", + SourceType: "env", + SourceKey: "DB_SECONDARY_DSN", + Value: "mysql://localhost/secondary", + InstanceKey: "secondary", + }, + }, + }, + { + name: "mixed yaml and environment population tracking", + envVars: map[string]string{ + "APP_NAME": "Test App", + "DB_PRIMARY_DRIVER": "postgres", + "DB_PRIMARY_DSN": "postgres://localhost/primary", + }, + yamlData: ` +debug: false +connections: + secondary: + driver: "mysql" + dsn: "mysql://localhost/secondary" +`, + expected: map[string]FieldPopulation{ + "AppName": { + FieldPath: "AppName", + FieldName: "AppName", + FieldType: "string", + FeederType: "*feeders.EnvFeeder", + SourceType: "env", + SourceKey: "APP_NAME", + Value: "Test App", + InstanceKey: "", + }, + "Debug": { + FieldPath: "Debug", + FieldName: "Debug", + FieldType: "bool", + FeederType: "*feeders.YamlFeeder", + SourceType: "yaml", + SourceKey: "debug", + Value: false, + InstanceKey: "", + }, + "Connections.primary.Driver": { + FieldPath: "Connections.primary.Driver", + FieldName: "Driver", + FieldType: "string", + FeederType: "*feeders.InstanceAwareEnvFeeder", + SourceType: "env", + SourceKey: "DB_PRIMARY_DRIVER", + Value: "postgres", + InstanceKey: "primary", + }, + "Connections.primary.DSN": { + FieldPath: "Connections.primary.DSN", + FieldName: "DSN", + FieldType: "string", + FeederType: "*feeders.InstanceAwareEnvFeeder", + SourceType: "env", + SourceKey: "DB_PRIMARY_DSN", + Value: "postgres://localhost/primary", + InstanceKey: "primary", + }, + "Connections.secondary.Driver": { + FieldPath: "Connections.secondary.Driver", + FieldName: "Driver", + FieldType: "string", + FeederType: "*feeders.YamlFeeder", + SourceType: "yaml", + SourceKey: "connections.secondary.driver", + Value: "mysql", + InstanceKey: "secondary", + }, + "Connections.secondary.DSN": { + FieldPath: "Connections.secondary.DSN", + FieldName: "DSN", + FieldType: "string", + FeederType: "*feeders.YamlFeeder", + SourceType: "yaml", + SourceKey: "connections.secondary.dsn", + Value: "mysql://localhost/secondary", + InstanceKey: "secondary", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set up environment variables + for key, value := range tt.envVars { + os.Setenv(key, value) + } + defer func() { + for key := range tt.envVars { + os.Unsetenv(key) + } + }() + + // Create logger that captures debug output + mockLogger := new(MockLogger) + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + // Create field tracker + _ = NewConfigFieldTracker(mockLogger) + + // Create configuration structures with tracking + type TestAppConfig struct { + AppName string `env:"APP_NAME"` + Debug bool `env:"APP_DEBUG"` + } + + appConfig := &TestAppConfig{} + + // Create field tracker + tracker := NewDefaultFieldTracker() + tracker.SetLogger(mockLogger) + + // Create configuration builder with field tracking + cfg := NewConfig() + cfg.SetVerboseDebug(true, mockLogger) + cfg.SetFieldTracker(tracker) + + // Add environment feeder + envFeeder := feeders.NewEnvFeeder() + cfg.AddFeeder(envFeeder) + + // Add the configuration structure + cfg.AddStructKey("app", appConfig) + + // Feed configuration + err := cfg.Feed() + require.NoError(t, err) + + // Verify that configuration was populated + if tt.name == "environment_variable_population_tracking" { + assert.Equal(t, "Test App", appConfig.AppName) + assert.True(t, appConfig.Debug) + + // Verify field tracking captured the populations + populations := tracker.FieldPopulations + assert.GreaterOrEqual(t, len(populations), 2, "Should track at least 2 field populations") + + // Find specific field populations + appNamePop := tracker.GetFieldPopulation("AppName") + if assert.NotNil(t, appNamePop, "AppName field should be tracked") { + assert.Equal(t, "Test App", appNamePop.Value) + assert.Equal(t, "env", appNamePop.SourceType) + assert.Equal(t, "APP_NAME", appNamePop.SourceKey) + } + + debugPop := tracker.GetFieldPopulation("Debug") + if assert.NotNil(t, debugPop, "Debug field should be tracked") { + assert.Equal(t, true, debugPop.Value) + assert.Equal(t, "env", debugPop.SourceType) + assert.Equal(t, "APP_DEBUG", debugPop.SourceKey) + } + } else { + // For mixed YAML scenarios, skip until YAML feeder supports field tracking + t.Skip("Mixed YAML and environment field tracking requires YAML feeder field tracking support") + } + }) + } +} + +// TestDetailedInstanceAwareFieldTracking tests specific instance-aware field tracking +// for the database module scenario that's causing issues for the user +func TestDetailedInstanceAwareFieldTracking(t *testing.T) { + // Set up environment variables exactly as user would + envVars := map[string]string{ + "DB_PRIMARY_DRIVER": "postgres", + "DB_PRIMARY_DSN": "postgres://user:pass@localhost:5432/primary_db", + "DB_PRIMARY_MAX_CONNS": "10", + "DB_SECONDARY_DRIVER": "mysql", + "DB_SECONDARY_DSN": "mysql://user:pass@localhost:3306/secondary_db", + "DB_SECONDARY_MAX_CONNS": "5", + } + + for key, value := range envVars { + os.Setenv(key, value) + } + defer func() { + for key := range envVars { + os.Unsetenv(key) + } + }() + + // Create mock logger to capture verbose output + mockLogger := new(MockLogger) + debugLogs := make([][]interface{}, 0) + mockLogger.On("Debug", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + debugLogs = append(debugLogs, args) + }).Return() + + // Create field tracker + _ = NewConfigFieldTracker(mockLogger) + + // This test should verify that we can track exactly: + // 1. Which environment variables are searched for each field + // 2. Which ones are found vs not found + // 3. Which feeder populated each field + // 4. The exact source key that was used + // 5. The instance key for instance-aware fields + + t.Skip("Detailed instance-aware field tracking not yet implemented") + + // After implementation, we should be able to verify: + // 1. That DB_PRIMARY_DSN was found and populated the primary.DSN field + // 2. That DB_SECONDARY_DSN was found and populated the secondary.DSN field + // 3. The exact search pattern used for each field + // 4. Whether any fields failed to populate and why + + // expectedSearches := []string{ + // "DB_PRIMARY_DRIVER", "DB_PRIMARY_DSN", "DB_PRIMARY_MAX_CONNS", + // "DB_SECONDARY_DRIVER", "DB_SECONDARY_DSN", "DB_SECONDARY_MAX_CONNS", + // } + + // expectedPopulations := map[string]FieldPopulation{ + // "Connections.primary.DSN": { + // FieldPath: "Connections.primary.DSN", + // SourceKey: "DB_PRIMARY_DSN", + // Value: "postgres://user:pass@localhost:5432/primary_db", + // InstanceKey: "primary", + // }, + // "Connections.secondary.DSN": { + // FieldPath: "Connections.secondary.DSN", + // SourceKey: "DB_SECONDARY_DSN", + // Value: "mysql://user:pass@localhost:3306/secondary_db", + // InstanceKey: "secondary", + // }, + // } +} + +// TestConfigDiffBasedFieldTracking tests an alternative approach using before/after diffs +// to determine which fields were populated by which feeders +func TestConfigDiffBasedFieldTracking(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + expectedFieldDiffs map[string]interface{} // field path -> expected new value + }{ + { + name: "basic field diff tracking", + envVars: map[string]string{ + "APP_NAME": "Test App", + "APP_DEBUG": "true", + }, + expectedFieldDiffs: map[string]interface{}{ + "AppName": "Test App", + "Debug": true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set up environment + for key, value := range tt.envVars { + os.Setenv(key, value) + } + defer func() { + for key := range tt.envVars { + os.Unsetenv(key) + } + }() + + type TestConfig struct { + AppName string `env:"APP_NAME"` + Debug bool `env:"APP_DEBUG"` + } + + _ = &TestConfig{} + + // This test should verify that we can use before/after comparison + // to determine which fields were populated by which feeders + t.Skip("Diff-based field tracking not yet implemented") + + // After implementation: + // beforeState := captureStructState(config) + // err := feedWithDiffTracking(config) + // require.NoError(t, err) + // afterState := captureStructState(config) + // diffs := computeFieldDiffs(beforeState, afterState) + + // for fieldPath, expectedValue := range tt.expectedFieldDiffs { + // assert.Contains(t, diffs, fieldPath) + // assert.Equal(t, expectedValue, diffs[fieldPath]) + // } + }) + } +} + +// TestVerboseDebugFieldVisibility tests that verbose debug logging provides +// sufficient visibility into field population for troubleshooting +func TestVerboseDebugFieldVisibility(t *testing.T) { + // Set up test environment + os.Setenv("TEST_FIELD", "test_value") + defer os.Unsetenv("TEST_FIELD") + + type TestConfig struct { + TestField string `env:"TEST_FIELD"` + } + + _ = &TestConfig{} + mockLogger := new(MockLogger) + + // Capture all debug log calls + var debugCalls [][]interface{} + mockLogger.On("Debug", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + debugCalls = append(debugCalls, args) + }).Return() + + // This test should verify that verbose debug logging includes: + // 1. Field name being processed + // 2. Environment variable name being searched + // 3. Whether the environment variable was found + // 4. The value that was set + // 5. Success/failure of field population + + t.Skip("Enhanced verbose debug field visibility not yet implemented") + + // After implementation, we should be able to verify debug logs contain: + // - "Processing field: TestField" + // - "Looking up environment variable: TEST_FIELD" + // - "Environment variable found: TEST_FIELD=test_value" + // - "Successfully set field value: TestField=test_value" + + // requiredLogMessages := []string{ + // "Processing field", + // "Looking up environment variable", + // "Environment variable found", + // "Successfully set field value", + // } + + // for _, requiredMsg := range requiredLogMessages { + // found := false + // for _, call := range debugCalls { + // if len(call) > 0 { + // if msg, ok := call[0].(string); ok && strings.Contains(msg, requiredMsg) { + // found = true + // break + // } + // } + // } + // assert.True(t, found, "Required debug message not found: %s", requiredMsg) + // } +} + +// These are helper functions for the unimplemented diff-based tracking approach +// They will be used once the implementation is complete + +// StructState represents the state of a struct at a point in time +type StructState struct { + Fields map[string]interface{} // field path -> value +} + +// captureStructState captures the current state of all fields in a struct +func captureStructState(structure interface{}) *StructState { + state := &StructState{ + Fields: make(map[string]interface{}), + } + + rv := reflect.ValueOf(structure) + if rv.Kind() == reflect.Ptr { + rv = rv.Elem() + } + + captureStructFields(rv, "", state.Fields) + return state +} + +// captureStructFields recursively captures all field values +func captureStructFields(rv reflect.Value, prefix string, fields map[string]interface{}) { + rt := rv.Type() + + for i := 0; i < rv.NumField(); i++ { + field := rv.Field(i) + fieldType := rt.Field(i) + + fieldPath := fieldType.Name + if prefix != "" { + fieldPath = prefix + "." + fieldType.Name + } + + switch field.Kind() { + case reflect.Struct: + captureStructFields(field, fieldPath, fields) + case reflect.Ptr: + if !field.IsNil() && field.Elem().Kind() == reflect.Struct { + captureStructFields(field.Elem(), fieldPath, fields) + } else if !field.IsNil() { + fields[fieldPath] = field.Elem().Interface() + } + case reflect.Map: + if !field.IsNil() { + for _, key := range field.MapKeys() { + mapValue := field.MapIndex(key) + mapFieldPath := fieldPath + "." + key.String() + if mapValue.Kind() == reflect.Struct { + captureStructFields(mapValue, mapFieldPath, fields) + } else { + fields[mapFieldPath] = mapValue.Interface() + } + } + } + case reflect.Invalid, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128, reflect.Array, + reflect.Chan, reflect.Func, reflect.Interface, reflect.Slice, reflect.String, reflect.UnsafePointer: + fields[fieldPath] = field.Interface() + default: + fields[fieldPath] = field.Interface() + } + } +} + +// computeFieldDiffs computes the differences between two struct states +func computeFieldDiffs(before, after *StructState) map[string]interface{} { + diffs := make(map[string]interface{}) + + // Find fields that changed + for fieldPath, afterValue := range after.Fields { + beforeValue, existed := before.Fields[fieldPath] + if !existed || !reflect.DeepEqual(beforeValue, afterValue) { + diffs[fieldPath] = afterValue + } + } + + return diffs +} + +var ( + // These prevent unused function warnings - they'll be used once implementation is complete + _ = captureStructState + _ = computeFieldDiffs +) diff --git a/config_full_flow_field_tracking_test.go b/config_full_flow_field_tracking_test.go new file mode 100644 index 00000000..8da63f59 --- /dev/null +++ b/config_full_flow_field_tracking_test.go @@ -0,0 +1,423 @@ +package modular + +import ( + "os" + "strings" + "testing" + + "github.com/GoCodeAlone/modular/feeders" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// setEnvVarsWithCleanup sets multiple environment variables with automatic cleanup +func setEnvVarsWithCleanup(t *testing.T, envVars map[string]string) { + var keys []string + for key, value := range envVars { + t.Setenv(key, value) + keys = append(keys, key) + } + + // Schedule cleanup of all variables when test finishes + t.Cleanup(func() { + for _, key := range keys { + os.Unsetenv(key) + } + }) +} + +// createTestConfig creates a new config with field tracking for tests +func createTestConfig() (*Config, FieldTracker, *MockLogger) { + // Create logger that captures debug output + mockLogger := new(MockLogger) + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + // Create field tracker + tracker := NewDefaultFieldTracker() + tracker.SetLogger(mockLogger) + + // Create configuration builder with field tracking + cfg := NewConfig() + cfg.SetVerboseDebug(true, mockLogger) + cfg.SetFieldTracker(tracker) + + return cfg, tracker, mockLogger +} + +// clearTestEnvironment clears all environment variables that could affect our tests +func clearTestEnvironment(t *testing.T) { + // Clear all potential test environment variables + testEnvVars := []string{ + // Test 1 variables + "TEST1_APP_NAME", "TEST1_APP_DEBUG", "TEST1_APP_PORT", + // Test 2 variables + "TEST2_DB_PRIMARY_DRIVER", "TEST2_DB_PRIMARY_DSN", "TEST2_DB_PRIMARY_MAX_CONNS", + "TEST2_DB_SECONDARY_DRIVER", "TEST2_DB_SECONDARY_DSN", "TEST2_DB_SECONDARY_MAX_CONNS", + // Test 3 variables + "TEST3_APP_NAME", "TEST3_APP_DEBUG", "TEST3_APP_PORT", + "TEST3_DB_PRIMARY_DSN", "TEST3_DB_PRIMARY_DRIVER", "TEST3_DB_PRIMARY_MAX_CONNS", + "TEST3_DB_SECONDARY_DSN", "TEST3_DB_SECONDARY_DRIVER", "TEST3_DB_SECONDARY_MAX_CONNS", + // Legacy env vars (just in case) + "APP_NAME", "APP_DEBUG", "APP_PORT", + "DB_PRIMARY_DRIVER", "DB_PRIMARY_DSN", "DB_PRIMARY_MAX_CONNS", + "DB_SECONDARY_DRIVER", "DB_SECONDARY_DSN", "DB_SECONDARY_MAX_CONNS", + } + + for _, key := range testEnvVars { + os.Unsetenv(key) + } +} + +// TestBasicEnvFeederWithConfigBuilder tests field tracking with basic environment feeder +func TestBasicEnvFeederWithConfigBuilder(t *testing.T) { + // Clear any environment variables from previous tests + clearTestEnvironment(t) + + // Set up environment variables for this specific test + setEnvVarsWithCleanup(t, map[string]string{ + "TEST1_APP_NAME": "Test App", + "TEST1_APP_DEBUG": "true", + "TEST1_APP_PORT": "8080", + }) + + cfg, tracker, mockLogger := createTestConfig() + + // Define a basic app config + type AppConfig struct { + AppName string `env:"TEST1_APP_NAME"` + Debug bool `env:"TEST1_APP_DEBUG"` + Port int `env:"TEST1_APP_PORT"` + } + + config := &AppConfig{} + + // Add environment feeder + envFeeder := feeders.NewEnvFeeder() + cfg.AddFeeder(envFeeder) + + // Add the configuration structure + cfg.AddStructKey("app", config) + + // Feed configuration + err := cfg.Feed() + require.NoError(t, err) + + // Debug: Check if field tracker has any populations + populations := tracker.(*DefaultFieldTracker).FieldPopulations + t.Logf("Field tracker has %d populations after feeding", len(populations)) + for i, pop := range populations { + t.Logf(" %d: %s -> %v (from %s:%s)", i, pop.FieldPath, pop.Value, pop.SourceType, pop.SourceKey) + } + + // Verify config was populated correctly + assert.Equal(t, "Test App", config.AppName) + assert.True(t, config.Debug) + assert.Equal(t, 8080, config.Port) + + // Verify field tracking captured all field populations + require.GreaterOrEqual(t, len(populations), 3, "Should track at least 3 field populations") + + // Verify specific field populations + appNamePop := findFieldPopulation(populations, "AppName") + require.NotNil(t, appNamePop, "AppName field population should be tracked") + assert.Equal(t, "Test App", appNamePop.Value) + assert.Equal(t, "env", appNamePop.SourceType) + assert.Equal(t, "TEST1_APP_NAME", appNamePop.SourceKey) + assert.Contains(t, appNamePop.SearchKeys, "TEST1_APP_NAME") + + debugPop := findFieldPopulation(populations, "Debug") + require.NotNil(t, debugPop, "Debug field population should be tracked") + assert.Equal(t, true, debugPop.Value) + assert.Equal(t, "env", debugPop.SourceType) + assert.Equal(t, "TEST1_APP_DEBUG", debugPop.SourceKey) + + portPop := findFieldPopulation(populations, "Port") + require.NotNil(t, portPop, "Port field population should be tracked") + assert.Equal(t, 8080, portPop.Value) + assert.Equal(t, "env", portPop.SourceType) + assert.Equal(t, "TEST1_APP_PORT", portPop.SourceKey) + + // Verify that field tracking was used and logged + mockLogger.AssertCalled(t, "Debug", mock.MatchedBy(func(msg string) bool { + return msg == "Field populated" + }), mock.Anything) +} + +// TestDatabaseModuleInstanceAwareDSN tests field tracking with instance-aware database configuration +func TestDatabaseModuleInstanceAwareDSN(t *testing.T) { + // Clear any environment variables from previous tests + clearTestEnvironment(t) + + // Set up instance-aware database configuration + // This simulates the user's specific scenario + setEnvVarsWithCleanup(t, map[string]string{ + "TEST2_DB_PRIMARY_DRIVER": "postgres", + "TEST2_DB_PRIMARY_DSN": "postgres://localhost/primary", + "TEST2_DB_PRIMARY_MAX_CONNS": "10", + "TEST2_DB_SECONDARY_DRIVER": "mysql", + "TEST2_DB_SECONDARY_DSN": "mysql://localhost/secondary", + "TEST2_DB_SECONDARY_MAX_CONNS": "5", + }) + + cfg, tracker, mockLogger := createTestConfig() + // Define database connection config (matching the Database module structure) + type DBConnection struct { + Driver string `env:"DRIVER"` + DSN string `env:"DSN"` + MaxConns int `env:"MAX_CONNS"` + } + + type DatabaseConfig struct { + Connections map[string]DBConnection `yaml:"connections"` + } + + config := &DatabaseConfig{ + Connections: map[string]DBConnection{ + "primary": {}, + "secondary": {}, + }, + } + + // Add instance-aware environment feeder + instanceFeeder := feeders.NewInstanceAwareEnvFeeder(func(instanceKey string) string { + return "TEST2_DB_" + strings.ToUpper(instanceKey) + "_" + }) + cfg.AddFeeder(instanceFeeder) + + // Add the configuration structure + cfg.AddStructKey("database", config) + + // Feed configuration - this should use the regular Feed method first + err := cfg.Feed() + require.NoError(t, err) + + // Now use FeedInstances specifically for the connections map + err = instanceFeeder.FeedInstances(config.Connections) + require.NoError(t, err) + + // Debug: Check if field tracker has any populations + populations := tracker.(*DefaultFieldTracker).FieldPopulations + t.Logf("Field tracker has %d populations after feeding", len(populations)) + for i, pop := range populations { + t.Logf(" %d: %s -> %v (from %s:%s, instance:%s)", i, pop.FieldPath, pop.Value, pop.SourceType, pop.SourceKey, pop.InstanceKey) + } + + // Verify config was populated correctly + require.Contains(t, config.Connections, "primary") + require.Contains(t, config.Connections, "secondary") + + primaryConn := config.Connections["primary"] + assert.Equal(t, "postgres", primaryConn.Driver) + assert.Equal(t, "postgres://localhost/primary", primaryConn.DSN) + assert.Equal(t, 10, primaryConn.MaxConns) + + secondaryConn := config.Connections["secondary"] + assert.Equal(t, "mysql", secondaryConn.Driver) + assert.Equal(t, "mysql://localhost/secondary", secondaryConn.DSN) + assert.Equal(t, 5, secondaryConn.MaxConns) + + // Verify field tracking captured all field populations with instance awareness + require.GreaterOrEqual(t, len(populations), 6, "Should track at least 6 field populations (3 fields × 2 instances)") + + // Verify primary instance DSN field tracking + primaryDSNPop := findInstanceFieldPopulation(populations, "DSN", "primary") + require.NotNil(t, primaryDSNPop, "Primary DSN field population should be tracked") + assert.Equal(t, "postgres://localhost/primary", primaryDSNPop.Value) + assert.Equal(t, "env", primaryDSNPop.SourceType) + assert.Equal(t, "TEST2_DB_PRIMARY_DSN", primaryDSNPop.SourceKey) + assert.Equal(t, "primary", primaryDSNPop.InstanceKey) + assert.Contains(t, primaryDSNPop.SearchKeys, "TEST2_DB_PRIMARY_DSN") + + // Verify secondary instance DSN field tracking + secondaryDSNPop := findInstanceFieldPopulation(populations, "DSN", "secondary") + require.NotNil(t, secondaryDSNPop, "Secondary DSN field population should be tracked") + assert.Equal(t, "mysql://localhost/secondary", secondaryDSNPop.Value) + assert.Equal(t, "env", secondaryDSNPop.SourceType) + assert.Equal(t, "TEST2_DB_SECONDARY_DSN", secondaryDSNPop.SourceKey) + assert.Equal(t, "secondary", secondaryDSNPop.InstanceKey) + assert.Contains(t, secondaryDSNPop.SearchKeys, "TEST2_DB_SECONDARY_DSN") + + // Verify that field tracking was used and logged + mockLogger.AssertCalled(t, "Debug", mock.MatchedBy(func(msg string) bool { + return msg == "Field populated" + }), mock.Anything) +} + +// TestMixedFeedersYamlAndEnv tests field tracking with mixed YAML and environment feeders +func TestMixedFeedersYamlAndEnv(t *testing.T) { + // Clear any environment variables from previous tests + clearTestEnvironment(t) + + // Only set the environment variables we want for this test + // Others should be clean due to automatic cleanup from previous tests + setEnvVarsWithCleanup(t, map[string]string{ + "TEST3_APP_NAME": "Test App from ENV", + "TEST3_DB_PRIMARY_DSN": "postgres://env/primary", + }) + + cfg, tracker, mockLogger := createTestConfig() + + // Create a temporary YAML file + yamlContent := ` +app: + name: "Test App from YAML" + debug: true + port: 9090 +database: + connections: + primary: + driver: "postgres" + max_conns: 20 + secondary: + driver: "mysql" + dsn: "mysql://yaml/secondary" +` + yamlFile, err := os.CreateTemp("", "test_config_*.yaml") + require.NoError(t, err) + defer os.Remove(yamlFile.Name()) + + _, err = yamlFile.WriteString(yamlContent) + require.NoError(t, err) + yamlFile.Close() + + // Define config structures + type AppConfig struct { + AppName string `env:"TEST3_APP_NAME" yaml:"name"` + Debug bool `env:"TEST3_APP_DEBUG" yaml:"debug"` + Port int `env:"TEST3_APP_PORT" yaml:"port"` + } + + type DBConnection struct { + Driver string `env:"DRIVER" yaml:"driver"` + DSN string `env:"DSN" yaml:"dsn"` + MaxConns int `env:"MAX_CONNS" yaml:"max_conns"` + } + + type DatabaseConfig struct { + Connections map[string]DBConnection `yaml:"connections"` + } + + type RootConfig struct { + App AppConfig `yaml:"app"` + Database DatabaseConfig `yaml:"database"` + } + + config := &RootConfig{ + Database: DatabaseConfig{ + Connections: map[string]DBConnection{ + "primary": {}, + "secondary": {}, + }, + }, + } + + // Add YAML feeder first (lower priority) + yamlFeeder := feeders.NewYamlFeeder(yamlFile.Name()) + cfg.AddFeeder(yamlFeeder) + + // Add environment feeder (higher priority) + envFeeder := feeders.NewEnvFeeder() + cfg.AddFeeder(envFeeder) + + // Add instance-aware environment feeder (highest priority for instance fields) + instanceFeeder := feeders.NewInstanceAwareEnvFeeder(func(instanceKey string) string { + return "TEST3_DB_" + strings.ToUpper(instanceKey) + "_" + }) + cfg.AddFeeder(instanceFeeder) + + // Add the configuration structure + cfg.AddStructKey("root", config) + + // Feed configuration + err = cfg.Feed() + require.NoError(t, err) + + // Now use FeedInstances specifically for the connections map with instance-aware env + t.Logf("Before FeedInstances: primary DSN = %s", config.Database.Connections["primary"].DSN) + err = instanceFeeder.FeedInstances(config.Database.Connections) + require.NoError(t, err) + t.Logf("After FeedInstances: primary DSN = %s", config.Database.Connections["primary"].DSN) + + // Debug: Check if field tracker has any populations + populations := tracker.(*DefaultFieldTracker).FieldPopulations + t.Logf("Field tracker has %d populations after feeding", len(populations)) + for i, pop := range populations { + t.Logf(" %d: %s -> %v (from %s:%s)", i, pop.FieldPath, pop.Value, pop.SourceType, pop.SourceKey) + } + + // Verify config was populated from correct sources + // APP_NAME should come from ENV (overriding YAML) + assert.Equal(t, "Test App from ENV", config.App.AppName) + // Debug should come from YAML (no ENV override) + assert.True(t, config.App.Debug) + // Port should come from YAML + assert.Equal(t, 9090, config.App.Port) + + // Primary DSN should come from ENV (set via instance-aware feeder) + primaryConn := config.Database.Connections["primary"] + assert.Equal(t, "postgres://env/primary", primaryConn.DSN) + // Primary driver should come from YAML + assert.Equal(t, "postgres", primaryConn.Driver) + + // Secondary DSN should come from YAML + secondaryConn := config.Database.Connections["secondary"] + assert.Equal(t, "mysql://yaml/secondary", secondaryConn.DSN) + + // Verify field tracking shows correct sources + + // AppName should be tracked as populated from ENV (last wins) + appNamePop := findLastFieldPopulation(populations, "AppName") + require.NotNil(t, appNamePop, "AppName field population should be tracked") + assert.Equal(t, "Test App from ENV", appNamePop.Value) + assert.Equal(t, "env", appNamePop.SourceType) + + // Debug should be tracked as populated from YAML + debugPop := findFieldPopulation(populations, "Debug") + require.NotNil(t, debugPop, "Debug field population should be tracked") + assert.Equal(t, true, debugPop.Value) + assert.Equal(t, "yaml", debugPop.SourceType) + + // Primary DSN should be tracked as populated from ENV with instance awareness + primaryDSNPop := findInstanceFieldPopulation(populations, "DSN", "primary") + require.NotNil(t, primaryDSNPop, "Primary DSN field population should be tracked") + assert.Equal(t, "postgres://env/primary", primaryDSNPop.Value) + assert.Equal(t, "env", primaryDSNPop.SourceType) + assert.Equal(t, "primary", primaryDSNPop.InstanceKey) + + // Verify that field tracking was used and logged + mockLogger.AssertCalled(t, "Debug", mock.MatchedBy(func(msg string) bool { + return msg == "Field populated" + }), mock.Anything) +} + +// Helper function to find a field population by field name +func findFieldPopulation(populations []FieldPopulation, fieldName string) *FieldPopulation { + for _, pop := range populations { + if pop.FieldName == fieldName { + return &pop + } + } + return nil +} + +// Helper function to find the last field population by field name (for override scenarios) +func findLastFieldPopulation(populations []FieldPopulation, fieldName string) *FieldPopulation { + var result *FieldPopulation + for _, pop := range populations { + if pop.FieldName == fieldName && pop.Value != nil { + result = &pop + } + } + return result +} + +// Helper function to find a field population by field name and instance key +func findInstanceFieldPopulation(populations []FieldPopulation, fieldName, instanceKey string) *FieldPopulation { + for _, pop := range populations { + if pop.FieldName == fieldName && pop.InstanceKey == instanceKey { + return &pop + } + } + return nil +} diff --git a/config_provider.go b/config_provider.go index f33c797b..030c584b 100644 --- a/config_provider.go +++ b/config_provider.go @@ -3,8 +3,6 @@ package modular import ( "fmt" "reflect" - - "github.com/golobby/config/v3" ) const mainConfigSection = "_main" @@ -81,7 +79,7 @@ func NewStdConfigProvider(cfg any) *StdConfigProvider { } // Config represents a configuration builder that can combine multiple feeders and structures. -// It extends the golobby/config library with additional functionality for the modular framework. +// It provides functionality for the modular framework to coordinate configuration loading. // // The Config builder allows you to: // - Add multiple configuration sources (files, environment, etc.) @@ -89,8 +87,10 @@ func NewStdConfigProvider(cfg any) *StdConfigProvider { // - Apply configuration to multiple struct targets // - Track which structs have been configured // - Enable verbose debugging for configuration processing +// - Track field-level population details type Config struct { - *config.Config + // Feeders contains all the registered configuration feeders + Feeders []Feeder // StructKeys maps struct identifiers to their configuration objects. // Used internally to track which configuration structures have been processed. StructKeys map[string]interface{} @@ -98,6 +98,8 @@ type Config struct { VerboseDebug bool // Logger is used for verbose debug logging Logger Logger + // FieldTracker tracks which fields are populated by which feeders + FieldTracker FieldTracker } // NewConfig creates a new configuration builder. @@ -112,10 +114,11 @@ type Config struct { // err := cfg.Feed() // Load configuration func NewConfig() *Config { return &Config{ - Config: config.New(), + Feeders: make([]Feeder, 0), StructKeys: make(map[string]interface{}), VerboseDebug: false, Logger: nil, + FieldTracker: NewDefaultFieldTracker(), } } @@ -124,6 +127,11 @@ func (c *Config) SetVerboseDebug(enabled bool, logger Logger) *Config { c.VerboseDebug = enabled c.Logger = logger + // Set logger on field tracker + if c.FieldTracker != nil { + c.FieldTracker.SetLogger(logger) + } + // Apply verbose debugging to any verbose-aware feeders for _, feeder := range c.Feeders { if verboseFeeder, ok := feeder.(VerboseAwareFeeder); ok { @@ -134,9 +142,9 @@ func (c *Config) SetVerboseDebug(enabled bool, logger Logger) *Config { return c } -// AddFeeder overrides the parent AddFeeder to support verbose debugging +// AddFeeder adds a configuration feeder to support verbose debugging and field tracking func (c *Config) AddFeeder(feeder Feeder) *Config { - c.Config.AddFeeder(feeder) + c.Feeders = append(c.Feeders, feeder) // If verbose debugging is enabled, apply it to this feeder if c.VerboseDebug && c.Logger != nil { @@ -144,6 +152,24 @@ func (c *Config) AddFeeder(feeder Feeder) *Config { verboseFeeder.SetVerboseDebug(true, c.Logger) } } + // If field tracking is enabled, apply it to this feeder + if c.FieldTracker != nil { + // Check for main package FieldTrackingFeeder interface + if trackingFeeder, ok := feeder.(FieldTrackingFeeder); ok { + trackingFeeder.SetFieldTracker(c.FieldTracker) + } else { + // Check for feeders package interface compatibility + // Use reflection to check if the feeder has a SetFieldTracker method + feederValue := reflect.ValueOf(feeder) + setFieldTrackerMethod := feederValue.MethodByName("SetFieldTracker") + if setFieldTrackerMethod.IsValid() { + // Create a bridge adapter and call SetFieldTracker + bridge := NewFieldTrackerBridge(c.FieldTracker) + args := []reflect.Value{reflect.ValueOf(bridge)} + setFieldTrackerMethod.Call(args) + } + } + } return c } @@ -154,84 +180,110 @@ func (c *Config) AddStructKey(key string, target interface{}) *Config { return c } -// Feed with validation applies defaults and validates configs after feeding -func (c *Config) Feed() error { - if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Starting config feed process", "structKeysCount", len(c.StructKeys), "feedersCount", len(c.Feeders)) +// SetFieldTracker sets the field tracker for capturing field population details +func (c *Config) SetFieldTracker(tracker FieldTracker) *Config { + c.FieldTracker = tracker + if c.Logger != nil { + c.FieldTracker.SetLogger(c.Logger) } - if err := c.Config.Feed(); err != nil { - if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Config feed failed", "error", err) + // Apply field tracking to any tracking-aware feeders + for _, feeder := range c.Feeders { + if trackingFeeder, ok := feeder.(FieldTrackingFeeder); ok { + trackingFeeder.SetFieldTracker(tracker) } - return fmt.Errorf("config feed error: %w", err) } + return c +} + +// Feed with validation applies defaults and validates configs after feeding +func (c *Config) Feed() error { if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Config feed completed, processing struct keys") + c.Logger.Debug("Starting config feed process", "structKeysCount", len(c.StructKeys), "feedersCount", len(c.Feeders)) } - for key, target := range c.StructKeys { + // If we have struct keys, feed them directly with field tracking + if len(c.StructKeys) > 0 { if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Processing struct key", "key", key, "targetType", reflect.TypeOf(target)) + c.Logger.Debug("Using enhanced feeding process with field tracking") } - for i, f := range c.Feeders { + // Feed each struct key with each feeder + for key, target := range c.StructKeys { if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Applying feeder to struct", "key", key, "feederIndex", i, "feederType", fmt.Sprintf("%T", f)) + c.Logger.Debug("Processing struct key", "key", key, "targetType", reflect.TypeOf(target)) } - cf, ok := f.(ComplexFeeder) - if !ok { + for i, f := range c.Feeders { if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Feeder is not a ComplexFeeder, skipping", "key", key, "feederType", fmt.Sprintf("%T", f)) + c.Logger.Debug("Applying feeder to struct", "key", key, "feederIndex", i, "feederType", fmt.Sprintf("%T", f)) + } + + // Try to use the feeder's Feed method directly for better field tracking + if err := f.Feed(target); err != nil { + if c.VerboseDebug && c.Logger != nil { + c.Logger.Debug("Feeder Feed method failed", "key", key, "feederType", fmt.Sprintf("%T", f), "error", err) + } + return fmt.Errorf("config feeder error: %w: %w", ErrConfigFeederError, err) + } + + // Also try ComplexFeeder if available (for instance-aware feeders) + if cf, ok := f.(ComplexFeeder); ok { + if c.VerboseDebug && c.Logger != nil { + c.Logger.Debug("Applying ComplexFeeder FeedKey", "key", key, "feederType", fmt.Sprintf("%T", f)) + } + + if err := cf.FeedKey(key, target); err != nil { + if c.VerboseDebug && c.Logger != nil { + c.Logger.Debug("ComplexFeeder FeedKey failed", "key", key, "feederType", fmt.Sprintf("%T", f), "error", err) + } + return fmt.Errorf("config feeder error: %w: %w", ErrConfigFeederError, err) + } } - continue - } - if err := cf.FeedKey(key, target); err != nil { if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("ComplexFeeder FeedKey failed", "key", key, "feederType", fmt.Sprintf("%T", f), "error", err) + c.Logger.Debug("Feeder applied successfully", "key", key, "feederType", fmt.Sprintf("%T", f)) } - return fmt.Errorf("config feeder error: %w: %w", ErrConfigFeederError, err) } + // Apply defaults and validate config if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("ComplexFeeder FeedKey succeeded", "key", key, "feederType", fmt.Sprintf("%T", f)) + c.Logger.Debug("Validating config for struct key", "key", key) } - } - - if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Validating config for struct key", "key", key) - } - // Apply defaults and validate config - if err := ValidateConfig(target); err != nil { - if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Config validation failed", "key", key, "error", err) + if err := ValidateConfig(target); err != nil { + if c.VerboseDebug && c.Logger != nil { + c.Logger.Debug("Config validation failed", "key", key, "error", err) + } + return fmt.Errorf("config validation error for %s: %w", key, err) } - return fmt.Errorf("config validation error for %s: %w", key, err) - } - if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Config validation succeeded", "key", key) - } - - // Call Setup if implemented - if setupable, ok := target.(ConfigSetup); ok { if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Calling Setup for config", "key", key) + c.Logger.Debug("Config validation succeeded", "key", key) } - if err := setupable.Setup(); err != nil { + + // Call Setup if implemented + if setupable, ok := target.(ConfigSetup); ok { if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Config setup failed", "key", key, "error", err) + c.Logger.Debug("Calling Setup for config", "key", key) + } + if err := setupable.Setup(); err != nil { + if c.VerboseDebug && c.Logger != nil { + c.Logger.Debug("Config setup failed", "key", key, "error", err) + } + return fmt.Errorf("%w for %s: %w", ErrConfigSetupError, key, err) + } + if c.VerboseDebug && c.Logger != nil { + c.Logger.Debug("Config setup succeeded", "key", key) } - return fmt.Errorf("%w for %s: %w", ErrConfigSetupError, key, err) - } - if c.VerboseDebug && c.Logger != nil { - c.Logger.Debug("Config setup succeeded", "key", key) } } + } else { + // No struct keys configured - this means no explicit structures were added + if c.VerboseDebug && c.Logger != nil { + c.Logger.Debug("No struct keys configured - skipping feed process") + } } if c.VerboseDebug && c.Logger != nil { @@ -368,7 +420,7 @@ func processMainConfig(app *StdApplication, cfgBuilder *Config, tempConfigs map[ return false } - cfgBuilder.AddStruct(tempMainCfg) + cfgBuilder.AddStructKey(mainConfigSection, tempMainCfg) tempConfigs[mainConfigSection] = mainCfgInfo app.logger.Debug("Added main config for loading", "type", reflect.TypeOf(mainCfg)) diff --git a/config_provider_test.go b/config_provider_test.go index 58acb5e6..478ba96d 100644 --- a/config_provider_test.go +++ b/config_provider_test.go @@ -64,7 +64,7 @@ func TestNewConfig(t *testing.T) { cfg := NewConfig() assert.NotNil(t, cfg) - assert.NotNil(t, cfg.Config) + assert.NotNil(t, cfg.Feeders) assert.NotNil(t, cfg.StructKeys) assert.Empty(t, cfg.StructKeys) } @@ -109,9 +109,10 @@ func TestConfig_Feed(t *testing.T) { cfg := NewConfig() feeder := new(MockComplexFeeder) feeder.On("Feed", mock.Anything).Return(nil) + feeder.On("FeedKey", "main", mock.Anything).Return(nil) feeder.On("FeedKey", "test", mock.Anything).Return(nil) cfg.AddFeeder(feeder) - cfg.AddStruct(&testCfg{}) + cfg.AddStructKey("main", &testCfg{}) cfg.AddStructKey("test", &testCfg{}) return cfg, feeder }, @@ -124,7 +125,7 @@ func TestConfig_Feed(t *testing.T) { feeder := new(MockComplexFeeder) feeder.On("Feed", mock.Anything).Return(ErrFeedFailed) cfg.AddFeeder(feeder) - cfg.AddStruct(&testCfg{}) + cfg.AddStructKey("main", &testCfg{}) return cfg, feeder }, expectFeedErr: true, @@ -136,9 +137,13 @@ func TestConfig_Feed(t *testing.T) { cfg := NewConfig() feeder := new(MockComplexFeeder) feeder.On("Feed", mock.Anything).Return(nil) + // Due to map iteration order being random, either key could be processed first + // If "test" is processed first, it will fail and stop processing + // If "main" is processed first, it will succeed, then "test" will fail + feeder.On("FeedKey", "main", mock.Anything).Return(nil).Maybe() feeder.On("FeedKey", "test", mock.Anything).Return(ErrFeedKeyFailed) cfg.AddFeeder(feeder) - cfg.AddStruct(&testCfg{}) + cfg.AddStructKey("main", &testCfg{}) cfg.AddStructKey("test", &testCfg{}) return cfg, feeder }, @@ -152,9 +157,10 @@ func TestConfig_Feed(t *testing.T) { cfg := NewConfig() feeder := new(MockComplexFeeder) feeder.On("Feed", mock.Anything).Return(nil) + feeder.On("FeedKey", "main", mock.Anything).Return(nil) feeder.On("FeedKey", "test", mock.Anything).Return(nil) cfg.AddFeeder(feeder) - cfg.AddStruct(&testCfg{}) + cfg.AddStructKey("main", &testCfg{}) cfg.AddStructKey("test", &testSetupCfg{}) return cfg, feeder }, @@ -166,9 +172,13 @@ func TestConfig_Feed(t *testing.T) { cfg := NewConfig() feeder := new(MockComplexFeeder) feeder.On("Feed", mock.Anything).Return(nil) - feeder.On("FeedKey", "test", mock.Anything).Return(nil) + // Due to map iteration order being random, either key could be processed first + // If "test" is processed first, it will succeed then fail at setup + // If "main" is processed first, it will succeed, then "test" will succeed and fail at setup + feeder.On("FeedKey", "main", mock.Anything).Return(nil).Maybe() + feeder.On("FeedKey", "test", mock.Anything).Return(nil).Maybe() cfg.AddFeeder(feeder) - cfg.AddStruct(&testCfg{}) + cfg.AddStructKey("main", &testCfg{}) cfg.AddStructKey("test", &testSetupCfg{shouldError: true}) return cfg, feeder }, @@ -353,13 +363,23 @@ func Test_loadAppConfig(t *testing.T) { }, setupFeeders: func() []Feeder { feeder := new(MockComplexFeeder) - // Setup for main config + // Setup to handle any Feed call - let the Run function determine the type feeder.On("Feed", mock.Anything).Return(nil).Run(func(args mock.Arguments) { - cfg := args.Get(0).(*testCfg) + if cfg, ok := args.Get(0).(*testCfg); ok { + cfg.Str = updatedValue + cfg.Num = 42 + } else if cfg, ok := args.Get(0).(*testSectionCfg); ok { + cfg.Enabled = true + cfg.Name = "updated" + } + }) + // Setup for main config FeedKey calls + feeder.On("FeedKey", "_main", mock.Anything).Return(nil).Run(func(args mock.Arguments) { + cfg := args.Get(1).(*testCfg) cfg.Str = updatedValue cfg.Num = 42 }) - // Setup for section config + // Setup for section config FeedKey calls feeder.On("FeedKey", "section1", mock.Anything).Return(nil).Run(func(args mock.Arguments) { cfg := args.Get(1).(*testSectionCfg) cfg.Enabled = true @@ -420,6 +440,10 @@ func Test_loadAppConfig(t *testing.T) { setupFeeders: func() []Feeder { feeder := new(MockComplexFeeder) feeder.On("Feed", mock.Anything).Return(nil) + // Due to map iteration order being random, either key could be processed first + // If "section1" is processed first, it will fail and stop processing + // If "_main" is processed first, it will succeed, then "section1" will fail + feeder.On("FeedKey", "_main", mock.Anything).Return(nil).Maybe() feeder.On("FeedKey", "section1", mock.Anything).Return(ErrFeedKeyFailed) return []Feeder{feeder} }, @@ -457,7 +481,16 @@ func Test_loadAppConfig(t *testing.T) { setupFeeders: func() []Feeder { feeder := new(MockComplexFeeder) feeder.On("Feed", mock.Anything).Return(nil).Run(func(args mock.Arguments) { - cfg := args.Get(0).(*testCfg) + if cfg, ok := args.Get(0).(*testCfg); ok { + cfg.Str = updatedValue + cfg.Num = 42 + } else if cfg, ok := args.Get(0).(*testSectionCfg); ok { + cfg.Enabled = true + cfg.Name = "updated" + } + }) + feeder.On("FeedKey", "_main", mock.Anything).Return(nil).Run(func(args mock.Arguments) { + cfg := args.Get(1).(*testCfg) cfg.Str = updatedValue cfg.Num = 42 }) diff --git a/feeders/DOCUMENTATION.md b/feeders/DOCUMENTATION.md new file mode 100644 index 00000000..a2c719a4 --- /dev/null +++ b/feeders/DOCUMENTATION.md @@ -0,0 +1,217 @@ +# Environment Variable Catalog and Feeder Integration + +This document describes how the Modular framework's enhanced configuration system handles environment variables and manages feeder precedence. + +## Overview + +The Modular framework uses a unified **Environment Catalog** system that combines environment variables from multiple sources: +- Operating System environment variables +- `.env` file variables +- Dynamically set variables + +This allows all environment-based feeders (EnvFeeder, AffixedEnvFeeder, InstanceAwareEnvFeeder, TenantAffixedEnvFeeder) to access variables from both OS environment and .env files with proper precedence handling. + +## Environment Catalog System + +### EnvCatalog Architecture + +The `EnvCatalog` provides: +- **Unified Access**: Single interface for all environment variables +- **Source Tracking**: Tracks whether variables come from OS env or .env files +- **Precedence Management**: OS environment always takes precedence over .env files +- **Thread Safety**: Concurrent access safe with mutex protection + +### Variable Precedence + +1. **OS Environment Variables** (highest precedence) +2. **.env File Variables** (lower precedence) + +When the same variable exists in both sources, the OS environment value is used. + +### Global Catalog + +- Single global catalog instance shared across all env-based feeders +- Initialized once and reused for performance +- Can be reset for testing scenarios + +## Feeder Types and Integration + +### File-Based Feeders +These feeders read from configuration files and populate structs directly: + +1. **YamlFeeder**: Reads YAML files, supports nested structures +2. **JSONFeeder**: Reads JSON files, handles complex object hierarchies +3. **TomlFeeder**: Reads TOML files, supports all TOML data types +4. **DotEnvFeeder**: Special hybrid - loads .env into catalog AND populates structs + +### Environment-Based Feeders +These feeders read from the unified Environment Catalog: + +1. **EnvFeeder**: Basic env var lookup using struct field `env` tags +2. **AffixedEnvFeeder**: Adds prefix/suffix to env variable names +3. **InstanceAwareEnvFeeder**: Handles instance-specific configurations +4. **TenantAffixedEnvFeeder**: Combines tenant-aware and affixed behavior + +### DotEnvFeeder Behavior + +The `DotEnvFeeder` has dual behavior: +1. **Catalog Population**: Loads .env variables into the global catalog for other env feeders +2. **Direct Population**: Populates config structs using catalog (respects OS env precedence) + +This allows other env-based feeders to access .env variables while maintaining proper precedence. + +## Field-Level Tracking + +All feeders support comprehensive field-level tracking that records: + +- **Field Path**: Complete field path (e.g., "Database.Connections.primary.DSN") +- **Field Type**: Data type of the field +- **Feeder Type**: Which feeder populated the field +- **Source Type**: Source category (env, yaml, json, toml, dotenv) +- **Source Key**: The actual key used (e.g., "DB_PRIMARY_DSN") +- **Value**: The value that was set +- **Search Keys**: All keys that were searched +- **Found Key**: The key that was actually found +- **Instance Info**: For instance-aware feeders + +### Tracking Usage + +```go +tracker := NewDefaultFieldTracker() +feeder.SetFieldTracker(tracker) + +// After feeding +populations := tracker.GetFieldPopulations() +for _, pop := range populations { + fmt.Printf("Field %s set to %v from %s\n", + pop.FieldPath, pop.Value, pop.SourceKey) +} +``` + +## Feeder Evaluation Order and Precedence + +### Recommended Order + +When using multiple feeders, the typical order is: + +1. **File-based feeders** (YAML/JSON/TOML) - set base configuration +2. **DotEnvFeeder** - load .env variables into catalog +3. **Environment-based feeders** - override with env-specific values + +### Precedence Rules + +**Within the same feeder type**: Last feeder wins (overwrites previous values) + +**Between feeder types**: Order of execution determines precedence + +**For environment variables**: OS environment always beats .env files + +### Example Multi-Feeder Setup + +```go +config := modular.NewConfig() + +// Base configuration from YAML +config.AddFeeder(feeders.NewYamlFeeder("config.yaml")) + +// Load .env into catalog for other env feeders +config.AddFeeder(feeders.NewDotEnvFeeder(".env")) + +// Environment-based overrides +config.AddFeeder(feeders.NewEnvFeeder()) +config.AddFeeder(feeders.NewAffixedEnvFeeder("APP_", "_PROD")) + +// Feed the configuration +err := config.Feed(&appConfig) +``` + +### Precedence Flow + +``` +YAML values → DotEnv values → OS Env values → Affixed Env values + (base) → (if not in OS) → (override) → (final override) +``` + +## Environment Variable Naming Patterns + +### EnvFeeder +Uses env tags directly: `env:"DATABASE_URL"` + +### AffixedEnvFeeder +Constructs: `PREFIX__ENVTAG__SUFFIX` +- Example: `PROD_` + `HOST` + `_ENV` = `PROD__HOST__ENV` +- Uses double underscores between components + +### InstanceAwareEnvFeeder +Constructs: `MODULE_INSTANCE_FIELD` +- Example: `DB_PRIMARY_DSN`, `DB_SECONDARY_DSN` + +### TenantAffixedEnvFeeder +Combines tenant ID with affixed pattern: +- Example: `APP_TENANT123__CONFIG__PROD` + +## Error Handling + +The system uses static error definitions to comply with linting rules: + +```go +var ( + ErrDotEnvInvalidStructureType = errors.New("expected pointer to struct") + ErrJSONCannotConvert = errors.New("cannot convert value to field type") + // ... more specific errors +) +``` + +Errors are wrapped with context using `fmt.Errorf("%w: %s", baseError, context)`. + +## Verbose Debug Logging + +All feeders support verbose debug logging for troubleshooting: + +```go +feeder.SetVerboseDebug(true, logger) +``` + +Debug output includes: +- Environment variable lookups and results +- Field processing steps +- Type conversion attempts +- Source tracking information +- Error details with context + +## Best Practices + +### Configuration Setup +1. Use file-based feeders for base configuration +2. Use DotEnvFeeder to load .env files for local development +3. Use env-based feeders for deployment-specific overrides +4. Set up field tracking for debugging and audit trails + +### Environment Variable Management +1. Use consistent naming patterns +2. Document env var precedence in your application +3. Test with both OS env and .env file scenarios +4. Use verbose debugging during development + +### Error Handling +1. Always check feeder errors during configuration loading +2. Use field tracking to identify configuration sources +3. Validate required fields after feeding +4. Provide clear error messages for missing configuration + +## Testing Considerations + +### Test Isolation +```go +// Reset catalog between tests +feeders.ResetGlobalEnvCatalog() + +// Use t.Setenv for test environment variables +t.Setenv("TEST_VAR", "test_value") +``` + +### Multi-Feeder Testing +Test various combinations of feeders to ensure proper precedence handling. + +### Field Tracking Validation +Verify that field tracking correctly reports source information for debugging and audit purposes. diff --git a/feeders/affixed_debug_test.go b/feeders/affixed_debug_test.go new file mode 100644 index 00000000..72529951 --- /dev/null +++ b/feeders/affixed_debug_test.go @@ -0,0 +1,59 @@ +package feeders + +import ( + "testing" +) + +func TestAffixedEnvFeederCatalogDebug(t *testing.T) { + // Reset global catalog + ResetGlobalEnvCatalog() + + // Set environment variables (double underscores per AffixedEnvFeeder pattern) + t.Setenv("PROD__HOST__ENV", "prod.example.com") + t.Setenv("PROD__PORT__ENV", "3306") + + type DatabaseConfig struct { + Host string `env:"HOST"` + Port int `env:"PORT"` + } + + var config DatabaseConfig + + // Create and test AffixedEnvFeeder + feeder := NewAffixedEnvFeeder("PROD_", "_ENV") + tracker := NewDefaultFieldTracker() + feeder.SetFieldTracker(tracker) + + // Enable verbose debug + logger := &TestLogger2{} + feeder.SetVerboseDebug(true, logger) + + err := feeder.Feed(&config) + if err != nil { + t.Fatalf("AffixedEnvFeeder failed: %v", err) + } + + t.Logf("Config after feeding: Host='%s', Port=%d", config.Host, config.Port) + + // Check field tracking + populations := tracker.GetFieldPopulations() + t.Logf("Field populations: %d", len(populations)) + for _, pop := range populations { + t.Logf(" Field: %s, Value: %v, SourceKey: %s, SourceType: %s", + pop.FieldPath, pop.Value, pop.SourceKey, pop.SourceType) + } + + // Verify results + if config.Host != "prod.example.com" { + t.Errorf("Expected Host 'prod.example.com', got '%s'", config.Host) + } + if config.Port != 3306 { + t.Errorf("Expected Port 3306, got %d", config.Port) + } +} + +type TestLogger2 struct{} + +func (l *TestLogger2) Debug(msg string, args ...any) { + // Just for debug output +} diff --git a/feeders/affixed_env.go b/feeders/affixed_env.go index 79707ef4..59e96f32 100644 --- a/feeders/affixed_env.go +++ b/feeders/affixed_env.go @@ -5,7 +5,6 @@ package feeders import ( "errors" "fmt" - "os" "reflect" "strings" @@ -29,6 +28,7 @@ type AffixedEnvFeeder struct { logger interface { Debug(msg string, args ...any) } + fieldTracker FieldTracker } // NewAffixedEnvFeeder creates a new AffixedEnvFeeder with the specified prefix and suffix @@ -38,6 +38,7 @@ func NewAffixedEnvFeeder(prefix, suffix string) AffixedEnvFeeder { Suffix: suffix, verboseDebug: false, logger: nil, + fieldTracker: nil, } } @@ -50,8 +51,13 @@ func (f *AffixedEnvFeeder) SetVerboseDebug(enabled bool, logger interface{ Debug } } +// SetFieldTracker sets the field tracker for recording field populations +func (f *AffixedEnvFeeder) SetFieldTracker(tracker FieldTracker) { + f.fieldTracker = tracker +} + // Feed reads environment variables and populates the provided structure -func (f AffixedEnvFeeder) Feed(structure interface{}) error { +func (f *AffixedEnvFeeder) Feed(structure interface{}) error { if f.verboseDebug && f.logger != nil { f.logger.Debug("AffixedEnvFeeder: Starting feed process", "structureType", reflect.TypeOf(structure), "prefix", f.Prefix, "suffix", f.Suffix) } @@ -72,7 +78,7 @@ func (f AffixedEnvFeeder) Feed(structure interface{}) error { } // fillStruct sets struct fields from environment variables -func (f AffixedEnvFeeder) fillStruct(rv reflect.Value, prefix, suffix string) error { +func (f *AffixedEnvFeeder) fillStruct(rv reflect.Value, prefix, suffix string) error { if prefix == "" && suffix == "" { if f.verboseDebug && f.logger != nil { f.logger.Debug("AffixedEnvFeeder: Both prefix and suffix are empty") @@ -91,7 +97,7 @@ func (f AffixedEnvFeeder) fillStruct(rv reflect.Value, prefix, suffix string) er } // processStructFields iterates through struct fields -func (f AffixedEnvFeeder) processStructFields(rv reflect.Value, prefix, suffix string) error { +func (f *AffixedEnvFeeder) processStructFields(rv reflect.Value, prefix, suffix string) error { if f.verboseDebug && f.logger != nil { f.logger.Debug("AffixedEnvFeeder: Processing struct fields", "numFields", rv.NumField(), "prefix", prefix, "suffix", suffix) } @@ -119,7 +125,7 @@ func (f AffixedEnvFeeder) processStructFields(rv reflect.Value, prefix, suffix s } // processField handles a single struct field -func (f AffixedEnvFeeder) processField(field reflect.Value, fieldType *reflect.StructField, prefix, suffix string) error { +func (f *AffixedEnvFeeder) processField(field reflect.Value, fieldType *reflect.StructField, prefix, suffix string) error { // Handle nested structs switch field.Kind() { case reflect.Struct: @@ -144,7 +150,8 @@ func (f AffixedEnvFeeder) processField(field reflect.Value, fieldType *reflect.S if f.verboseDebug && f.logger != nil { f.logger.Debug("AffixedEnvFeeder: Found env tag", "fieldName", fieldType.Name, "envTag", envTag) } - return f.setFieldFromEnv(field, envTag, prefix, suffix) + fieldPath := fieldType.Name + return f.setFieldFromEnv(field, fieldType, envTag, fieldPath, prefix, suffix) } else if f.verboseDebug && f.logger != nil { f.logger.Debug("AffixedEnvFeeder: No env tag found", "fieldName", fieldType.Name) } @@ -154,7 +161,7 @@ func (f AffixedEnvFeeder) processField(field reflect.Value, fieldType *reflect.S } // setFieldFromEnv sets a field value from an environment variable -func (f AffixedEnvFeeder) setFieldFromEnv(field reflect.Value, envTag, prefix, suffix string) error { +func (f *AffixedEnvFeeder) setFieldFromEnv(field reflect.Value, fieldType *reflect.StructField, envTag, fieldPath, prefix, suffix string) error { // Build environment variable name envName := strings.ToUpper(envTag) if prefix != "" { @@ -169,17 +176,42 @@ func (f AffixedEnvFeeder) setFieldFromEnv(field reflect.Value, envTag, prefix, s } // Get and apply environment variable if exists - if envValue := os.Getenv(envName); envValue != "" { + catalog := GetGlobalEnvCatalog() + envValue, exists := catalog.Get(envName) + if exists && envValue != "" { if f.verboseDebug && f.logger != nil { - f.logger.Debug("AffixedEnvFeeder: Environment variable found", "envName", envName, "envValue", envValue) + source := catalog.GetSource(envName) + f.logger.Debug("AffixedEnvFeeder: Environment variable found", "envName", envName, "envValue", envValue, "source", source) } err := setFieldValue(field, envValue) - if err != nil && f.verboseDebug && f.logger != nil { - f.logger.Debug("AffixedEnvFeeder: Failed to set field value", "envName", envName, "envValue", envValue, "error", err) - } else if f.verboseDebug && f.logger != nil { + if err != nil { + if f.verboseDebug && f.logger != nil { + f.logger.Debug("AffixedEnvFeeder: Failed to set field value", "envName", envName, "envValue", envValue, "error", err) + } + return err + } + + // Record field population + if f.fieldTracker != nil { + convertedValue, _ := cast.FromType(envValue, field.Type()) + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldType.Name, + FieldType: field.Type().String(), + FeederType: "AffixedEnvFeeder", + SourceType: "env_affixed", + SourceKey: envName, + Value: convertedValue, + SearchKeys: []string{envName}, + FoundKey: envName, + } + f.fieldTracker.RecordFieldPopulation(fp) + } + + if f.verboseDebug && f.logger != nil { f.logger.Debug("AffixedEnvFeeder: Successfully set field value", "envName", envName, "envValue", envValue) } - return err + return nil } else if f.verboseDebug && f.logger != nil { f.logger.Debug("AffixedEnvFeeder: Environment variable not found or empty", "envName", envName) } diff --git a/feeders/affixed_env_field_tracking_test.go b/feeders/affixed_env_field_tracking_test.go new file mode 100644 index 00000000..5d7a1e06 --- /dev/null +++ b/feeders/affixed_env_field_tracking_test.go @@ -0,0 +1,170 @@ +package feeders + +import ( + "fmt" + "os" + "testing" +) + +// TestConfig struct for AffixedEnv field tracking tests +type TestAffixedEnvConfig struct { + Name string `env:"NAME"` + Port int `env:"PORT"` + Enabled bool `env:"ENABLED"` + Debug string `env:"DEBUG"` +} + +func TestAffixedEnvFeeder_FieldTracking(t *testing.T) { + // Set up environment variables with prefix and suffix + envVars := map[string]string{ + "APP__NAME__PROD": "test-app", + "APP__PORT__PROD": "8080", + "APP__ENABLED__PROD": "true", + "APP__DEBUG__PROD": "verbose", + "OTHER_VAR": "ignored", // Should not be matched + } + + // Set environment variables for test + for key, value := range envVars { + os.Setenv(key, value) + } + defer func() { + // Clean up after test + for key := range envVars { + os.Unsetenv(key) + } + }() + + // Create feeder and field tracker + feeder := NewAffixedEnvFeeder("APP_", "_PROD") + tracker := NewDefaultFieldTracker() + feeder.SetFieldTracker(tracker) + + // Test config structure + var config TestAffixedEnvConfig + + // Feed the configuration + err := feeder.Feed(&config) + if err != nil { + t.Fatalf("Failed to feed config: %v", err) + } + + // Verify configuration was populated correctly + if config.Name != "test-app" { + t.Errorf("Expected Name to be 'test-app', got %s", config.Name) + } + if config.Port != 8080 { + t.Errorf("Expected Port to be 8080, got %d", config.Port) + } + if !config.Enabled { + t.Errorf("Expected Enabled to be true, got %v", config.Enabled) + } + if config.Debug != "verbose" { + t.Errorf("Expected Debug to be 'verbose', got %s", config.Debug) + } + + // Get field populations + populations := tracker.GetFieldPopulations() + + // Verify we have tracking information for all fields + expectedFields := []string{"Name", "Port", "Enabled", "Debug"} + + for _, fieldPath := range expectedFields { + found := false + for _, pop := range populations { + if pop.FieldPath == fieldPath { + found = true + // Verify basic tracking information + if pop.FeederType != "AffixedEnvFeeder" { + t.Errorf("Expected FeederType 'AffixedEnvFeeder' for field %s, got %s", fieldPath, pop.FeederType) + } + if pop.SourceType != "env_affixed" { + t.Errorf("Expected SourceType 'env_affixed' for field %s, got %s", fieldPath, pop.SourceType) + } + if pop.SourceKey == "" { + t.Errorf("Expected non-empty SourceKey for field %s", fieldPath) + } + if pop.Value == nil { + t.Errorf("Expected non-nil Value for field %s", fieldPath) + } + break + } + } + if !found { + t.Errorf("Field tracking not found for field: %s", fieldPath) + } + } + + // Verify specific field values and source keys in tracking + for _, pop := range populations { + switch pop.FieldPath { + case "Name": + if fmt.Sprintf("%v", pop.Value) != "test-app" { + t.Errorf("Expected tracked value 'test-app' for Name, got %v", pop.Value) + } + if pop.SourceKey != "APP__NAME__PROD" { + t.Errorf("Expected SourceKey 'APP__NAME__PROD' for Name, got %s", pop.SourceKey) + } + case "Port": + if fmt.Sprintf("%v", pop.Value) != "8080" { + t.Errorf("Expected tracked value '8080' for Port, got %v", pop.Value) + } + if pop.SourceKey != "APP__PORT__PROD" { + t.Errorf("Expected SourceKey 'APP__PORT__PROD' for Port, got %s", pop.SourceKey) + } + case "Enabled": + if fmt.Sprintf("%v", pop.Value) != "true" { + t.Errorf("Expected tracked value 'true' for Enabled, got %v", pop.Value) + } + if pop.SourceKey != "APP__ENABLED__PROD" { + t.Errorf("Expected SourceKey 'APP__ENABLED__PROD' for Enabled, got %s", pop.SourceKey) + } + case "Debug": + if fmt.Sprintf("%v", pop.Value) != "verbose" { + t.Errorf("Expected tracked value 'verbose' for Debug, got %v", pop.Value) + } + if pop.SourceKey != "APP__DEBUG__PROD" { + t.Errorf("Expected SourceKey 'APP__DEBUG__PROD' for Debug, got %s", pop.SourceKey) + } + } + } +} + +func TestAffixedEnvFeeder_SetFieldTracker(t *testing.T) { + feeder := NewAffixedEnvFeeder("PREFIX_", "_SUFFIX") + tracker := NewDefaultFieldTracker() + + // Test that SetFieldTracker method exists and can be called + feeder.SetFieldTracker(tracker) + + // The actual tracking functionality is tested in TestAffixedEnvFeeder_FieldTracking +} + +func TestAffixedEnvFeeder_WithoutFieldTracker(t *testing.T) { + // Set up environment variables + os.Setenv("TEST__NAME__DEV", "test-app") + os.Setenv("TEST__PORT__DEV", "8080") + defer func() { + os.Unsetenv("TEST__NAME__DEV") + os.Unsetenv("TEST__PORT__DEV") + }() + + // Create feeder without field tracker + feeder := NewAffixedEnvFeeder("TEST_", "_DEV") + + var config TestAffixedEnvConfig + + // Should work without field tracker + err := feeder.Feed(&config) + if err != nil { + t.Fatalf("Failed to feed config without field tracker: %v", err) + } + + // Verify configuration was populated correctly + if config.Name != "test-app" { + t.Errorf("Expected Name to be 'test-app', got %s", config.Name) + } + if config.Port != 8080 { + t.Errorf("Expected Port to be 8080, got %d", config.Port) + } +} diff --git a/feeders/consistency_test.go b/feeders/consistency_test.go new file mode 100644 index 00000000..67ecb192 --- /dev/null +++ b/feeders/consistency_test.go @@ -0,0 +1,250 @@ +package feeders + +import ( + "os" + "reflect" + "testing" +) + +// TestConsistentBehavior verifies that feeders behave consistently regardless of field tracking state +func TestConsistentBehavior(t *testing.T) { + tests := []struct { + name string + fileContent string + fileExt string + }{ + { + name: "YAML_Feeder", + fileContent: ` +app: + name: TestApp + version: "1.0" + debug: true +database: + host: localhost + port: 5432 +`, + fileExt: ".yaml", + }, + { + name: "JSON_Feeder", + fileContent: `{ + "app": { + "name": "TestApp", + "version": "1.0", + "debug": true + }, + "database": { + "host": "localhost", + "port": 5432 + } +}`, + fileExt: ".json", + }, + { + name: "TOML_Feeder", + fileContent: ` +[app] +name = "TestApp" +version = "1.0" +debug = true + +[database] +host = "localhost" +port = 5432 +`, + fileExt: ".toml", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create temporary file + tempFile, err := os.CreateTemp("", "test-*"+tt.fileExt) + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + if _, err := tempFile.Write([]byte(tt.fileContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + // Test based on file extension + switch tt.fileExt { + case ".yaml": + t.Run("YAML_consistency", func(t *testing.T) { + testYAMLConsistency(t, tempFile.Name()) + }) + case ".json": + t.Run("JSON_consistency", func(t *testing.T) { + testJSONConsistency(t, tempFile.Name()) + }) + case ".toml": + t.Run("TOML_consistency", func(t *testing.T) { + testTOMLConsistency(t, tempFile.Name()) + }) + } + }) + } +} + +func testYAMLConsistency(t *testing.T, filePath string) { + type Config struct { + App struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Debug bool `yaml:"debug"` + } `yaml:"app"` + Database struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + } `yaml:"database"` + } + + // Test without field tracking + var configWithoutTracking Config + feederWithoutTracking := NewYamlFeeder(filePath) + err := feederWithoutTracking.Feed(&configWithoutTracking) + if err != nil { + t.Fatalf("Expected no error without field tracking, got %v", err) + } + + // Test with field tracking + var configWithTracking Config + feederWithTracking := NewYamlFeeder(filePath) + tracker := NewDefaultFieldTracker() + feederWithTracking.SetFieldTracker(tracker) + err = feederWithTracking.Feed(&configWithTracking) + if err != nil { + t.Fatalf("Expected no error with field tracking, got %v", err) + } + + // Verify that both configs are identical + if !reflect.DeepEqual(configWithoutTracking, configWithTracking) { + t.Errorf("YAML configs should be identical regardless of field tracking state") + t.Errorf("Without tracking: %+v", configWithoutTracking) + t.Errorf("With tracking: %+v", configWithTracking) + } + + verifyConfigValues(t, configWithTracking.App.Name, configWithTracking.App.Version, configWithTracking.App.Debug, configWithTracking.Database.Host, configWithTracking.Database.Port) + + // Verify field tracking recorded something (when enabled) + populations := tracker.GetFieldPopulations() + if len(populations) == 0 { + t.Error("Expected field populations to be recorded when field tracking is enabled") + } +} + +func testJSONConsistency(t *testing.T, filePath string) { + type Config struct { + App struct { + Name string `json:"name"` + Version string `json:"version"` + Debug bool `json:"debug"` + } `json:"app"` + Database struct { + Host string `json:"host"` + Port int `json:"port"` + } `json:"database"` + } + + // Test without field tracking + var configWithoutTracking Config + feederWithoutTracking := NewJSONFeeder(filePath) + err := feederWithoutTracking.Feed(&configWithoutTracking) + if err != nil { + t.Fatalf("Expected no error without field tracking, got %v", err) + } + + // Test with field tracking + var configWithTracking Config + feederWithTracking := NewJSONFeeder(filePath) + tracker := NewDefaultFieldTracker() + feederWithTracking.SetFieldTracker(tracker) + err = feederWithTracking.Feed(&configWithTracking) + if err != nil { + t.Fatalf("Expected no error with field tracking, got %v", err) + } + + // Verify that both configs are identical + if !reflect.DeepEqual(configWithoutTracking, configWithTracking) { + t.Errorf("JSON configs should be identical regardless of field tracking state") + t.Errorf("Without tracking: %+v", configWithoutTracking) + t.Errorf("With tracking: %+v", configWithTracking) + } + + verifyConfigValues(t, configWithTracking.App.Name, configWithTracking.App.Version, configWithTracking.App.Debug, configWithTracking.Database.Host, configWithTracking.Database.Port) + + // Verify field tracking recorded something (when enabled) + populations := tracker.GetFieldPopulations() + if len(populations) == 0 { + t.Error("Expected field populations to be recorded when field tracking is enabled") + } +} + +func testTOMLConsistency(t *testing.T, filePath string) { + type Config struct { + App struct { + Name string `toml:"name"` + Version string `toml:"version"` + Debug bool `toml:"debug"` + } `toml:"app"` + Database struct { + Host string `toml:"host"` + Port int `toml:"port"` + } `toml:"database"` + } + + // Test without field tracking + var configWithoutTracking Config + feederWithoutTracking := NewTomlFeeder(filePath) + err := feederWithoutTracking.Feed(&configWithoutTracking) + if err != nil { + t.Fatalf("Expected no error without field tracking, got %v", err) + } + + // Test with field tracking + var configWithTracking Config + feederWithTracking := NewTomlFeeder(filePath) + tracker := NewDefaultFieldTracker() + feederWithTracking.SetFieldTracker(tracker) + err = feederWithTracking.Feed(&configWithTracking) + if err != nil { + t.Fatalf("Expected no error with field tracking, got %v", err) + } + + // Verify that both configs are identical + if !reflect.DeepEqual(configWithoutTracking, configWithTracking) { + t.Errorf("TOML configs should be identical regardless of field tracking state") + t.Errorf("Without tracking: %+v", configWithoutTracking) + t.Errorf("With tracking: %+v", configWithTracking) + } + + verifyConfigValues(t, configWithTracking.App.Name, configWithTracking.App.Version, configWithTracking.App.Debug, configWithTracking.Database.Host, configWithTracking.Database.Port) + + // Verify field tracking recorded something (when enabled) + populations := tracker.GetFieldPopulations() + if len(populations) == 0 { + t.Error("Expected field populations to be recorded when field tracking is enabled") + } +} + +func verifyConfigValues(t *testing.T, name, version string, debug bool, host string, port int) { + if name != "TestApp" { + t.Errorf("Expected App.Name to be 'TestApp', got '%s'", name) + } + if version != "1.0" { + t.Errorf("Expected App.Version to be '1.0', got '%s'", version) + } + if !debug { + t.Errorf("Expected App.Debug to be true, got false") + } + if host != "localhost" { + t.Errorf("Expected Database.Host to be 'localhost', got '%s'", host) + } + if port != 5432 { + t.Errorf("Expected Database.Port to be 5432, got %d", port) + } +} diff --git a/feeders/dot_env.go b/feeders/dot_env.go index 1bb9143e..d06b6b7d 100644 --- a/feeders/dot_env.go +++ b/feeders/dot_env.go @@ -5,16 +5,19 @@ import ( "fmt" "os" "reflect" + "strconv" "strings" ) -// DotEnvFeeder is a feeder that reads .env files with optional verbose debug logging +// DotEnvFeeder is a feeder that reads .env files and populates configuration directly from the parsed values type DotEnvFeeder struct { Path string verboseDebug bool logger interface { Debug(msg string, args ...any) } + fieldTracker FieldTracker + envVars map[string]string // in-memory storage of parsed .env variables } // NewDotEnvFeeder creates a new DotEnvFeeder that reads from the specified .env file @@ -23,6 +26,8 @@ func NewDotEnvFeeder(filePath string) *DotEnvFeeder { Path: filePath, verboseDebug: false, logger: nil, + fieldTracker: nil, + envVars: make(map[string]string), } } @@ -35,35 +40,47 @@ func (f *DotEnvFeeder) SetVerboseDebug(enabled bool, logger interface{ Debug(msg } } -// Feed reads the .env file and populates the provided structure +// SetFieldTracker sets the field tracker for recording field populations +func (f *DotEnvFeeder) SetFieldTracker(tracker FieldTracker) { + f.fieldTracker = tracker +} + +// Feed reads the .env file and populates the provided structure directly func (f *DotEnvFeeder) Feed(structure interface{}) error { if f.verboseDebug && f.logger != nil { f.logger.Debug("DotEnvFeeder: Starting feed process", "filePath", f.Path, "structureType", reflect.TypeOf(structure)) } - // Load environment variables from .env file - err := f.loadDotEnvFile() + // Parse the .env file into memory first (for tracking purposes) + err := f.parseDotEnvFile() if err != nil { if f.verboseDebug && f.logger != nil { - f.logger.Debug("DotEnvFeeder: Failed to load .env file", "filePath", f.Path, "error", err) + f.logger.Debug("DotEnvFeeder: Failed to parse .env file", "filePath", f.Path, "error", err) } - return fmt.Errorf("failed to load .env file: %w", err) + return fmt.Errorf("failed to parse .env file: %w", err) } - // Use the env feeder logic to populate the structure - envFeeder := &EnvFeeder{ - verboseDebug: f.verboseDebug, - logger: f.logger, + // Load into global environment catalog for other env feeders to use + catalog := GetGlobalEnvCatalog() + catalogErr := catalog.LoadFromDotEnv(f.Path) + if catalogErr != nil && f.verboseDebug && f.logger != nil { + f.logger.Debug("DotEnvFeeder: Failed to load into global catalog", "error", catalogErr) + // Don't fail the operation if catalog loading fails } - return envFeeder.Feed(structure) + + // Populate the structure from the global catalog (respects OS env precedence) + return f.populateStructFromCatalog(structure, "") } -// loadDotEnvFile loads environment variables from the .env file -func (f *DotEnvFeeder) loadDotEnvFile() error { +// parseDotEnvFile parses the .env file into the envVars map +func (f *DotEnvFeeder) parseDotEnvFile() error { if f.verboseDebug && f.logger != nil { - f.logger.Debug("DotEnvFeeder: Loading .env file", "filePath", f.Path) + f.logger.Debug("DotEnvFeeder: Parsing .env file", "filePath", f.Path) } + // Clear existing parsed values + f.envVars = make(map[string]string) + file, err := os.Open(f.Path) if err != nil { if f.verboseDebug && f.logger != nil { @@ -104,14 +121,12 @@ func (f *DotEnvFeeder) loadDotEnvFile() error { } if f.verboseDebug && f.logger != nil { - f.logger.Debug("DotEnvFeeder: Successfully loaded .env file", "filePath", f.Path, "linesProcessed", lineNum) + f.logger.Debug("DotEnvFeeder: Successfully parsed .env file", "filePath", f.Path, "linesProcessed", lineNum, "varsFound", len(f.envVars)) } return nil } -// parseEnvLine parses a single line from the .env file -var ErrDotEnvInvalidLineFormat = fmt.Errorf("invalid .env line format") - +// parseEnvLine parses a single line from the .env file and stores it in memory func (f *DotEnvFeeder) parseEnvLine(line string, lineNum int) error { // Find the first = character idx := strings.Index(line, "=") @@ -130,10 +145,153 @@ func (f *DotEnvFeeder) parseEnvLine(line string, lineNum int) error { } if f.verboseDebug && f.logger != nil { - f.logger.Debug("DotEnvFeeder: Setting environment variable", "key", key, "value", value, "lineNum", lineNum) + f.logger.Debug("DotEnvFeeder: Parsed variable", "key", key, "value", value, "lineNum", lineNum) } - // Set the environment variable - os.Setenv(key, value) + // Store the variable in memory (do NOT set in environment) + f.envVars[key] = value return nil } + +// populateStructFromCatalog populates struct fields from the global environment catalog +func (f *DotEnvFeeder) populateStructFromCatalog(structure interface{}, prefix string) error { + structValue := reflect.ValueOf(structure) + if structValue.Kind() != reflect.Ptr || structValue.Elem().Kind() != reflect.Struct { + return wrapDotEnvStructureError(structure) + } + + return f.processStructFieldsFromCatalog(structValue.Elem(), prefix) +} + +// processStructFieldsFromCatalog iterates through struct fields and populates them from the global catalog +func (f *DotEnvFeeder) processStructFieldsFromCatalog(rv reflect.Value, prefix string) error { + structType := rv.Type() + catalog := GetGlobalEnvCatalog() + + for i := 0; i < rv.NumField(); i++ { + field := rv.Field(i) + fieldType := structType.Field(i) + + // Skip unexported fields + if !field.CanSet() { + continue + } + + // Get env tag + envTag := fieldType.Tag.Get("env") + if envTag == "" || envTag == "-" { + // Handle nested structs + if field.Kind() == reflect.Struct { + // For DotEnv, we don't use prefixes since env tags should be complete + err := f.processStructFieldsFromCatalog(field, prefix) + if err != nil { + return err + } + } + continue + } + + // Build the field path for tracking + fieldPath := fieldType.Name + if prefix != "" { + fieldPath = prefix + "." + fieldPath + } + + // For DotEnv, use the env tag directly as it should contain the complete variable name + envKey := envTag + + // Get value from catalog + value, exists := catalog.Get(envKey) + if !exists || value == "" { + if f.verboseDebug && f.logger != nil { + f.logger.Debug("DotEnvFeeder: Environment variable not found", "envKey", envKey, "fieldPath", fieldPath) + } + continue + } + + if f.verboseDebug && f.logger != nil { + source := catalog.GetSource(envKey) + f.logger.Debug("DotEnvFeeder: Setting field from catalog", "envKey", envKey, "value", value, "fieldPath", fieldPath, "source", source) + } + + // Set the field value + err := f.setFieldValue(field, fieldType, value, fieldPath, envKey) + if err != nil { + return err + } + } + + return nil +} + +// setFieldValue sets a field value from .env data with type conversion +func (f *DotEnvFeeder) setFieldValue(field reflect.Value, fieldType reflect.StructField, value, fieldPath, envKey string) error { + if f.verboseDebug && f.logger != nil { + f.logger.Debug("DotEnvFeeder: Setting field", "fieldPath", fieldPath, "envKey", envKey, "value", value, "fieldType", field.Type()) + } + + // Convert the string value to the appropriate type + convertedValue, err := f.convertStringToType(value, field.Type()) + if err != nil { + return fmt.Errorf("failed to convert value '%s' for field %s: %w", value, fieldPath, err) + } + + // Set the field value + field.Set(reflect.ValueOf(convertedValue)) + + // Record field population + if f.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldType.Name, + FieldType: field.Type().String(), + FeederType: "DotEnvFeeder", + SourceType: "dot_env_file", + SourceKey: envKey, + Value: convertedValue, + SearchKeys: []string{envKey}, + FoundKey: envKey, + } + f.fieldTracker.RecordFieldPopulation(fp) + } + + return nil +} + +// convertStringToType converts a string value to the target type +func (f *DotEnvFeeder) convertStringToType(value string, targetType reflect.Type) (interface{}, error) { + switch targetType.Kind() { + case reflect.String: + return value, nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + intVal, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot convert '%s' to %s: %w", value, targetType.Kind(), err) + } + return reflect.ValueOf(intVal).Convert(targetType).Interface(), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + uintVal, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return nil, fmt.Errorf("cannot convert '%s' to %s: %w", value, targetType.Kind(), err) + } + return reflect.ValueOf(uintVal).Convert(targetType).Interface(), nil + case reflect.Float32, reflect.Float64: + floatVal, err := strconv.ParseFloat(value, 64) + if err != nil { + return nil, fmt.Errorf("cannot convert '%s' to %s: %w", value, targetType.Kind(), err) + } + return reflect.ValueOf(floatVal).Convert(targetType).Interface(), nil + case reflect.Bool: + boolVal, err := strconv.ParseBool(value) + if err != nil { + return nil, fmt.Errorf("cannot convert '%s' to bool: %w", value, err) + } + return boolVal, nil + case reflect.Invalid, reflect.Uintptr, reflect.Complex64, reflect.Complex128, + reflect.Array, reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, + reflect.Ptr, reflect.Slice, reflect.Struct, reflect.UnsafePointer: + return nil, wrapDotEnvUnsupportedTypeError(targetType.Kind().String()) + default: + return nil, wrapDotEnvUnsupportedTypeError(targetType.Kind().String()) + } +} diff --git a/feeders/dot_env_field_tracking_test.go b/feeders/dot_env_field_tracking_test.go new file mode 100644 index 00000000..3f8073d2 --- /dev/null +++ b/feeders/dot_env_field_tracking_test.go @@ -0,0 +1,165 @@ +package feeders + +import ( + "fmt" + "os" + "testing" +) + +// TestConfig struct for DotEnv field tracking tests +type TestDotEnvConfig struct { + Name string `env:"NAME"` + Port int `env:"PORT"` + Enabled bool `env:"ENABLED"` + Debug string `env:"DEBUG"` +} + +func TestDotEnvFeeder_FieldTracking(t *testing.T) { + // Create test .env file + envContent := `NAME=test-app +PORT=8080 +ENABLED=true +DEBUG=verbose +UNUSED_VAR=ignored +` + + // Create temporary .env file + tmpFile, err := os.CreateTemp("", "test_config_*.env") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + + if _, err := tmpFile.WriteString(envContent); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tmpFile.Close() + + // Create feeder and field tracker + feeder := NewDotEnvFeeder(tmpFile.Name()) + tracker := NewDefaultFieldTracker() + feeder.SetFieldTracker(tracker) + + // Test config structure + var config TestDotEnvConfig + + // Feed the configuration + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Failed to feed config: %v", err) + } + + // Verify configuration was populated correctly + if config.Name != "test-app" { + t.Errorf("Expected Name to be 'test-app', got %s", config.Name) + } + if config.Port != 8080 { + t.Errorf("Expected Port to be 8080, got %d", config.Port) + } + if !config.Enabled { + t.Errorf("Expected Enabled to be true, got %v", config.Enabled) + } + if config.Debug != "verbose" { + t.Errorf("Expected Debug to be 'verbose', got %s", config.Debug) + } + + // Get field populations + populations := tracker.GetFieldPopulations() + + // Verify we have tracking information for all fields + expectedFields := []string{"Name", "Port", "Enabled", "Debug"} + + for _, fieldPath := range expectedFields { + found := false + for _, pop := range populations { + if pop.FieldPath == fieldPath { + found = true + // Verify basic tracking information + if pop.FeederType != "DotEnvFeeder" { + t.Errorf("Expected FeederType 'DotEnvFeeder' for field %s, got %s", fieldPath, pop.FeederType) + } + if pop.SourceType != "dot_env_file" { + t.Errorf("Expected SourceType 'dot_env_file' for field %s, got %s", fieldPath, pop.SourceType) + } + if pop.SourceKey == "" { + t.Errorf("Expected non-empty SourceKey for field %s", fieldPath) + } + if pop.Value == nil { + t.Errorf("Expected non-nil Value for field %s", fieldPath) + } + break + } + } + if !found { + t.Errorf("Field tracking not found for field: %s", fieldPath) + } + } + + // Verify specific field values in tracking + for _, pop := range populations { + switch pop.FieldPath { + case "Name": + if fmt.Sprintf("%v", pop.Value) != "test-app" { + t.Errorf("Expected tracked value 'test-app' for Name, got %v", pop.Value) + } + case "Port": + if fmt.Sprintf("%v", pop.Value) != "8080" { + t.Errorf("Expected tracked value '8080' for Port, got %v", pop.Value) + } + case "Enabled": + if fmt.Sprintf("%v", pop.Value) != "true" { + t.Errorf("Expected tracked value 'true' for Enabled, got %v", pop.Value) + } + case "Debug": + if fmt.Sprintf("%v", pop.Value) != "verbose" { + t.Errorf("Expected tracked value 'verbose' for Debug, got %v", pop.Value) + } + } + } +} + +func TestDotEnvFeeder_SetFieldTracker(t *testing.T) { + feeder := NewDotEnvFeeder("test.env") + tracker := NewDefaultFieldTracker() + + // Test that SetFieldTracker method exists and can be called + feeder.SetFieldTracker(tracker) + + // The actual tracking functionality is tested in TestDotEnvFeeder_FieldTracking +} + +func TestDotEnvFeeder_WithoutFieldTracker(t *testing.T) { + // Create test .env file + envContent := `NAME=test-app +PORT=8080` + + tmpFile, err := os.CreateTemp("", "test_config_*.env") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + + if _, err := tmpFile.WriteString(envContent); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tmpFile.Close() + + // Create feeder without field tracker + feeder := NewDotEnvFeeder(tmpFile.Name()) + + var config TestDotEnvConfig + + // Should work without field tracker + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Failed to feed config without field tracker: %v", err) + } + + // Verify configuration was populated correctly + if config.Name != "test-app" { + t.Errorf("Expected Name to be 'test-app', got %s", config.Name) + } + if config.Port != 8080 { + t.Errorf("Expected Port to be 8080, got %d", config.Port) + } +} diff --git a/feeders/dotenv_debug_test.go b/feeders/dotenv_debug_test.go new file mode 100644 index 00000000..0e1ecef5 --- /dev/null +++ b/feeders/dotenv_debug_test.go @@ -0,0 +1,60 @@ +package feeders + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDotEnvFeederDebug(t *testing.T) { + // Create test .env file + envContent := []byte(` +DB_HOST=localhost +DB_PORT=5432 +DB_USER=postgres +DB_PASS=secret +`) + + tempFile := filepath.Join(os.TempDir(), "debug_test.env") + err := os.WriteFile(tempFile, envContent, 0600) + if err != nil { + t.Fatalf("Failed to create test .env file: %v", err) + } + defer os.Remove(tempFile) + + type Config struct { + DB struct { + Host string `env:"DB_HOST"` + Port int `env:"DB_PORT"` + User string `env:"DB_USER"` + Password string `env:"DB_PASS"` + } + } + + var config Config + feeder := NewDotEnvFeeder(tempFile) + + // Enable debug logging + logger := &TestLogger2{} + feeder.SetVerboseDebug(true, logger) + + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Feed failed: %v", err) + } + + t.Logf("Config after feeding:") + t.Logf(" DB.Host: '%s'", config.DB.Host) + t.Logf(" DB.Port: %d", config.DB.Port) + t.Logf(" DB.User: '%s'", config.DB.User) + t.Logf(" DB.Password: '%s'", config.DB.Password) + + // Test direct catalog access + catalog := GetGlobalEnvCatalog() + value, exists := catalog.Get("DB_HOST") + t.Logf("Catalog DB_HOST: value='%s', exists=%v", value, exists) + + // Test OS access + osValue := os.Getenv("DB_HOST") + t.Logf("OS DB_HOST: '%s'", osValue) +} diff --git a/feeders/env.go b/feeders/env.go index ae12a6ec..83b4ed95 100644 --- a/feeders/env.go +++ b/feeders/env.go @@ -2,17 +2,17 @@ package feeders import ( "fmt" - "os" "reflect" "strings" ) -// EnvFeeder is a feeder that reads environment variables with optional verbose debug logging +// EnvFeeder is a feeder that reads environment variables with optional verbose debug logging and field tracking type EnvFeeder struct { verboseDebug bool logger interface { Debug(msg string, args ...any) } + fieldTracker FieldTracker } // NewEnvFeeder creates a new EnvFeeder that reads from environment variables @@ -32,6 +32,14 @@ func (f *EnvFeeder) SetVerboseDebug(enabled bool, logger interface{ Debug(msg st } } +// SetFieldTracker sets the field tracker for this feeder +func (f *EnvFeeder) SetFieldTracker(tracker FieldTracker) { + f.fieldTracker = tracker + if f.verboseDebug && f.logger != nil { + f.logger.Debug("EnvFeeder: Field tracker set", "hasTracker", tracker != nil) + } +} + // Feed implements the Feeder interface with optional verbose logging func (f *EnvFeeder) Feed(structure interface{}) error { if f.verboseDebug && f.logger != nil { @@ -64,7 +72,7 @@ func (f *EnvFeeder) Feed(structure interface{}) error { f.logger.Debug("EnvFeeder: Processing struct fields", "structType", inputType.Elem()) } - err := f.processStructFields(reflect.ValueOf(structure).Elem(), "") + err := f.processStructFields(reflect.ValueOf(structure).Elem(), "", "") if f.verboseDebug && f.logger != nil { if err != nil { @@ -78,22 +86,28 @@ func (f *EnvFeeder) Feed(structure interface{}) error { } // processStructFields processes all fields in a struct with optional verbose logging -func (f *EnvFeeder) processStructFields(rv reflect.Value, prefix string) error { +func (f *EnvFeeder) processStructFields(rv reflect.Value, prefix, parentPath string) error { structType := rv.Type() if f.verboseDebug && f.logger != nil { - f.logger.Debug("EnvFeeder: Processing struct", "structType", structType, "numFields", rv.NumField(), "prefix", prefix) + f.logger.Debug("EnvFeeder: Processing struct", "structType", structType, "numFields", rv.NumField(), "prefix", prefix, "parentPath", parentPath) } for i := 0; i < rv.NumField(); i++ { field := rv.Field(i) fieldType := structType.Field(i) + // Build field path + fieldPath := fieldType.Name + if parentPath != "" { + fieldPath = parentPath + "." + fieldType.Name + } + if f.verboseDebug && f.logger != nil { - f.logger.Debug("EnvFeeder: Processing field", "fieldName", fieldType.Name, "fieldType", fieldType.Type, "fieldKind", field.Kind()) + f.logger.Debug("EnvFeeder: Processing field", "fieldName", fieldType.Name, "fieldType", fieldType.Type, "fieldKind", field.Kind(), "fieldPath", fieldPath) } - if err := f.processField(field, &fieldType, prefix); err != nil { + if err := f.processField(field, &fieldType, prefix, fieldPath); err != nil { if f.verboseDebug && f.logger != nil { f.logger.Debug("EnvFeeder: Field processing failed", "fieldName", fieldType.Name, "error", err) } @@ -108,20 +122,20 @@ func (f *EnvFeeder) processStructFields(rv reflect.Value, prefix string) error { } // processField handles a single struct field with optional verbose logging -func (f *EnvFeeder) processField(field reflect.Value, fieldType *reflect.StructField, prefix string) error { +func (f *EnvFeeder) processField(field reflect.Value, fieldType *reflect.StructField, prefix, fieldPath string) error { // Handle nested structs switch field.Kind() { case reflect.Struct: if f.verboseDebug && f.logger != nil { - f.logger.Debug("EnvFeeder: Processing nested struct", "fieldName", fieldType.Name, "structType", field.Type()) + f.logger.Debug("EnvFeeder: Processing nested struct", "fieldName", fieldType.Name, "structType", field.Type(), "fieldPath", fieldPath) } - return f.processStructFields(field, prefix) + return f.processStructFields(field, prefix, fieldPath) case reflect.Pointer: if !field.IsZero() && field.Elem().Kind() == reflect.Struct { if f.verboseDebug && f.logger != nil { - f.logger.Debug("EnvFeeder: Processing nested struct pointer", "fieldName", fieldType.Name, "structType", field.Elem().Type()) + f.logger.Debug("EnvFeeder: Processing nested struct pointer", "fieldName", fieldType.Name, "structType", field.Elem().Type(), "fieldPath", fieldPath) } - return f.processStructFields(field.Elem(), prefix) + return f.processStructFields(field.Elem(), prefix, fieldPath) } case reflect.Invalid, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, @@ -130,50 +144,89 @@ func (f *EnvFeeder) processField(field reflect.Value, fieldType *reflect.StructF // Check for env tag for primitive types and other non-struct types if envTag, exists := fieldType.Tag.Lookup("env"); exists { if f.verboseDebug && f.logger != nil { - f.logger.Debug("EnvFeeder: Found env tag", "fieldName", fieldType.Name, "envTag", envTag) + f.logger.Debug("EnvFeeder: Found env tag", "fieldName", fieldType.Name, "envTag", envTag, "fieldPath", fieldPath) } - return f.setFieldFromEnv(field, envTag, prefix, fieldType.Name) + return f.setFieldFromEnv(field, envTag, prefix, fieldType.Name, fieldPath) } else if f.verboseDebug && f.logger != nil { - f.logger.Debug("EnvFeeder: No env tag found", "fieldName", fieldType.Name) + f.logger.Debug("EnvFeeder: No env tag found", "fieldName", fieldType.Name, "fieldPath", fieldPath) } } return nil } -// setFieldFromEnv sets a field value from an environment variable with optional verbose logging -func (f *EnvFeeder) setFieldFromEnv(field reflect.Value, envTag, prefix, fieldName string) error { +// setFieldFromEnv sets a field value from an environment variable with optional verbose logging and field tracking +func (f *EnvFeeder) setFieldFromEnv(field reflect.Value, envTag, prefix, fieldName, fieldPath string) error { // Build environment variable name with prefix envName := strings.ToUpper(envTag) if prefix != "" { envName = strings.ToUpper(prefix) + envName } + // Track what we're searching for + searchKeys := []string{envName} + if f.verboseDebug && f.logger != nil { - f.logger.Debug("EnvFeeder: Looking up environment variable", "fieldName", fieldName, "envName", envName, "envTag", envTag, "prefix", prefix) + f.logger.Debug("EnvFeeder: Looking up environment variable", "fieldName", fieldName, "envName", envName, "envTag", envTag, "prefix", prefix, "fieldPath", fieldPath) } // Get and apply environment variable if exists - envValue := os.Getenv(envName) - if envValue != "" { + catalog := GetGlobalEnvCatalog() + envValue, exists := catalog.Get(envName) + if exists && envValue != "" { if f.verboseDebug && f.logger != nil { - f.logger.Debug("EnvFeeder: Environment variable found", "fieldName", fieldName, "envName", envName, "envValue", envValue) + source := catalog.GetSource(envName) + f.logger.Debug("EnvFeeder: Environment variable found", "fieldName", fieldName, "envName", envName, "envValue", envValue, "fieldPath", fieldPath, "source", source) } err := setFieldValue(field, envValue) if err != nil { if f.verboseDebug && f.logger != nil { - f.logger.Debug("EnvFeeder: Failed to set field value", "fieldName", fieldName, "envName", envName, "envValue", envValue, "error", err) + f.logger.Debug("EnvFeeder: Failed to set field value", "fieldName", fieldName, "envName", envName, "envValue", envValue, "error", err, "fieldPath", fieldPath) } return err } + // Record field population if tracker is available + if f.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldName, + FieldType: field.Type().String(), + FeederType: "*feeders.EnvFeeder", + SourceType: "env", + SourceKey: envName, + Value: field.Interface(), + InstanceKey: "", + SearchKeys: searchKeys, + FoundKey: envName, + } + f.fieldTracker.RecordFieldPopulation(fp) + } + if f.verboseDebug && f.logger != nil { - f.logger.Debug("EnvFeeder: Successfully set field value", "fieldName", fieldName, "envName", envName, "envValue", envValue) + f.logger.Debug("EnvFeeder: Successfully set field value", "fieldName", fieldName, "envName", envName, "envValue", envValue, "fieldPath", fieldPath) } } else { + // Record that we searched but didn't find + if f.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldName, + FieldType: field.Type().String(), + FeederType: "*feeders.EnvFeeder", + SourceType: "env", + SourceKey: "", + Value: nil, + InstanceKey: "", + SearchKeys: searchKeys, + FoundKey: "", + } + f.fieldTracker.RecordFieldPopulation(fp) + } + if f.verboseDebug && f.logger != nil { - f.logger.Debug("EnvFeeder: Environment variable not found or empty", "fieldName", fieldName, "envName", envName) + f.logger.Debug("EnvFeeder: Environment variable not found or empty", "fieldName", fieldName, "envName", envName, "fieldPath", fieldPath) } } diff --git a/feeders/env_catalog.go b/feeders/env_catalog.go new file mode 100644 index 00000000..7bae3c34 --- /dev/null +++ b/feeders/env_catalog.go @@ -0,0 +1,212 @@ +package feeders + +import ( + "bufio" + "fmt" + "os" + "strings" + "sync" +) + +// parseDotEnvFile parses a .env file and returns the key-value pairs +func parseDotEnvFile(filename string) (map[string]string, error) { + result := make(map[string]string) + + file, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("failed to open .env file: %w", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + lineNum := 0 + for scanner.Scan() { + lineNum++ + line := strings.TrimSpace(scanner.Text()) + + // Skip empty lines and comments + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + // Parse key=value pairs + idx := strings.Index(line, "=") + if idx == -1 { + return nil, fmt.Errorf("%w at line %d: %s", ErrDotEnvInvalidLineFormat, lineNum, line) + } + + key := strings.TrimSpace(line[:idx]) + value := strings.TrimSpace(line[idx+1:]) + + // Remove quotes if present + if len(value) >= 2 { + if (value[0] == '"' && value[len(value)-1] == '"') || (value[0] == '\'' && value[len(value)-1] == '\'') { + value = value[1 : len(value)-1] + } + } + + result[key] = value + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("scanner error: %w", err) + } + + return result, nil +} + +// EnvCatalog manages a unified view of environment variables from multiple sources +type EnvCatalog struct { + variables map[string]string + mutex sync.RWMutex + sources map[string]string // tracks which source provided each variable +} + +// NewEnvCatalog creates a new environment variable catalog +func NewEnvCatalog() *EnvCatalog { + catalog := &EnvCatalog{ + variables: make(map[string]string), + sources: make(map[string]string), + } + // Load OS environment variables + catalog.loadOSEnvironment() + return catalog +} + +// loadOSEnvironment loads all OS environment variables into the catalog +func (c *EnvCatalog) loadOSEnvironment() { + c.mutex.Lock() + defer c.mutex.Unlock() + + for _, env := range os.Environ() { + if parts := strings.SplitN(env, "=", 2); len(parts) == 2 { + key, value := parts[0], parts[1] + c.variables[key] = value + c.sources[key] = "os_env" + } + } +} + +// LoadFromDotEnv loads variables from a .env file into the catalog +func (c *EnvCatalog) LoadFromDotEnv(filename string) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + dotEnvVars, err := parseDotEnvFile(filename) + if err != nil { + return err + } + + for key, value := range dotEnvVars { + // Only set if not already present (OS env and existing values take precedence) + if _, exists := c.variables[key]; !exists { + // Also check OS environment before setting + if osValue := os.Getenv(key); osValue != "" { + // OS environment takes precedence, set that instead + c.variables[key] = osValue + c.sources[key] = "os_env" + } else { + // No OS env, use .env value + c.variables[key] = value + c.sources[key] = "dotenv:" + filename + } + } + } + + return nil +} + +// Set manually sets a variable in the catalog +func (c *EnvCatalog) Set(key, value, source string) { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.variables[key] = value + c.sources[key] = source +} + +// Get retrieves a variable from the catalog, always checking current OS environment +func (c *EnvCatalog) Get(key string) (string, bool) { + c.mutex.RLock() + defer c.mutex.RUnlock() + + // Always check OS environment first for the most current value + if osValue := os.Getenv(key); osValue != "" { + // If OS env value differs from cached, update cache + if cachedValue, exists := c.variables[key]; !exists || cachedValue != osValue { + c.mutex.RUnlock() + c.mutex.Lock() + c.variables[key] = osValue + if c.sources[key] == "" { + c.sources[key] = "os_env" + } + c.mutex.Unlock() + c.mutex.RLock() + } + return osValue, true + } + + // If not in OS environment, check our internal catalog (for dotenv values) + if value, exists := c.variables[key]; exists { + return value, true + } + + return "", false +} + +// GetSource returns the source that provided a variable +func (c *EnvCatalog) GetSource(key string) string { + c.mutex.RLock() + defer c.mutex.RUnlock() + + return c.sources[key] +} + +// GetAll returns all variables in the catalog +func (c *EnvCatalog) GetAll() map[string]string { + c.mutex.RLock() + defer c.mutex.RUnlock() + + result := make(map[string]string, len(c.variables)) + for k, v := range c.variables { + result[k] = v + } + return result +} + +// Clear removes all variables from the catalog +func (c *EnvCatalog) Clear() { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.variables = make(map[string]string) + c.sources = make(map[string]string) +} + +// ClearDynamicEnvCache clears dynamically loaded environment variables from cache +// This is useful for testing when environment variables change between tests +func (c *EnvCatalog) ClearDynamicEnvCache() { + c.mutex.Lock() + defer c.mutex.Unlock() + + // Remove dynamically loaded env vars but keep initial OS env and dotenv + for key, source := range c.sources { + if source == "os_env_dynamic" { + delete(c.variables, key) + delete(c.sources, key) + } + } +} + +// Global catalog instance for all env-based feeders to use +var globalEnvCatalog = NewEnvCatalog() + +// GetGlobalEnvCatalog returns the global environment catalog +func GetGlobalEnvCatalog() *EnvCatalog { + return globalEnvCatalog +} + +// ResetGlobalEnvCatalog resets the global environment catalog (useful for testing) +func ResetGlobalEnvCatalog() { + globalEnvCatalog = NewEnvCatalog() +} diff --git a/feeders/env_catalog_integration_test.go b/feeders/env_catalog_integration_test.go new file mode 100644 index 00000000..7c5df5bd --- /dev/null +++ b/feeders/env_catalog_integration_test.go @@ -0,0 +1,208 @@ +package feeders + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +// TestEnvCatalogIntegration tests the unified environment catalog with mixed feeders +func TestEnvCatalogIntegration(t *testing.T) { + // Create a temporary .env file + tmpDir := t.TempDir() + envFile := filepath.Join(tmpDir, ".env") + + envContent := `# Test .env file +APP_NAME=MyApp +APP_VERSION=2.0 +DEBUG=true +DATABASE_HOST=localhost +DATABASE_PORT=5432 +` + err := os.WriteFile(envFile, []byte(envContent), 0600) + if err != nil { + t.Fatalf("Failed to create test .env file: %v", err) + } + + // Test configuration structures + type DatabaseConfig struct { + Host string `env:"HOST"` + Port int `env:"PORT"` + } + + type AppConfig struct { + Name string `env:"APP_NAME"` + Version string `env:"APP_VERSION"` + Debug bool `env:"DEBUG"` + Database DatabaseConfig + } + + t.Run("DotEnv + EnvFeeder integration", func(t *testing.T) { + // Reset global catalog for clean test + ResetGlobalEnvCatalog() + + // Set some OS environment variables that will override .env + t.Setenv("APP_VERSION", "3.0") // This should override .env value + + var config AppConfig + + // First, load .env file using DotEnvFeeder + dotEnvFeeder := NewDotEnvFeeder(envFile) + tracker := NewDefaultFieldTracker() + dotEnvFeeder.SetFieldTracker(tracker) + + err := dotEnvFeeder.Feed(&config) + if err != nil { + t.Fatalf("DotEnvFeeder failed: %v", err) + } + + // Verify .env values were loaded + if config.Name != "MyApp" { + t.Errorf("Expected Name 'MyApp', got '%s'", config.Name) + } + if config.Version != "3.0" { // OS env should override .env + t.Errorf("Expected Version '3.0' (OS override), got '%s'", config.Version) + } + if !config.Debug { + t.Errorf("Expected Debug true, got %v", config.Debug) + } + + // Now use EnvFeeder to populate additional fields not handled by DotEnv + envFeeder := NewEnvFeeder() + envFeeder.SetFieldTracker(tracker) + + // This should work because .env values are now in the global catalog + err = envFeeder.Feed(&config) + if err != nil { + t.Fatalf("EnvFeeder failed: %v", err) + } + + // Verify field tracking captured both sources + populations := tracker.GetFieldPopulations() + foundSources := make(map[string]string) + for _, pop := range populations { + foundSources[pop.FieldPath] = pop.SourceType + } + + // Should have entries from both feeders + if len(populations) < 3 { + t.Errorf("Expected at least 3 field populations, got %d", len(populations)) + } + }) + + t.Run("DotEnv + AffixedEnvFeeder integration", func(t *testing.T) { + // Reset global catalog + ResetGlobalEnvCatalog() + + // Set up environment variables with prefix/suffix pattern + // AffixedEnvFeeder with prefix "PROD_" and suffix "_ENV" + // constructs: prefix + "_" + envTag + "_" + suffix + // For env:"HOST" -> "PROD_" + "_" + "HOST" + "_" + "_ENV" = "PROD__HOST__ENV" + t.Setenv("PROD__HOST__ENV", "prod.example.com") + t.Setenv("PROD__PORT__ENV", "3306") + + var config AppConfig + + // Load .env first + dotEnvFeeder := NewDotEnvFeeder(envFile) + err := dotEnvFeeder.Feed(&config) + if err != nil { + t.Fatalf("DotEnvFeeder failed: %v", err) + } + + // Now use AffixedEnvFeeder for database config + affixedFeeder := NewAffixedEnvFeeder("PROD_", "_ENV") + tracker := NewDefaultFieldTracker() + affixedFeeder.SetFieldTracker(tracker) + + err = affixedFeeder.Feed(&config.Database) + if err != nil { + t.Fatalf("AffixedEnvFeeder failed: %v", err) + } + + // Verify values from both sources + if config.Name != "MyApp" { // From .env + t.Errorf("Expected Name 'MyApp', got '%s'", config.Name) + } + if config.Database.Host != "prod.example.com" { // From affixed env + t.Errorf("Expected Database.Host 'prod.example.com', got '%s'", config.Database.Host) + } + if config.Database.Port != 3306 { // From affixed env + t.Errorf("Expected Database.Port 3306, got %d", config.Database.Port) + } + + // Verify field tracking + populations := tracker.GetFieldPopulations() + if len(populations) != 2 { + t.Errorf("Expected 2 field populations for database, got %d", len(populations)) + } + + // Check that source tracking is correct + for _, pop := range populations { + if pop.SourceType != "env_affixed" { + t.Errorf("Expected source type 'env_affixed', got '%s'", pop.SourceType) + } + } + }) + + t.Run("Feeder evaluation order", func(t *testing.T) { + // Reset global catalog + ResetGlobalEnvCatalog() + + // Test that OS environment takes precedence over .env + t.Setenv("APP_NAME", "OSOverride") + + var config AppConfig + + // Load .env first + dotEnvFeeder := NewDotEnvFeeder(envFile) + err := dotEnvFeeder.Feed(&config) + if err != nil { + t.Fatalf("DotEnvFeeder failed: %v", err) + } + + // OS env should take precedence + if config.Name != "OSOverride" { + t.Errorf("Expected Name 'OSOverride' (OS precedence), got '%s'", config.Name) + } + + // But .env values should still be available for other fields + if config.Version != "2.0" { + t.Errorf("Expected Version '2.0' (from .env), got '%s'", config.Version) + } + }) + + t.Run("Catalog source tracking", func(t *testing.T) { + // Reset global catalog + ResetGlobalEnvCatalog() + + catalog := GetGlobalEnvCatalog() + + // Load .env file + err := catalog.LoadFromDotEnv(envFile) + if err != nil { + t.Fatalf("Failed to load .env into catalog: %v", err) + } + + // Set OS env var + t.Setenv("TEST_OS_VAR", "os_value") + + // Check sources + envSource := catalog.GetSource("APP_NAME") + if envSource != fmt.Sprintf("dotenv:%s", envFile) { + t.Errorf("Expected source 'dotenv:%s', got '%s'", envFile, envSource) + } + + // Get OS var (should be detected and cached) + osValue, exists := catalog.Get("TEST_OS_VAR") + if !exists || osValue != "os_value" { + t.Errorf("Expected OS var 'os_value', got '%s', exists: %v", osValue, exists) + } + + osSource := catalog.GetSource("TEST_OS_VAR") + if osSource != "os_env" { + t.Errorf("Expected source 'os_env', got '%s'", osSource) + } + }) +} diff --git a/feeders/env_var_debug_test.go b/feeders/env_var_debug_test.go new file mode 100644 index 00000000..34abc653 --- /dev/null +++ b/feeders/env_var_debug_test.go @@ -0,0 +1,52 @@ +package feeders + +import ( + "os" + "testing" +) + +func TestEnvVarConstruction(t *testing.T) { + // Set environment variables + t.Setenv("PROD_HOST_ENV", "prod.example.com") + t.Setenv("PROD_PORT_ENV", "3306") + + // Test catalog directly + catalog := GetGlobalEnvCatalog() + + // Check if catalog can find these vars + value1, exists1 := catalog.Get("PROD_HOST_ENV") + t.Logf("PROD_HOST_ENV: value='%s', exists=%v", value1, exists1) + + value2, exists2 := catalog.Get("PROD_PORT_ENV") + t.Logf("PROD_PORT_ENV: value='%s', exists=%v", value2, exists2) + + // Test direct OS lookup + osValue1 := os.Getenv("PROD_HOST_ENV") + osValue2 := os.Getenv("PROD_PORT_ENV") + t.Logf("Direct OS: PROD_HOST_ENV='%s', PROD_PORT_ENV='%s'", osValue1, osValue2) + + // Test what AffixedEnvFeeder constructs + // With prefix "PROD_" and suffix "_ENV", for field tagged env:"HOST" + // It should construct: ToUpper("PROD_") + "_" + ToUpper("HOST") + "_" + ToUpper("_ENV") + // = "PROD_" + "_" + "HOST" + "_" + "_ENV" = "PROD__HOST__ENV" + + expectedVar1 := "PROD__HOST__ENV" + expectedVar2 := "PROD__PORT__ENV" + + testValue1, testExists1 := catalog.Get(expectedVar1) + testValue2, testExists2 := catalog.Get(expectedVar2) + t.Logf("Expected vars: %s='%s' (exists=%v), %s='%s' (exists=%v)", + expectedVar1, testValue1, testExists1, + expectedVar2, testValue2, testExists2) + + // Set the expected variables + t.Setenv(expectedVar1, "prod.example.com") + t.Setenv(expectedVar2, "3306") + + // Test again + testValue1b, testExists1b := catalog.Get(expectedVar1) + testValue2b, testExists2b := catalog.Get(expectedVar2) + t.Logf("After setting expected vars: %s='%s' (exists=%v), %s='%s' (exists=%v)", + expectedVar1, testValue1b, testExists1b, + expectedVar2, testValue2b, testExists2b) +} diff --git a/feeders/errors.go b/feeders/errors.go new file mode 100644 index 00000000..e6222595 --- /dev/null +++ b/feeders/errors.go @@ -0,0 +1,103 @@ +package feeders + +import ( + "errors" + "fmt" +) + +// Static error definitions for feeders to comply with linting rules + +// DotEnv feeder errors +var ( + ErrDotEnvInvalidStructureType = errors.New("expected pointer to struct") + ErrDotEnvUnsupportedType = errors.New("unsupported type") + ErrDotEnvInvalidLineFormat = errors.New("invalid .env line format") +) + +// JSON feeder errors +var ( + ErrJSONExpectedMapForStruct = errors.New("expected map for struct field") + ErrJSONCannotConvert = errors.New("cannot convert value to field type") + ErrJSONCannotConvertSliceElement = errors.New("cannot convert slice element") + ErrJSONExpectedArrayForSlice = errors.New("expected array for slice field") +) + +// TOML feeder errors +var ( + ErrTomlExpectedMapForStruct = errors.New("expected map for struct field") + ErrTomlCannotConvert = errors.New("cannot convert value to field type") + ErrTomlCannotConvertSliceElement = errors.New("cannot convert slice element") + ErrTomlExpectedArrayForSlice = errors.New("expected array for slice field") +) + +// YAML feeder errors +var ( + ErrYamlFieldCannotBeSet = errors.New("field cannot be set") + ErrYamlUnsupportedFieldType = errors.New("unsupported field type") + ErrYamlTypeConversion = errors.New("type conversion error") + ErrYamlBoolConversion = errors.New("cannot convert string to bool") +) + +// General feeder errors +var ( + ErrJsonFeederUnavailable = errors.New("json feeder unavailable") + ErrTomlFeederUnavailable = errors.New("toml feeder unavailable") +) + +// Helper functions to create wrapped errors with context +func wrapDotEnvStructureError(got interface{}) error { + return fmt.Errorf("%w, got %T", ErrDotEnvInvalidStructureType, got) +} + +func wrapDotEnvUnsupportedTypeError(typeName string) error { + return fmt.Errorf("%w: %s", ErrDotEnvUnsupportedType, typeName) +} + +func wrapJSONMapError(fieldPath string, got interface{}) error { + return fmt.Errorf("%w %s, got %T", ErrJSONExpectedMapForStruct, fieldPath, got) +} + +func wrapJSONConvertError(value interface{}, fieldType, fieldPath string) error { + return fmt.Errorf("%w %T to %s for field %s", ErrJSONCannotConvert, value, fieldType, fieldPath) +} + +func wrapJSONSliceElementError(item interface{}, elemType, fieldPath string, index int) error { + return fmt.Errorf("%w %T to %s for field %s[%d]", ErrJSONCannotConvertSliceElement, item, elemType, fieldPath, index) +} + +func wrapJSONArrayError(fieldPath string, got interface{}) error { + return fmt.Errorf("%w %s, got %T", ErrJSONExpectedArrayForSlice, fieldPath, got) +} + +func wrapTomlMapError(fieldPath string, got interface{}) error { + return fmt.Errorf("%w %s, got %T", ErrTomlExpectedMapForStruct, fieldPath, got) +} + +func wrapTomlConvertError(value interface{}, fieldType, fieldPath string) error { + return fmt.Errorf("%w %T to %s for field %s", ErrTomlCannotConvert, value, fieldType, fieldPath) +} + +func wrapTomlSliceElementError(item interface{}, elemType, fieldPath string, index int) error { + return fmt.Errorf("%w %T to %s for field %s[%d]", ErrTomlCannotConvertSliceElement, item, elemType, fieldPath, index) +} + +func wrapTomlArrayError(fieldPath string, got interface{}) error { + return fmt.Errorf("%w %s, got %T", ErrTomlExpectedArrayForSlice, fieldPath, got) +} + +// YAML error wrapper functions +func wrapYamlFieldCannotBeSetError() error { + return fmt.Errorf("%w", ErrYamlFieldCannotBeSet) +} + +func wrapYamlUnsupportedFieldTypeError(fieldType string) error { + return fmt.Errorf("%w: %s", ErrYamlUnsupportedFieldType, fieldType) +} + +func wrapYamlTypeConversionError(fromType, toType string) error { + return fmt.Errorf("%w: cannot convert %s to %s", ErrYamlTypeConversion, fromType, toType) +} + +func wrapYamlBoolConversionError(value string) error { + return fmt.Errorf("%w: '%s'", ErrYamlBoolConversion, value) +} diff --git a/feeders/field_tracking.go b/feeders/field_tracking.go new file mode 100644 index 00000000..70db1554 --- /dev/null +++ b/feeders/field_tracking.go @@ -0,0 +1,43 @@ +package feeders + +// FieldPopulation represents a single field population event +type FieldPopulation struct { + FieldPath string // Full path to the field (e.g., "Connections.primary.DSN") + FieldName string // Name of the field + FieldType string // Type of the field + FeederType string // Type of feeder that populated it + SourceType string // Type of source (env, yaml, etc.) + SourceKey string // Source key that was used (e.g., "DB_PRIMARY_DSN") + Value interface{} // Value that was set + InstanceKey string // Instance key for instance-aware fields + SearchKeys []string // All keys that were searched for this field + FoundKey string // The key that was actually found +} + +// FieldTracker interface allows feeders to report which fields they populate +type FieldTracker interface { + // RecordFieldPopulation records that a field was populated by a feeder + RecordFieldPopulation(fp FieldPopulation) +} + +// DefaultFieldTracker is a basic implementation of FieldTracker +type DefaultFieldTracker struct { + populations []FieldPopulation +} + +// NewDefaultFieldTracker creates a new DefaultFieldTracker +func NewDefaultFieldTracker() *DefaultFieldTracker { + return &DefaultFieldTracker{ + populations: make([]FieldPopulation, 0), + } +} + +// RecordFieldPopulation records that a field was populated by a feeder +func (t *DefaultFieldTracker) RecordFieldPopulation(fp FieldPopulation) { + t.populations = append(t.populations, fp) +} + +// GetFieldPopulations returns all recorded field populations +func (t *DefaultFieldTracker) GetFieldPopulations() []FieldPopulation { + return t.populations +} diff --git a/feeders/instance_aware_env.go b/feeders/instance_aware_env.go index f0365289..9729e56d 100644 --- a/feeders/instance_aware_env.go +++ b/feeders/instance_aware_env.go @@ -2,7 +2,6 @@ package feeders import ( "fmt" - "os" "reflect" "strings" ) @@ -16,13 +15,14 @@ var ( type InstancePrefixFunc func(instanceKey string) string // InstanceAwareEnvFeeder is a feeder that can handle environment variables for multiple instances -// of the same configuration type using instance-specific prefixes +// of the same configuration type using instance-specific prefixes with field tracking support type InstanceAwareEnvFeeder struct { prefixFunc InstancePrefixFunc verboseDebug bool logger interface { Debug(msg string, args ...any) } + fieldTracker FieldTracker } // Ensure InstanceAwareEnvFeeder implements all required interfaces @@ -51,6 +51,14 @@ func (f *InstanceAwareEnvFeeder) SetVerboseDebug(enabled bool, logger interface{ } } +// SetFieldTracker sets the field tracker for this feeder +func (f *InstanceAwareEnvFeeder) SetFieldTracker(tracker FieldTracker) { + f.fieldTracker = tracker + if f.verboseDebug && f.logger != nil { + f.logger.Debug("InstanceAwareEnvFeeder: Field tracker set", "hasTracker", tracker != nil) + } +} + // Feed implements the basic Feeder interface for single instances (backward compatibility) func (f *InstanceAwareEnvFeeder) Feed(structure interface{}) error { if f.verboseDebug && f.logger != nil { @@ -84,7 +92,7 @@ func (f *InstanceAwareEnvFeeder) Feed(structure interface{}) error { } // For single instance, use no prefix - err := f.feedStructWithPrefix(reflect.ValueOf(structure).Elem(), "") + err := f.feedStructWithPrefix(reflect.ValueOf(structure).Elem(), "", "") if f.verboseDebug && f.logger != nil { if err != nil { @@ -136,7 +144,7 @@ func (f *InstanceAwareEnvFeeder) FeedKey(instanceKey string, structure interface f.logger.Debug("InstanceAwareEnvFeeder: No prefix function configured, using empty prefix", "instanceKey", instanceKey) } - err := f.feedStructWithPrefix(reflect.ValueOf(structure).Elem(), prefix) + err := f.feedStructWithPrefix(reflect.ValueOf(structure).Elem(), prefix, instanceKey) if f.verboseDebug && f.logger != nil { if err != nil { @@ -204,28 +212,34 @@ func (f *InstanceAwareEnvFeeder) FeedInstances(instances interface{}) error { } // feedStructWithPrefix feeds a struct with environment variables using the specified prefix -func (f *InstanceAwareEnvFeeder) feedStructWithPrefix(rv reflect.Value, prefix string) error { +func (f *InstanceAwareEnvFeeder) feedStructWithPrefix(rv reflect.Value, prefix, instanceKey string) error { if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: Starting feedStructWithPrefix", "structType", rv.Type(), "prefix", prefix) + f.logger.Debug("InstanceAwareEnvFeeder: Starting feedStructWithPrefix", "structType", rv.Type(), "prefix", prefix, "instanceKey", instanceKey) } - return f.processStructFieldsWithPrefix(rv, prefix) + return f.processStructFieldsWithPrefix(rv, prefix, "", instanceKey) } // processStructFieldsWithPrefix iterates through struct fields with prefix -func (f *InstanceAwareEnvFeeder) processStructFieldsWithPrefix(rv reflect.Value, prefix string) error { +func (f *InstanceAwareEnvFeeder) processStructFieldsWithPrefix(rv reflect.Value, prefix, parentPath, instanceKey string) error { if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: Processing struct fields", "structType", rv.Type(), "numFields", rv.NumField(), "prefix", prefix) + f.logger.Debug("InstanceAwareEnvFeeder: Processing struct fields", "structType", rv.Type(), "numFields", rv.NumField(), "prefix", prefix, "parentPath", parentPath, "instanceKey", instanceKey) } for i := 0; i < rv.NumField(); i++ { field := rv.Field(i) fieldType := rv.Type().Field(i) + // Build field path + fieldPath := fieldType.Name + if parentPath != "" { + fieldPath = parentPath + "." + fieldType.Name + } + if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: Processing field", "fieldName", fieldType.Name, "fieldType", fieldType.Type, "fieldKind", field.Kind(), "prefix", prefix) + f.logger.Debug("InstanceAwareEnvFeeder: Processing field", "fieldName", fieldType.Name, "fieldType", fieldType.Type, "fieldKind", field.Kind(), "prefix", prefix, "fieldPath", fieldPath, "instanceKey", instanceKey) } - if err := f.processFieldWithPrefix(field, &fieldType, prefix); err != nil { + if err := f.processFieldWithPrefix(field, &fieldType, prefix, fieldPath, instanceKey); err != nil { if f.verboseDebug && f.logger != nil { f.logger.Debug("InstanceAwareEnvFeeder: Field processing failed", "fieldName", fieldType.Name, "prefix", prefix, "error", err) } @@ -240,20 +254,20 @@ func (f *InstanceAwareEnvFeeder) processStructFieldsWithPrefix(rv reflect.Value, } // processFieldWithPrefix handles a single struct field with prefix -func (f *InstanceAwareEnvFeeder) processFieldWithPrefix(field reflect.Value, fieldType *reflect.StructField, prefix string) error { +func (f *InstanceAwareEnvFeeder) processFieldWithPrefix(field reflect.Value, fieldType *reflect.StructField, prefix, fieldPath, instanceKey string) error { // Handle nested structs switch field.Kind() { case reflect.Struct: if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: Processing nested struct", "fieldName", fieldType.Name, "structType", field.Type(), "prefix", prefix) + f.logger.Debug("InstanceAwareEnvFeeder: Processing nested struct", "fieldName", fieldType.Name, "structType", field.Type(), "prefix", prefix, "fieldPath", fieldPath, "instanceKey", instanceKey) } - return f.processStructFieldsWithPrefix(field, prefix) + return f.processStructFieldsWithPrefix(field, prefix, fieldPath, instanceKey) case reflect.Pointer: if !field.IsZero() && field.Elem().Kind() == reflect.Struct { if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: Processing nested struct pointer", "fieldName", fieldType.Name, "structType", field.Elem().Type(), "prefix", prefix) + f.logger.Debug("InstanceAwareEnvFeeder: Processing nested struct pointer", "fieldName", fieldType.Name, "structType", field.Elem().Type(), "prefix", prefix, "fieldPath", fieldPath, "instanceKey", instanceKey) } - return f.processStructFieldsWithPrefix(field.Elem(), prefix) + return f.processStructFieldsWithPrefix(field.Elem(), prefix, fieldPath, instanceKey) } case reflect.Invalid, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, @@ -263,50 +277,89 @@ func (f *InstanceAwareEnvFeeder) processFieldWithPrefix(field reflect.Value, fie // Check for env tag for primitive types and other non-struct types if envTag, exists := fieldType.Tag.Lookup("env"); exists { if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: Found env tag", "fieldName", fieldType.Name, "envTag", envTag, "prefix", prefix) + f.logger.Debug("InstanceAwareEnvFeeder: Found env tag", "fieldName", fieldType.Name, "envTag", envTag, "prefix", prefix, "fieldPath", fieldPath, "instanceKey", instanceKey) } - return f.setFieldFromEnvWithPrefix(field, envTag, prefix) + return f.setFieldFromEnvWithPrefix(field, envTag, prefix, fieldType.Name, fieldPath, instanceKey) } else if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: No env tag found", "fieldName", fieldType.Name, "prefix", prefix) + f.logger.Debug("InstanceAwareEnvFeeder: No env tag found", "fieldName", fieldType.Name, "prefix", prefix, "fieldPath", fieldPath, "instanceKey", instanceKey) } } return nil } -// setFieldFromEnvWithPrefix sets a field value from an environment variable with prefix -func (f *InstanceAwareEnvFeeder) setFieldFromEnvWithPrefix(field reflect.Value, envTag, prefix string) error { +// setFieldFromEnvWithPrefix sets a field value from an environment variable with prefix and field tracking +func (f *InstanceAwareEnvFeeder) setFieldFromEnvWithPrefix(field reflect.Value, envTag, prefix, fieldName, fieldPath, instanceKey string) error { // Build environment variable name with prefix envName := strings.ToUpper(envTag) if prefix != "" { envName = strings.ToUpper(prefix) + envName } + // Track what we're searching for + searchKeys := []string{envName} + if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: Looking up environment variable", "envName", envName, "envTag", envTag, "prefix", prefix) + f.logger.Debug("InstanceAwareEnvFeeder: Looking up environment variable", "envName", envName, "envTag", envTag, "prefix", prefix, "fieldName", fieldName, "fieldPath", fieldPath, "instanceKey", instanceKey) } // Get and apply environment variable if exists - envValue := os.Getenv(envName) - if envValue != "" { + catalog := GetGlobalEnvCatalog() + envValue, exists := catalog.Get(envName) + if exists && envValue != "" { if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: Environment variable found", "envName", envName, "envValue", envValue) + source := catalog.GetSource(envName) + f.logger.Debug("InstanceAwareEnvFeeder: Environment variable found", "envName", envName, "envValue", envValue, "fieldPath", fieldPath, "instanceKey", instanceKey, "source", source) } err := setFieldValue(field, envValue) if err != nil { if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: Failed to set field value", "envName", envName, "envValue", envValue, "error", err) + f.logger.Debug("InstanceAwareEnvFeeder: Failed to set field value", "envName", envName, "envValue", envValue, "error", err, "fieldPath", fieldPath, "instanceKey", instanceKey) } return err } + // Record field population if tracker is available + if f.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldName, + FieldType: field.Type().String(), + FeederType: "*feeders.InstanceAwareEnvFeeder", + SourceType: "env", + SourceKey: envName, + Value: field.Interface(), + InstanceKey: instanceKey, + SearchKeys: searchKeys, + FoundKey: envName, + } + f.fieldTracker.RecordFieldPopulation(fp) + } + if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: Successfully set field value", "envName", envName, "envValue", envValue) + f.logger.Debug("InstanceAwareEnvFeeder: Successfully set field value", "envName", envName, "envValue", envValue, "fieldPath", fieldPath, "instanceKey", instanceKey) } } else { + // Record that we searched but didn't find + if f.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldName, + FieldType: field.Type().String(), + FeederType: "*feeders.InstanceAwareEnvFeeder", + SourceType: "env", + SourceKey: "", + Value: nil, + InstanceKey: instanceKey, + SearchKeys: searchKeys, + FoundKey: "", + } + f.fieldTracker.RecordFieldPopulation(fp) + } + if f.verboseDebug && f.logger != nil { - f.logger.Debug("InstanceAwareEnvFeeder: Environment variable not found or empty", "envName", envName) + f.logger.Debug("InstanceAwareEnvFeeder: Environment variable not found or empty", "envName", envName, "fieldPath", fieldPath, "instanceKey", instanceKey) } } diff --git a/feeders/json.go b/feeders/json.go index eba5ac17..29150828 100644 --- a/feeders/json.go +++ b/feeders/json.go @@ -3,9 +3,9 @@ package feeders import ( "encoding/json" "fmt" + "os" "reflect" - - "github.com/golobby/config/v3/pkg/feeder" + "strings" ) // Feeder interface for common operations @@ -51,19 +51,21 @@ func feedKey( // JSONFeeder is a feeder that reads JSON files with optional verbose debug logging type JSONFeeder struct { - feeder.Json + Path string verboseDebug bool logger interface { Debug(msg string, args ...any) } + fieldTracker FieldTracker } // NewJSONFeeder creates a new JSONFeeder that reads from the specified JSON file -func NewJSONFeeder(filePath string) JSONFeeder { - return JSONFeeder{ - Json: feeder.Json{Path: filePath}, +func NewJSONFeeder(filePath string) *JSONFeeder { + return &JSONFeeder{ + Path: filePath, verboseDebug: false, logger: nil, + fieldTracker: nil, } } @@ -77,12 +79,14 @@ func (j *JSONFeeder) SetVerboseDebug(enabled bool, logger interface{ Debug(msg s } // Feed reads the JSON file and populates the provided structure -func (j JSONFeeder) Feed(structure interface{}) error { +func (j *JSONFeeder) Feed(structure interface{}) error { if j.verboseDebug && j.logger != nil { j.logger.Debug("JSONFeeder: Starting feed process", "filePath", j.Path, "structureType", reflect.TypeOf(structure)) } - err := j.Json.Feed(structure) + // Always use custom parsing logic for consistency + err := j.feedWithTracking(structure) + if j.verboseDebug && j.logger != nil { if err != nil { j.logger.Debug("JSONFeeder: Feed completed with error", "filePath", j.Path, "error", err) @@ -97,7 +101,7 @@ func (j JSONFeeder) Feed(structure interface{}) error { } // FeedKey reads a JSON file and extracts a specific key -func (j JSONFeeder) FeedKey(key string, target interface{}) error { +func (j *JSONFeeder) FeedKey(key string, target interface{}) error { if j.verboseDebug && j.logger != nil { j.logger.Debug("JSONFeeder: Starting FeedKey process", "filePath", j.Path, "key", key, "targetType", reflect.TypeOf(target)) } @@ -113,3 +117,188 @@ func (j JSONFeeder) FeedKey(key string, target interface{}) error { } return err } + +// SetFieldTracker sets the field tracker for recording field populations +func (j *JSONFeeder) SetFieldTracker(tracker FieldTracker) { + j.fieldTracker = tracker +} + +// feedWithTracking reads the JSON file and populates the provided structure with field tracking +func (j *JSONFeeder) feedWithTracking(structure interface{}) error { + // Read and parse the JSON file manually for consistent behavior + data, err := os.ReadFile(j.Path) + if err != nil { + return fmt.Errorf("failed to read JSON file %s: %w", j.Path, err) + } + + var jsonData map[string]interface{} + if err := json.Unmarshal(data, &jsonData); err != nil { + return fmt.Errorf("failed to parse JSON file %s: %w", j.Path, err) + } + + // Check if we're dealing with a struct pointer + structValue := reflect.ValueOf(structure) + if structValue.Kind() != reflect.Ptr || structValue.Elem().Kind() != reflect.Struct { + // Not a struct pointer, fall back to standard JSON unmarshaling + if j.verboseDebug && j.logger != nil { + j.logger.Debug("JSONFeeder: Not a struct pointer, using standard JSON unmarshaling", "structureType", reflect.TypeOf(structure)) + } + if err := json.Unmarshal(data, structure); err != nil { + return fmt.Errorf("failed to unmarshal JSON data: %w", err) + } + return nil + } + + // Process the structure with field tracking + return j.processStructFields(reflect.ValueOf(structure).Elem(), jsonData, "") +} + +// processStructFields iterates through struct fields and populates them from JSON data +func (j *JSONFeeder) processStructFields(rv reflect.Value, jsonData map[string]interface{}, fieldPrefix string) error { + structType := rv.Type() + + for i := 0; i < rv.NumField(); i++ { + field := rv.Field(i) + fieldType := structType.Field(i) + + // Skip unexported fields + if !field.CanSet() { + continue + } + + // Get JSON tag or use field name + jsonTag := fieldType.Tag.Get("json") + var jsonKey string + + if jsonTag == "" { + // No JSON tag, use field name + jsonKey = fieldType.Name + } else if jsonTag == "-" { + // Explicitly skipped + continue + } else { + // Handle json tag with options (e.g., "field,omitempty") + jsonKey = strings.Split(jsonTag, ",")[0] + if jsonKey == "" { + jsonKey = fieldType.Name + } + } + + fieldPath := fieldType.Name // Use struct field name for path + if fieldPrefix != "" { + fieldPath = fieldPrefix + "." + fieldType.Name + } + + // Check if this key exists in the JSON data + if value, exists := jsonData[jsonKey]; exists { + if err := j.processField(field, fieldType, value, fieldPath); err != nil { + return err + } + } + } + + return nil +} + +// processField processes a single field, handling nested structs, slices, and basic types +func (j *JSONFeeder) processField(field reflect.Value, fieldType reflect.StructField, value interface{}, fieldPath string) error { + fieldKind := field.Kind() + + switch fieldKind { + case reflect.Struct: + // Handle nested structs + if nestedMap, ok := value.(map[string]interface{}); ok { + return j.processStructFields(field, nestedMap, fieldPath) + } + return wrapJSONMapError(fieldPath, value) + + case reflect.Slice: + // Handle slices + return j.setSliceFromJSON(field, value, fieldPath) + + case reflect.Invalid, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128, reflect.Array, + reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.String, + reflect.UnsafePointer: + // Handle basic types and unsupported types + return j.setFieldFromJSON(field, value, fieldPath) + + default: + // Handle any remaining types + return j.setFieldFromJSON(field, value, fieldPath) + } +} + +// setFieldFromJSON sets a field value from JSON data with type conversion +func (j *JSONFeeder) setFieldFromJSON(field reflect.Value, value interface{}, fieldPath string) error { + // Convert and set the value + convertedValue := reflect.ValueOf(value) + if convertedValue.Type().ConvertibleTo(field.Type()) { + field.Set(convertedValue.Convert(field.Type())) + + // Record field population + if j.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldPath, + FieldType: field.Type().String(), + FeederType: "JSONFeeder", + SourceType: "json_file", + SourceKey: fieldPath, + Value: value, + SearchKeys: []string{fieldPath}, + FoundKey: fieldPath, + } + j.fieldTracker.RecordFieldPopulation(fp) + } + + return nil + } + + return wrapJSONConvertError(value, field.Type().String(), fieldPath) +} + +// setSliceFromJSON sets a slice field from JSON array data +func (j *JSONFeeder) setSliceFromJSON(field reflect.Value, value interface{}, fieldPath string) error { + // Handle slice values + if arrayValue, ok := value.([]interface{}); ok { + sliceType := field.Type() + elemType := sliceType.Elem() + + newSlice := reflect.MakeSlice(sliceType, len(arrayValue), len(arrayValue)) + + for i, item := range arrayValue { + elem := newSlice.Index(i) + convertedItem := reflect.ValueOf(item) + + if convertedItem.Type().ConvertibleTo(elemType) { + elem.Set(convertedItem.Convert(elemType)) + } else { + return wrapJSONSliceElementError(item, elemType.String(), fieldPath, i) + } + } + + field.Set(newSlice) + + // Record field population for the slice + if j.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldPath, + FieldType: field.Type().String(), + FeederType: "JSONFeeder", + SourceType: "json_file", + SourceKey: fieldPath, + Value: value, + SearchKeys: []string{fieldPath}, + FoundKey: fieldPath, + } + j.fieldTracker.RecordFieldPopulation(fp) + } + + return nil + } + + return wrapJSONArrayError(fieldPath, value) +} diff --git a/feeders/json_field_tracking_test.go b/feeders/json_field_tracking_test.go new file mode 100644 index 00000000..ad79bddd --- /dev/null +++ b/feeders/json_field_tracking_test.go @@ -0,0 +1,185 @@ +package feeders + +import ( + "fmt" + "os" + "testing" +) + +// TestConfig struct for JSON field tracking tests +type TestJSONConfig struct { + Name string `json:"name"` + Port int `json:"port"` + Enabled bool `json:"enabled"` + Tags []string `json:"tags"` + DB TestDBConfig `json:"db"` +} + +type TestDBConfig struct { + Host string `json:"host"` + Port int `json:"port"` + Database string `json:"database"` +} + +func TestJSONFeeder_FieldTracking(t *testing.T) { + // Create test JSON file + jsonContent := `{ + "name": "test-app", + "port": 8080, + "enabled": true, + "tags": ["web", "api"], + "db": { + "host": "localhost", + "port": 5432, + "database": "testdb" + } +}` + + // Create temporary JSON file + tmpFile, err := os.CreateTemp("", "test_config_*.json") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + + if _, err := tmpFile.WriteString(jsonContent); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tmpFile.Close() + + // Create feeder and field tracker + feeder := NewJSONFeeder(tmpFile.Name()) + tracker := NewDefaultFieldTracker() + feeder.SetFieldTracker(tracker) + + // Test config structure + var config TestJSONConfig + + // Feed the configuration + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Failed to feed config: %v", err) + } + + // Verify configuration was populated correctly + if config.Name != "test-app" { + t.Errorf("Expected Name to be 'test-app', got %s", config.Name) + } + if config.Port != 8080 { + t.Errorf("Expected Port to be 8080, got %d", config.Port) + } + if !config.Enabled { + t.Errorf("Expected Enabled to be true, got %v", config.Enabled) + } + if len(config.Tags) != 2 || config.Tags[0] != "web" || config.Tags[1] != "api" { + t.Errorf("Expected Tags to be ['web', 'api'], got %v", config.Tags) + } + if config.DB.Host != "localhost" { + t.Errorf("Expected DB.Host to be 'localhost', got %s", config.DB.Host) + } + if config.DB.Port != 5432 { + t.Errorf("Expected DB.Port to be 5432, got %d", config.DB.Port) + } + if config.DB.Database != "testdb" { + t.Errorf("Expected DB.Database to be 'testdb', got %s", config.DB.Database) + } + + // Get field populations + populations := tracker.GetFieldPopulations() + + // Verify we have tracking information for all fields + expectedFields := []string{"Name", "Port", "Enabled", "Tags", "DB.Host", "DB.Port", "DB.Database"} + + for _, fieldPath := range expectedFields { + found := false + for _, pop := range populations { + if pop.FieldPath == fieldPath { + found = true + // Verify basic tracking information + if pop.FeederType != "JSONFeeder" { + t.Errorf("Expected FeederType 'JSONFeeder' for field %s, got %s", fieldPath, pop.FeederType) + } + if pop.SourceType != "json_file" { + t.Errorf("Expected SourceType 'json_file' for field %s, got %s", fieldPath, pop.SourceType) + } + if pop.SourceKey == "" { + t.Errorf("Expected non-empty SourceKey for field %s", fieldPath) + } + if pop.Value == nil { + t.Errorf("Expected non-nil Value for field %s", fieldPath) + } + break + } + } + if !found { + t.Errorf("Field tracking not found for field: %s", fieldPath) + } + } + + // Verify specific field values in tracking + for _, pop := range populations { + switch pop.FieldPath { + case "Name": + if fmt.Sprintf("%v", pop.Value) != "test-app" { + t.Errorf("Expected tracked value 'test-app' for Name, got %v", pop.Value) + } + case "Port": + if fmt.Sprintf("%v", pop.Value) != "8080" { + t.Errorf("Expected tracked value '8080' for Port, got %v", pop.Value) + } + case "Enabled": + if fmt.Sprintf("%v", pop.Value) != "true" { + t.Errorf("Expected tracked value 'true' for Enabled, got %v", pop.Value) + } + case "DB.Host": + if fmt.Sprintf("%v", pop.Value) != "localhost" { + t.Errorf("Expected tracked value 'localhost' for DB.Host, got %v", pop.Value) + } + } + } +} + +func TestJSONFeeder_SetFieldTracker(t *testing.T) { + feeder := NewJSONFeeder("test.json") + tracker := NewDefaultFieldTracker() + + // Test that SetFieldTracker method exists and can be called + feeder.SetFieldTracker(tracker) + + // The actual tracking functionality is tested in TestJSONFeeder_FieldTracking +} + +func TestJSONFeeder_WithoutFieldTracker(t *testing.T) { + // Create test JSON file + jsonContent := `{"name": "test-app", "port": 8080}` + + tmpFile, err := os.CreateTemp("", "test_config_*.json") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + + if _, err := tmpFile.WriteString(jsonContent); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tmpFile.Close() + + // Create feeder without field tracker + feeder := NewJSONFeeder(tmpFile.Name()) + + var config TestJSONConfig + + // Should work without field tracker + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Failed to feed config without field tracker: %v", err) + } + + // Verify configuration was populated correctly + if config.Name != "test-app" { + t.Errorf("Expected Name to be 'test-app', got %s", config.Name) + } + if config.Port != 8080 { + t.Errorf("Expected Port to be 8080, got %d", config.Port) + } +} diff --git a/feeders/tenant_affixed_env.go b/feeders/tenant_affixed_env.go index 1803f3d6..607185d8 100644 --- a/feeders/tenant_affixed_env.go +++ b/feeders/tenant_affixed_env.go @@ -14,22 +14,25 @@ type TenantAffixedEnvFeeder struct { // NewTenantAffixedEnvFeeder creates a new TenantAffixedEnvFeeder with the given prefix and suffix functions // The prefix and suffix functions are used to modify the prefix and suffix of the environment variables // before they are used to set the struct fields -// The prefix and suffix functions should take a string and return a string // The prefix function is used to modify the prefix of the environment variables // The suffix function is used to modify the suffix of the environment variables func NewTenantAffixedEnvFeeder(prefix, suffix func(string) string) TenantAffixedEnvFeeder { - affixedFeeder := &AffixedEnvFeeder{} - return TenantAffixedEnvFeeder{ - AffixedEnvFeeder: affixedFeeder, - SetPrefixFunc: func(p string) { - affixedFeeder.Prefix = prefix(p) - }, - SetSuffixFunc: func(s string) { - affixedFeeder.Suffix = suffix(s) - }, - verboseDebug: false, - logger: nil, + affixedFeeder := NewAffixedEnvFeeder("", "") // Initialize with empty prefix and suffix + result := TenantAffixedEnvFeeder{ + AffixedEnvFeeder: &affixedFeeder, // Take address of the struct + verboseDebug: false, + logger: nil, } + + // Set the function closures to modify the affixed feeder + result.SetPrefixFunc = func(p string) { + result.Prefix = prefix(p) + } + result.SetSuffixFunc = func(s string) { + result.Suffix = suffix(s) + } + + return result } // SetVerboseDebug enables or disables verbose debug logging @@ -44,3 +47,11 @@ func (f *TenantAffixedEnvFeeder) SetVerboseDebug(enabled bool, logger interface{ f.logger.Debug("Verbose tenant affixed environment feeder debugging enabled") } } + +// SetFieldTracker sets the field tracker for recording field populations +func (f *TenantAffixedEnvFeeder) SetFieldTracker(tracker FieldTracker) { + // Delegate to the embedded AffixedEnvFeeder + if f.AffixedEnvFeeder != nil { + f.AffixedEnvFeeder.SetFieldTracker(tracker) + } +} diff --git a/feeders/tenant_affixed_env_debug_test.go b/feeders/tenant_affixed_env_debug_test.go new file mode 100644 index 00000000..7a482bd0 --- /dev/null +++ b/feeders/tenant_affixed_env_debug_test.go @@ -0,0 +1,65 @@ +package feeders + +import ( + "fmt" + "os" + "testing" +) + +func TestTenantAffixedEnvFeeder_Debug(t *testing.T) { + // Set up multiple environment variables to test + envVars := []string{ + "APP_test123__NAME__PROD", + "APP_TEST123__NAME__PROD", + "APP_test123_NAME_PROD", + "APP_TEST123_NAME_PROD", + } + + for _, envVar := range envVars { + os.Setenv(envVar, "tenant-app") + fmt.Printf("Set env var: %s\n", envVar) + } + defer func() { + for _, envVar := range envVars { + os.Unsetenv(envVar) + } + }() + + // Create tenant affixed feeder + prefixFunc := func(tenantId string) string { + return "APP_" + tenantId + "_" + } + suffixFunc := func(environment string) string { + return "_" + environment + } + + feeder := NewTenantAffixedEnvFeeder(prefixFunc, suffixFunc) + feeder.SetPrefixFunc("test123") + feeder.SetSuffixFunc("PROD") + + fmt.Printf("Final prefix: %s\n", feeder.Prefix) + fmt.Printf("Final suffix: %s\n", feeder.Suffix) + + // Expected env var name: APP_test123_ + _ + NAME + _ + _PROD = APP_test123__NAME__PROD + expectedEnvVar := "APP_test123__NAME__PROD" + fmt.Printf("Expected env var name: %s\n", expectedEnvVar) + fmt.Printf("Actual value in env: %s\n", os.Getenv(expectedEnvVar)) + + // But AffixedEnvFeeder does ToUpper on the env tag, so let's check uppercase version + expectedEnvVarUpper := "APP_test123__NAME__PROD" + fmt.Printf("Expected env var name (upper): %s\n", expectedEnvVarUpper) + fmt.Printf("Actual value in env (upper): %s\n", os.Getenv(expectedEnvVarUpper)) + + // Simple test config + var config struct { + Name string `env:"NAME"` + } + + // Feed the configuration + err := feeder.Feed(&config) + if err != nil { + t.Fatalf("Failed to feed config: %v", err) + } + + fmt.Printf("Config populated - Name: %s\n", config.Name) +} diff --git a/feeders/tenant_affixed_env_field_tracking_test.go b/feeders/tenant_affixed_env_field_tracking_test.go new file mode 100644 index 00000000..283f619a --- /dev/null +++ b/feeders/tenant_affixed_env_field_tracking_test.go @@ -0,0 +1,150 @@ +package feeders + +import ( + "fmt" + "os" + "testing" +) + +// TestConfig struct for TenantAffixedEnv field tracking tests +type TestTenantAffixedEnvConfig struct { + Name string `env:"NAME"` + Port int `env:"PORT"` + Enabled bool `env:"ENABLED"` +} + +func TestTenantAffixedEnvFeeder_FieldTracking(t *testing.T) { + // Set up environment variables for tenant "test123" + // The AffixedEnvFeeder converts prefix/suffix to uppercase and constructs env vars as: + // ToUpper(prefix) + "_" + ToUpper(envTag) + "_" + ToUpper(suffix) + // With prefix "APP_test123_" -> "APP_TEST123_" and suffix "_PROD" -> "_PROD": + // APP_TEST123_ + _ + NAME + _ + _PROD = APP_TEST123__NAME__PROD + envVars := map[string]string{ + "APP_TEST123__NAME__PROD": "tenant-app", + "APP_TEST123__PORT__PROD": "9090", + "APP_TEST123__ENABLED__PROD": "true", + "OTHER_VAR": "ignored", // Should not be matched + } + + // Set environment variables for test + for key, value := range envVars { + os.Setenv(key, value) + } + defer func() { + // Clean up after test + for key := range envVars { + os.Unsetenv(key) + } + }() + + // Create tenant affixed feeder + // Prefix function: "APP_" + tenantId + "_" = "APP_test123_" + // Suffix function: "_" + environment = "_PROD" + prefixFunc := func(tenantId string) string { + return "APP_" + tenantId + "_" + } + suffixFunc := func(environment string) string { + return "_" + environment + } + + feeder := NewTenantAffixedEnvFeeder(prefixFunc, suffixFunc) + + // Set tenant and environment + feeder.SetPrefixFunc("test123") + feeder.SetSuffixFunc("PROD") + + // Set field tracker + tracker := NewDefaultFieldTracker() + feeder.SetFieldTracker(tracker) + + // Test config structure + var config TestTenantAffixedEnvConfig + + // Feed the configuration + err := feeder.Feed(&config) + if err != nil { + t.Fatalf("Failed to feed config: %v", err) + } + + // Verify configuration was populated correctly + if config.Name != "tenant-app" { + t.Errorf("Expected Name to be 'tenant-app', got %s", config.Name) + } + if config.Port != 9090 { + t.Errorf("Expected Port to be 9090, got %d", config.Port) + } + if !config.Enabled { + t.Errorf("Expected Enabled to be true, got %v", config.Enabled) + } + + // Get field populations + populations := tracker.GetFieldPopulations() + + // Verify we have tracking information for all fields + expectedFields := []string{"Name", "Port", "Enabled"} + + for _, fieldPath := range expectedFields { + found := false + for _, pop := range populations { + if pop.FieldPath == fieldPath { + found = true + // Verify basic tracking information + if pop.FeederType != "AffixedEnvFeeder" { // Since it delegates to AffixedEnvFeeder + t.Errorf("Expected FeederType 'AffixedEnvFeeder' for field %s, got %s", fieldPath, pop.FeederType) + } + if pop.SourceType != "env_affixed" { + t.Errorf("Expected SourceType 'env_affixed' for field %s, got %s", fieldPath, pop.SourceType) + } + if pop.SourceKey == "" { + t.Errorf("Expected non-empty SourceKey for field %s", fieldPath) + } + if pop.Value == nil { + t.Errorf("Expected non-nil Value for field %s", fieldPath) + } + break + } + } + if !found { + t.Errorf("Field tracking not found for field: %s", fieldPath) + } + } + + // Verify specific field values and source keys in tracking + for _, pop := range populations { + switch pop.FieldPath { + case "Name": + if fmt.Sprintf("%v", pop.Value) != "tenant-app" { + t.Errorf("Expected tracked value 'tenant-app' for Name, got %v", pop.Value) + } + if pop.SourceKey != "APP_TEST123__NAME__PROD" { + t.Errorf("Expected SourceKey 'APP_TEST123__NAME__PROD' for Name, got %s", pop.SourceKey) + } + case "Port": + if fmt.Sprintf("%v", pop.Value) != "9090" { + t.Errorf("Expected tracked value '9090' for Port, got %v", pop.Value) + } + if pop.SourceKey != "APP_TEST123__PORT__PROD" { + t.Errorf("Expected SourceKey 'APP_TEST123__PORT__PROD' for Port, got %s", pop.SourceKey) + } + case "Enabled": + if fmt.Sprintf("%v", pop.Value) != "true" { + t.Errorf("Expected tracked value 'true' for Enabled, got %v", pop.Value) + } + if pop.SourceKey != "APP_TEST123__ENABLED__PROD" { + t.Errorf("Expected SourceKey 'APP_TEST123__ENABLED__PROD' for Enabled, got %s", pop.SourceKey) + } + } + } +} + +func TestTenantAffixedEnvFeeder_SetFieldTracker(t *testing.T) { + prefixFunc := func(tenantId string) string { return "PREFIX_" } + suffixFunc := func(environment string) string { return "_SUFFIX" } + feeder := NewTenantAffixedEnvFeeder(prefixFunc, suffixFunc) + tracker := NewDefaultFieldTracker() + + // Test that SetFieldTracker method exists and can be called + feeder.SetFieldTracker(tracker) + + // The actual tracking functionality is tested in TestTenantAffixedEnvFeeder_FieldTracking +} diff --git a/feeders/toml.go b/feeders/toml.go index 4762df4c..673ac114 100644 --- a/feeders/toml.go +++ b/feeders/toml.go @@ -2,27 +2,30 @@ package feeders import ( "fmt" + "os" "reflect" + "strings" "github.com/BurntSushi/toml" - "github.com/golobby/config/v3/pkg/feeder" ) // TomlFeeder is a feeder that reads TOML files with optional verbose debug logging type TomlFeeder struct { - feeder.Toml + Path string verboseDebug bool logger interface { Debug(msg string, args ...any) } + fieldTracker FieldTracker } // NewTomlFeeder creates a new TomlFeeder that reads from the specified TOML file -func NewTomlFeeder(filePath string) TomlFeeder { - return TomlFeeder{ - Toml: feeder.Toml{Path: filePath}, +func NewTomlFeeder(filePath string) *TomlFeeder { + return &TomlFeeder{ + Path: filePath, verboseDebug: false, logger: nil, + fieldTracker: nil, } } @@ -35,13 +38,20 @@ func (t *TomlFeeder) SetVerboseDebug(enabled bool, logger interface{ Debug(msg s } } +// SetFieldTracker sets the field tracker for recording field populations +func (t *TomlFeeder) SetFieldTracker(tracker FieldTracker) { + t.fieldTracker = tracker +} + // Feed reads the TOML file and populates the provided structure -func (t TomlFeeder) Feed(structure interface{}) error { +func (t *TomlFeeder) Feed(structure interface{}) error { if t.verboseDebug && t.logger != nil { t.logger.Debug("TomlFeeder: Starting feed process", "filePath", t.Path, "structureType", reflect.TypeOf(structure)) } - err := t.Toml.Feed(structure) + // Always use custom parsing logic for consistency + err := t.feedWithTracking(structure) + if t.verboseDebug && t.logger != nil { if err != nil { t.logger.Debug("TomlFeeder: Feed completed with error", "filePath", t.Path, "error", err) @@ -56,7 +66,7 @@ func (t TomlFeeder) Feed(structure interface{}) error { } // FeedKey reads a TOML file and extracts a specific key -func (t TomlFeeder) FeedKey(key string, target interface{}) error { +func (t *TomlFeeder) FeedKey(key string, target interface{}) error { if t.verboseDebug && t.logger != nil { t.logger.Debug("TomlFeeder: Starting FeedKey process", "filePath", t.Path, "key", key, "targetType", reflect.TypeOf(target)) } @@ -72,3 +82,183 @@ func (t TomlFeeder) FeedKey(key string, target interface{}) error { } return err } + +// feedWithTracking reads the TOML file and populates the provided structure with field tracking +func (t *TomlFeeder) feedWithTracking(structure interface{}) error { + // Read and parse the TOML file manually for consistent behavior + data, err := os.ReadFile(t.Path) + if err != nil { + return fmt.Errorf("failed to read TOML file %s: %w", t.Path, err) + } + + var tomlData map[string]interface{} + if err := toml.Unmarshal(data, &tomlData); err != nil { + return fmt.Errorf("failed to parse TOML file %s: %w", t.Path, err) + } + + // Check if we're dealing with a struct pointer + structValue := reflect.ValueOf(structure) + if structValue.Kind() != reflect.Ptr || structValue.Elem().Kind() != reflect.Struct { + // Not a struct pointer, fall back to standard TOML unmarshaling + if t.verboseDebug && t.logger != nil { + t.logger.Debug("TomlFeeder: Not a struct pointer, using standard TOML unmarshaling", "structureType", reflect.TypeOf(structure)) + } + if err := toml.Unmarshal(data, structure); err != nil { + return fmt.Errorf("failed to unmarshal TOML data: %w", err) + } + return nil + } + + // Process the structure with field tracking + return t.processStructFields(reflect.ValueOf(structure).Elem(), tomlData, "") +} + +// processStructFields iterates through struct fields and populates them from TOML data +func (t *TomlFeeder) processStructFields(rv reflect.Value, tomlData map[string]interface{}, fieldPrefix string) error { + structType := rv.Type() + + for i := 0; i < rv.NumField(); i++ { + field := rv.Field(i) + fieldType := structType.Field(i) + + // Skip unexported fields + if !field.CanSet() { + continue + } + + // Get TOML tag or use field name + tomlTag := fieldType.Tag.Get("toml") + var tomlKey string + + if tomlTag == "" { + // No TOML tag, use field name + tomlKey = fieldType.Name + } else if tomlTag == "-" { + // Explicitly skipped + continue + } else { + // Handle toml tag with options (e.g., "field,omitempty") + tomlKey = strings.Split(tomlTag, ",")[0] + if tomlKey == "" { + tomlKey = fieldType.Name + } + } + + fieldPath := fieldType.Name // Use struct field name for path + if fieldPrefix != "" { + fieldPath = fieldPrefix + "." + fieldType.Name + } + + // Check if this key exists in the TOML data + if value, exists := tomlData[tomlKey]; exists { + if err := t.processField(field, fieldType, value, fieldPath); err != nil { + return err + } + } + } + + return nil +} + +// processField processes a single field, handling nested structs, slices, and basic types +func (t *TomlFeeder) processField(field reflect.Value, fieldType reflect.StructField, value interface{}, fieldPath string) error { + fieldKind := field.Kind() + + switch fieldKind { + case reflect.Struct: + // Handle nested structs + if nestedMap, ok := value.(map[string]interface{}); ok { + return t.processStructFields(field, nestedMap, fieldPath) + } + return wrapTomlMapError(fieldPath, value) + + case reflect.Slice: + // Handle slices + return t.setSliceFromTOML(field, value, fieldPath) + + case reflect.Invalid, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128, reflect.Array, + reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.String, + reflect.UnsafePointer: + // Handle basic types and unsupported types + return t.setFieldFromTOML(field, value, fieldPath) + + default: + // Handle any remaining types + return t.setFieldFromTOML(field, value, fieldPath) + } +} + +// setFieldFromTOML sets a field value from TOML data with type conversion +func (t *TomlFeeder) setFieldFromTOML(field reflect.Value, value interface{}, fieldPath string) error { + // Convert and set the value + convertedValue := reflect.ValueOf(value) + if convertedValue.Type().ConvertibleTo(field.Type()) { + field.Set(convertedValue.Convert(field.Type())) + + // Record field population + if t.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldPath, + FieldType: field.Type().String(), + FeederType: "TomlFeeder", + SourceType: "toml_file", + SourceKey: fieldPath, + Value: value, + SearchKeys: []string{fieldPath}, + FoundKey: fieldPath, + } + t.fieldTracker.RecordFieldPopulation(fp) + } + + return nil + } + + return wrapTomlConvertError(value, field.Type().String(), fieldPath) +} + +// setSliceFromTOML sets a slice field from TOML array data +func (t *TomlFeeder) setSliceFromTOML(field reflect.Value, value interface{}, fieldPath string) error { + // Handle slice values + if arrayValue, ok := value.([]interface{}); ok { + sliceType := field.Type() + elemType := sliceType.Elem() + + newSlice := reflect.MakeSlice(sliceType, len(arrayValue), len(arrayValue)) + + for i, item := range arrayValue { + elem := newSlice.Index(i) + convertedItem := reflect.ValueOf(item) + + if convertedItem.Type().ConvertibleTo(elemType) { + elem.Set(convertedItem.Convert(elemType)) + } else { + return wrapTomlSliceElementError(item, elemType.String(), fieldPath, i) + } + } + + field.Set(newSlice) + + // Record field population for the slice + if t.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldPath, + FieldType: field.Type().String(), + FeederType: "TomlFeeder", + SourceType: "toml_file", + SourceKey: fieldPath, + Value: value, + SearchKeys: []string{fieldPath}, + FoundKey: fieldPath, + } + t.fieldTracker.RecordFieldPopulation(fp) + } + + return nil + } + + return wrapTomlArrayError(fieldPath, value) +} diff --git a/feeders/toml_field_tracking_test.go b/feeders/toml_field_tracking_test.go new file mode 100644 index 00000000..0a242f49 --- /dev/null +++ b/feeders/toml_field_tracking_test.go @@ -0,0 +1,185 @@ +package feeders + +import ( + "fmt" + "os" + "testing" +) + +// TestConfig struct for TOML field tracking tests +type TestTOMLConfig struct { + Name string `toml:"name"` + Port int `toml:"port"` + Enabled bool `toml:"enabled"` + Tags []string `toml:"tags"` + DB TestTOMLDBConfig `toml:"db"` +} + +type TestTOMLDBConfig struct { + Host string `toml:"host"` + Port int `toml:"port"` + Database string `toml:"database"` +} + +func TestTomlFeeder_FieldTracking(t *testing.T) { + // Create test TOML file + tomlContent := `name = "test-app" +port = 8080 +enabled = true +tags = ["web", "api"] + +[db] +host = "localhost" +port = 5432 +database = "testdb" +` + + // Create temporary TOML file + tmpFile, err := os.CreateTemp("", "test_config_*.toml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + + if _, err := tmpFile.WriteString(tomlContent); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tmpFile.Close() + + // Create feeder and field tracker + feeder := NewTomlFeeder(tmpFile.Name()) + tracker := NewDefaultFieldTracker() + feeder.SetFieldTracker(tracker) + + // Test config structure + var config TestTOMLConfig + + // Feed the configuration + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Failed to feed config: %v", err) + } + + // Verify configuration was populated correctly + if config.Name != "test-app" { + t.Errorf("Expected Name to be 'test-app', got %s", config.Name) + } + if config.Port != 8080 { + t.Errorf("Expected Port to be 8080, got %d", config.Port) + } + if !config.Enabled { + t.Errorf("Expected Enabled to be true, got %v", config.Enabled) + } + if len(config.Tags) != 2 || config.Tags[0] != "web" || config.Tags[1] != "api" { + t.Errorf("Expected Tags to be ['web', 'api'], got %v", config.Tags) + } + if config.DB.Host != "localhost" { + t.Errorf("Expected DB.Host to be 'localhost', got %s", config.DB.Host) + } + if config.DB.Port != 5432 { + t.Errorf("Expected DB.Port to be 5432, got %d", config.DB.Port) + } + if config.DB.Database != "testdb" { + t.Errorf("Expected DB.Database to be 'testdb', got %s", config.DB.Database) + } + + // Get field populations + populations := tracker.GetFieldPopulations() + + // Verify we have tracking information for all fields + expectedFields := []string{"Name", "Port", "Enabled", "Tags", "DB.Host", "DB.Port", "DB.Database"} + + for _, fieldPath := range expectedFields { + found := false + for _, pop := range populations { + if pop.FieldPath == fieldPath { + found = true + // Verify basic tracking information + if pop.FeederType != "TomlFeeder" { + t.Errorf("Expected FeederType 'TomlFeeder' for field %s, got %s", fieldPath, pop.FeederType) + } + if pop.SourceType != "toml_file" { + t.Errorf("Expected SourceType 'toml_file' for field %s, got %s", fieldPath, pop.SourceType) + } + if pop.SourceKey == "" { + t.Errorf("Expected non-empty SourceKey for field %s", fieldPath) + } + if pop.Value == nil { + t.Errorf("Expected non-nil Value for field %s", fieldPath) + } + break + } + } + if !found { + t.Errorf("Field tracking not found for field: %s", fieldPath) + } + } + + // Verify specific field values in tracking + for _, pop := range populations { + switch pop.FieldPath { + case "Name": + if fmt.Sprintf("%v", pop.Value) != "test-app" { + t.Errorf("Expected tracked value 'test-app' for Name, got %v", pop.Value) + } + case "Port": + if fmt.Sprintf("%v", pop.Value) != "8080" { + t.Errorf("Expected tracked value '8080' for Port, got %v", pop.Value) + } + case "Enabled": + if fmt.Sprintf("%v", pop.Value) != "true" { + t.Errorf("Expected tracked value 'true' for Enabled, got %v", pop.Value) + } + case "DB.Host": + if fmt.Sprintf("%v", pop.Value) != "localhost" { + t.Errorf("Expected tracked value 'localhost' for DB.Host, got %v", pop.Value) + } + } + } +} + +func TestTomlFeeder_SetFieldTracker(t *testing.T) { + feeder := NewTomlFeeder("test.toml") + tracker := NewDefaultFieldTracker() + + // Test that SetFieldTracker method exists and can be called + feeder.SetFieldTracker(tracker) + + // The actual tracking functionality is tested in TestTomlFeeder_FieldTracking +} + +func TestTomlFeeder_WithoutFieldTracker(t *testing.T) { + // Create test TOML file + tomlContent := `name = "test-app" +port = 8080` + + tmpFile, err := os.CreateTemp("", "test_config_*.toml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tmpFile.Name()) + + if _, err := tmpFile.WriteString(tomlContent); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tmpFile.Close() + + // Create feeder without field tracker + feeder := NewTomlFeeder(tmpFile.Name()) + + var config TestTOMLConfig + + // Should work without field tracker + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Failed to feed config without field tracker: %v", err) + } + + // Verify configuration was populated correctly + if config.Name != "test-app" { + t.Errorf("Expected Name to be 'test-app', got %s", config.Name) + } + if config.Port != 8080 { + t.Errorf("Expected Port to be 8080, got %d", config.Port) + } +} diff --git a/feeders/unified_catalog_integration_test.go b/feeders/unified_catalog_integration_test.go new file mode 100644 index 00000000..cdbd1ce5 --- /dev/null +++ b/feeders/unified_catalog_integration_test.go @@ -0,0 +1,242 @@ +package feeders + +import ( + "os" + "path/filepath" + "testing" +) + +func TestUnifiedEnvCatalogIntegration(t *testing.T) { + // Reset catalog + ResetGlobalEnvCatalog() + + // Create test .env file + envContent := []byte(` +# Database configuration +DB_HOST=dotenv-host +DB_PORT=5432 +DB_USER=dotenv-user + +# App configuration +APP_ENV=development +APP_DEBUG=true +`) + + tempFile := filepath.Join(os.TempDir(), "integration_test.env") + err := os.WriteFile(tempFile, envContent, 0600) + if err != nil { + t.Fatalf("Failed to create test .env file: %v", err) + } + defer os.Remove(tempFile) + + // Set some OS environment variables that should override .env + t.Setenv("DB_HOST", "os-env-host") // Should override .env value + t.Setenv("APP_PORT", "8080") // Only in OS env + + type Config struct { + Database struct { + Host string `env:"DB_HOST"` + Port int `env:"DB_PORT"` + User string `env:"DB_USER"` + } + App struct { + Environment string `env:"APP_ENV"` + Debug bool `env:"APP_DEBUG"` + Port int `env:"APP_PORT"` + } + } + + var config Config + + // Set up field tracking + tracker := NewDefaultFieldTracker() + + // Step 1: Load .env file into catalog + dotEnvFeeder := NewDotEnvFeeder(tempFile) + dotEnvFeeder.SetFieldTracker(tracker) + err = dotEnvFeeder.Feed(&config) + if err != nil { + t.Fatalf("DotEnvFeeder failed: %v", err) + } + + // Step 2: Use EnvFeeder to populate remaining fields + envFeeder := NewEnvFeeder() + envFeeder.SetFieldTracker(tracker) + err = envFeeder.Feed(&config) + if err != nil { + t.Fatalf("EnvFeeder failed: %v", err) + } + + // Verify results + t.Run("verify_os_env_precedence", func(t *testing.T) { + // DB_HOST should be from OS env (precedence over .env) + if config.Database.Host != "os-env-host" { + t.Errorf("Expected DB_HOST='os-env-host' (OS env), got '%s'", config.Database.Host) + } + }) + + t.Run("verify_dotenv_values", func(t *testing.T) { + // DB_PORT should be from .env (not in OS env) + if config.Database.Port != 5432 { + t.Errorf("Expected DB_PORT=5432 (from .env), got %d", config.Database.Port) + } + + // DB_USER should be from .env (not in OS env) + if config.Database.User != "dotenv-user" { + t.Errorf("Expected DB_USER='dotenv-user' (from .env), got '%s'", config.Database.User) + } + + // APP_ENV should be from .env (not in OS env) + if config.App.Environment != "development" { + t.Errorf("Expected APP_ENV='development' (from .env), got '%s'", config.App.Environment) + } + + // APP_DEBUG should be from .env (not in OS env) + if !config.App.Debug { + t.Errorf("Expected APP_DEBUG=true (from .env), got %v", config.App.Debug) + } + }) + + t.Run("verify_os_only_values", func(t *testing.T) { + // APP_PORT should be from OS env (only in OS env) + if config.App.Port != 8080 { + t.Errorf("Expected APP_PORT=8080 (OS env only), got %d", config.App.Port) + } + }) + t.Run("verify_field_tracking", func(t *testing.T) { + populations := tracker.GetFieldPopulations() + if len(populations) == 0 { + t.Fatal("No field populations recorded") + } + + sourceMap := make(map[string]string) + for _, pop := range populations { + sourceMap[pop.SourceKey] = pop.SourceType + } + + // Check that we have tracking - all should be "env" since EnvFeeder runs last + // and reads from the unified catalog (which includes both OS env and .env values) + foundEnv := false + + for key, sourceType := range sourceMap { + t.Logf("Field tracking: %s from %s", key, sourceType) + if sourceType == "env" { + foundEnv = true + } + } + + if !foundEnv { + t.Error("Expected fields to be tracked as coming from env feeder") + } + + // Verify that the precedence is working correctly in the final values + // DB_HOST should be "os-env-host" (OS env precedence over .env) + if config.Database.Host != "os-env-host" { + t.Errorf("Expected DB_HOST='os-env-host' (OS env precedence), got '%s'", config.Database.Host) + } + // DB_PORT should be 5432 (.env value, not in OS env) + if config.Database.Port != 5432 { + t.Errorf("Expected DB_PORT=5432 (.env value), got %d", config.Database.Port) + } + }) + + t.Run("test_catalog_source_tracking", func(t *testing.T) { + catalog := GetGlobalEnvCatalog() + + // Test source tracking + hostSource := catalog.GetSource("DB_HOST") + portSource := catalog.GetSource("DB_PORT") + appPortSource := catalog.GetSource("APP_PORT") + + if hostSource != "os_env" { + t.Errorf("Expected DB_HOST source to be 'os_env', got '%s'", hostSource) + } + + if portSource != "dotenv:"+tempFile { + t.Errorf("Expected DB_PORT source to be 'dotenv:...', got '%s'", portSource) + } + + if appPortSource != "os_env" { + t.Errorf("Expected APP_PORT source to be 'os_env', got '%s'", appPortSource) + } + }) +} + +func TestMultiFeederCombination(t *testing.T) { + // Reset catalog + ResetGlobalEnvCatalog() + + // Create test .env file + envContent := []byte(` +# Base configuration +BASE_HOST=dotenv-base-host +BASE_PORT=3000 +`) + + tempFile := filepath.Join(os.TempDir(), "multi_feeder_test.env") + err := os.WriteFile(tempFile, envContent, 0600) + if err != nil { + t.Fatalf("Failed to create test .env file: %v", err) + } + defer os.Remove(tempFile) + + // Set OS environment variables + t.Setenv("OS_OVERRIDE_HOST", "os-host") + t.Setenv("PROD__CONFIG__ENV", "production") // For AffixedEnvFeeder + + type Config struct { + BaseHost string `env:"BASE_HOST"` + BasePort int `env:"BASE_PORT"` + OverrideHost string `env:"OS_OVERRIDE_HOST"` + ProdConfig string `env:"CONFIG"` // For AffixedEnvFeeder + } + + var config Config + tracker := NewDefaultFieldTracker() + + // Feed in order: DotEnv → Env → AffixedEnv + feeders := []interface { + Feed(interface{}) error + SetFieldTracker(FieldTracker) + }{ + NewDotEnvFeeder(tempFile), + NewEnvFeeder(), + &AffixedEnvFeeder{Prefix: "PROD_", Suffix: "_ENV"}, + } + + for i, feeder := range feeders { + feeder.SetFieldTracker(tracker) + err := feeder.Feed(&config) + if err != nil { + t.Fatalf("Feeder %d failed: %v", i, err) + } + } + + // Verify final values + if config.BaseHost != "dotenv-base-host" { + t.Errorf("Expected BaseHost='dotenv-base-host', got '%s'", config.BaseHost) + } + + if config.BasePort != 3000 { + t.Errorf("Expected BasePort=3000, got %d", config.BasePort) + } + + if config.OverrideHost != "os-host" { + t.Errorf("Expected OverrideHost='os-host', got '%s'", config.OverrideHost) + } + + if config.ProdConfig != "production" { + t.Errorf("Expected ProdConfig='production', got '%s'", config.ProdConfig) + } + + // Verify field tracking shows correct sources + populations := tracker.GetFieldPopulations() + if len(populations) == 0 { + t.Fatal("No field populations recorded") + } + + t.Logf("Recorded %d field populations:", len(populations)) + for _, pop := range populations { + t.Logf(" %s = %v (from %s, key: %s)", pop.FieldPath, pop.Value, pop.SourceType, pop.SourceKey) + } +} diff --git a/feeders/yaml.go b/feeders/yaml.go index 3f6cb858..7eab58d3 100644 --- a/feeders/yaml.go +++ b/feeders/yaml.go @@ -2,27 +2,30 @@ package feeders import ( "fmt" + "os" "reflect" + "strconv" - "github.com/golobby/config/v3/pkg/feeder" "gopkg.in/yaml.v3" ) // YamlFeeder is a feeder that reads YAML files with optional verbose debug logging type YamlFeeder struct { - feeder.Yaml + Path string verboseDebug bool logger interface { Debug(msg string, args ...any) } + fieldTracker FieldTracker } // NewYamlFeeder creates a new YamlFeeder that reads from the specified YAML file -func NewYamlFeeder(filePath string) YamlFeeder { - return YamlFeeder{ - Yaml: feeder.Yaml{Path: filePath}, +func NewYamlFeeder(filePath string) *YamlFeeder { + return &YamlFeeder{ + Path: filePath, verboseDebug: false, logger: nil, + fieldTracker: nil, } } @@ -35,13 +38,20 @@ func (y *YamlFeeder) SetVerboseDebug(enabled bool, logger interface{ Debug(msg s } } +// SetFieldTracker sets the field tracker for recording field populations +func (y *YamlFeeder) SetFieldTracker(tracker FieldTracker) { + y.fieldTracker = tracker +} + // Feed reads the YAML file and populates the provided structure -func (y YamlFeeder) Feed(structure interface{}) error { +func (y *YamlFeeder) Feed(structure interface{}) error { if y.verboseDebug && y.logger != nil { y.logger.Debug("YamlFeeder: Starting feed process", "filePath", y.Path, "structureType", reflect.TypeOf(structure)) } - err := y.Yaml.Feed(structure) + // Always use custom parsing logic for consistency + err := y.feedWithTracking(structure) + if y.verboseDebug && y.logger != nil { if err != nil { y.logger.Debug("YamlFeeder: Feed completed with error", "filePath", y.Path, "error", err) @@ -56,7 +66,7 @@ func (y YamlFeeder) Feed(structure interface{}) error { } // FeedKey reads a YAML file and extracts a specific key -func (y YamlFeeder) FeedKey(key string, target interface{}) error { +func (y *YamlFeeder) FeedKey(key string, target interface{}) error { if y.verboseDebug && y.logger != nil { y.logger.Debug("YamlFeeder: Starting FeedKey process", "filePath", y.Path, "key", key, "targetType", reflect.TypeOf(target)) } @@ -106,3 +116,425 @@ func (y YamlFeeder) FeedKey(key string, target interface{}) error { } return nil } + +// feedWithTracking processes YAML data with field tracking support +func (y *YamlFeeder) feedWithTracking(structure interface{}) error { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Starting feedWithTracking", "filePath", y.Path, "structureType", reflect.TypeOf(structure)) + } + + // Read YAML file + content, err := os.ReadFile(y.Path) + if err != nil { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Failed to read YAML file", "filePath", y.Path, "error", err) + } + return fmt.Errorf("failed to read YAML file: %w", err) + } + + // Check if we're dealing with a struct pointer + structValue := reflect.ValueOf(structure) + if structValue.Kind() != reflect.Ptr || structValue.Elem().Kind() != reflect.Struct { + // Not a struct pointer, fall back to standard YAML unmarshaling + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Not a struct pointer, using standard YAML unmarshaling", "structureType", reflect.TypeOf(structure)) + } + if err := yaml.Unmarshal(content, structure); err != nil { + return fmt.Errorf("failed to unmarshal YAML data: %w", err) + } + return nil + } + + // Parse YAML content + data := make(map[string]interface{}) + if err := yaml.Unmarshal(content, &data); err != nil { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Failed to parse YAML content", "filePath", y.Path, "error", err) + } + return fmt.Errorf("failed to parse YAML content: %w", err) + } + + // Process the structure fields with tracking + return y.processStructFields(reflect.ValueOf(structure).Elem(), data, "") +} + +// processStructFields processes struct fields and tracks field populations from YAML data +func (y *YamlFeeder) processStructFields(rv reflect.Value, data map[string]interface{}, parentPath string) error { + structType := rv.Type() + + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Processing struct fields", "structType", structType, "numFields", rv.NumField(), "parentPath", parentPath) + } + + for i := 0; i < rv.NumField(); i++ { + field := rv.Field(i) + fieldType := structType.Field(i) + + // Build field path + fieldPath := fieldType.Name + if parentPath != "" { + fieldPath = parentPath + "." + fieldType.Name + } + + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Processing field", "fieldName", fieldType.Name, "fieldType", fieldType.Type, "fieldPath", fieldPath) + } + + if err := y.processField(field, &fieldType, data, fieldPath); err != nil { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Field processing failed", "fieldName", fieldType.Name, "error", err) + } + return fmt.Errorf("error in field '%s': %w", fieldType.Name, err) + } + } + return nil +} + +// processField handles a single struct field with YAML data and field tracking +func (y *YamlFeeder) processField(field reflect.Value, fieldType *reflect.StructField, data map[string]interface{}, fieldPath string) error { + // Handle nested structs + switch field.Kind() { + case reflect.Map: + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Processing map field", "fieldName", fieldType.Name, "fieldPath", fieldPath) + } + + // Check if there's a yaml tag for this map + if yamlTag, exists := fieldType.Tag.Lookup("yaml"); exists { + // Look for map data using the yaml tag + if mapData, found := data[yamlTag]; found { + if mapDataTyped, ok := mapData.(map[string]interface{}); ok { + return y.setMapFromYaml(field, mapDataTyped, fieldType.Name, fieldPath) + } else { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Map YAML data is not a map[string]interface{}", "fieldName", fieldType.Name, "yamlTag", yamlTag, "dataType", reflect.TypeOf(mapData)) + } + } + } else { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Map YAML data not found", "fieldName", fieldType.Name, "yamlTag", yamlTag) + } + } + } + case reflect.Struct: + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Processing nested struct", "fieldName", fieldType.Name, "fieldPath", fieldPath) + } + + // Check if there's a yaml tag for this nested struct + if yamlTag, exists := fieldType.Tag.Lookup("yaml"); exists { + // Look for nested data using the yaml tag + if nestedData, found := data[yamlTag]; found { + if nestedMap, ok := nestedData.(map[string]interface{}); ok { + return y.processStructFields(field, nestedMap, fieldPath) + } else { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Nested YAML data is not a map", "fieldName", fieldType.Name, "yamlTag", yamlTag, "dataType", reflect.TypeOf(nestedData)) + } + } + } else { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Nested YAML data not found", "fieldName", fieldType.Name, "yamlTag", yamlTag) + } + } + } else { + // No yaml tag, use the same data map + return y.processStructFields(field, data, fieldPath) + } + case reflect.Pointer: + if !field.IsZero() && field.Elem().Kind() == reflect.Struct { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Processing nested struct pointer", "fieldName", fieldType.Name, "fieldPath", fieldPath) + } + + // Check if there's a yaml tag for this nested struct pointer + if yamlTag, exists := fieldType.Tag.Lookup("yaml"); exists { + // Look for nested data using the yaml tag + if nestedData, found := data[yamlTag]; found { + if nestedMap, ok := nestedData.(map[string]interface{}); ok { + return y.processStructFields(field.Elem(), nestedMap, fieldPath) + } else { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Nested YAML data is not a map", "fieldName", fieldType.Name, "yamlTag", yamlTag, "dataType", reflect.TypeOf(nestedData)) + } + } + } else { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Nested YAML data not found", "fieldName", fieldType.Name, "yamlTag", yamlTag) + } + } + } else { + // No yaml tag, use the same data map + return y.processStructFields(field.Elem(), data, fieldPath) + } + } + case reflect.Invalid, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128, reflect.Array, + reflect.Chan, reflect.Func, reflect.Interface, reflect.Slice, reflect.String, reflect.UnsafePointer: + // Check for yaml tag for primitive types and other non-struct types + if yamlTag, exists := fieldType.Tag.Lookup("yaml"); exists { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Found yaml tag", "fieldName", fieldType.Name, "yamlTag", yamlTag, "fieldPath", fieldPath) + } + return y.setFieldFromYaml(field, yamlTag, data, fieldType.Name, fieldPath) + } else if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: No yaml tag found", "fieldName", fieldType.Name, "fieldPath", fieldPath) + } + default: + // Check for yaml tag for primitive types and other non-struct types + if yamlTag, exists := fieldType.Tag.Lookup("yaml"); exists { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Found yaml tag", "fieldName", fieldType.Name, "yamlTag", yamlTag, "fieldPath", fieldPath) + } + return y.setFieldFromYaml(field, yamlTag, data, fieldType.Name, fieldPath) + } else if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: No yaml tag found", "fieldName", fieldType.Name, "fieldPath", fieldPath) + } + } + + return nil +} + +// setFieldFromYaml sets a field value from YAML data with field tracking +func (y *YamlFeeder) setFieldFromYaml(field reflect.Value, yamlTag string, data map[string]interface{}, fieldName, fieldPath string) error { + // Find the value in YAML data + searchKeys := []string{yamlTag} + var foundValue interface{} + var foundKey string + + if value, exists := data[yamlTag]; exists { + foundValue = value + foundKey = yamlTag + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Found YAML value", "fieldName", fieldName, "yamlKey", yamlTag, "value", value, "fieldPath", fieldPath) + } + } + + if foundValue != nil { + // Set the field value + err := y.setFieldValue(field, foundValue) + if err != nil { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Failed to set field value", "fieldName", fieldName, "yamlKey", yamlTag, "value", foundValue, "error", err, "fieldPath", fieldPath) + } + return err + } + + // Record field population if tracker is available + if y.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldName, + FieldType: field.Type().String(), + FeederType: "*feeders.YamlFeeder", + SourceType: "yaml", + SourceKey: foundKey, + Value: field.Interface(), + InstanceKey: "", + SearchKeys: searchKeys, + FoundKey: foundKey, + } + y.fieldTracker.RecordFieldPopulation(fp) + } + + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Successfully set field value", "fieldName", fieldName, "yamlKey", yamlTag, "value", foundValue, "fieldPath", fieldPath) + } + } else { + // Record that we searched but didn't find + if y.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldName, + FieldType: field.Type().String(), + FeederType: "*feeders.YamlFeeder", + SourceType: "yaml", + SourceKey: "", + Value: nil, + InstanceKey: "", + SearchKeys: searchKeys, + FoundKey: "", + } + y.fieldTracker.RecordFieldPopulation(fp) + } + + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: YAML value not found", "fieldName", fieldName, "yamlKey", yamlTag, "fieldPath", fieldPath) + } + } + + return nil +} + +// setMapFromYaml sets a map field value from YAML data with field tracking +func (y *YamlFeeder) setMapFromYaml(field reflect.Value, yamlData map[string]interface{}, fieldName, fieldPath string) error { + if !field.CanSet() { + return wrapYamlFieldCannotBeSetError() + } + + mapType := field.Type() + keyType := mapType.Key() + valueType := mapType.Elem() + + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Setting map from YAML", "fieldName", fieldName, "mapType", mapType, "keyType", keyType, "valueType", valueType) + } + + // Create a new map + newMap := reflect.MakeMap(mapType) + + // Handle different value types + switch valueType.Kind() { + case reflect.Struct: + // Map of structs, like map[string]DBConnection + for key, value := range yamlData { + if valueMap, ok := value.(map[string]interface{}); ok { + // Create a new instance of the struct type + structValue := reflect.New(valueType).Elem() + + // Process the struct fields + if err := y.processStructFields(structValue, valueMap, fieldPath+"."+key); err != nil { + return fmt.Errorf("error processing map entry '%s': %w", key, err) + } + + // Set the map entry + keyValue := reflect.ValueOf(key) + newMap.SetMapIndex(keyValue, structValue) + } else { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Map entry is not a map", "key", key, "valueType", reflect.TypeOf(value)) + } + } + } + case reflect.Invalid, reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128, reflect.Array, + reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice, reflect.String, + reflect.UnsafePointer: + // Map of primitive types - use direct conversion + for key, value := range yamlData { + keyValue := reflect.ValueOf(key) + valueReflect := reflect.ValueOf(value) + + if valueReflect.Type().ConvertibleTo(valueType) { + convertedValue := valueReflect.Convert(valueType) + newMap.SetMapIndex(keyValue, convertedValue) + } else { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Cannot convert map value", "key", key, "valueType", valueReflect.Type(), "targetType", valueType) + } + } + } + default: + // Map of primitive types - use direct conversion + for key, value := range yamlData { + keyValue := reflect.ValueOf(key) + valueReflect := reflect.ValueOf(value) + + if valueReflect.Type().ConvertibleTo(valueType) { + convertedValue := valueReflect.Convert(valueType) + newMap.SetMapIndex(keyValue, convertedValue) + } else { + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Cannot convert map value", "key", key, "valueType", valueReflect.Type(), "targetType", valueType) + } + } + } + } + + // Set the field to the new map + field.Set(newMap) + + // Record field population if tracker is available + if y.fieldTracker != nil { + fp := FieldPopulation{ + FieldPath: fieldPath, + FieldName: fieldName, + FieldType: field.Type().String(), + FeederType: "*feeders.YamlFeeder", + SourceType: "yaml", + SourceKey: fieldName, // For maps, use the field name as the source key + Value: field.Interface(), + InstanceKey: "", + SearchKeys: []string{fieldName}, + FoundKey: fieldName, + } + y.fieldTracker.RecordFieldPopulation(fp) + } + + if y.verboseDebug && y.logger != nil { + y.logger.Debug("YamlFeeder: Successfully set map field", "fieldName", fieldName, "mapSize", newMap.Len()) + } + + return nil +} + +// setFieldValue sets a reflect.Value from an interface{} value +func (y *YamlFeeder) setFieldValue(field reflect.Value, value interface{}) error { + if !field.CanSet() { + return wrapYamlFieldCannotBeSetError() + } + + valueReflect := reflect.ValueOf(value) + if !valueReflect.IsValid() { + return nil // Skip nil values + } + + // Handle type conversion + if valueReflect.Type().ConvertibleTo(field.Type()) { + field.Set(valueReflect.Convert(field.Type())) + return nil + } + + // Handle string conversion for basic types + if valueReflect.Kind() == reflect.String { + str := valueReflect.String() + switch field.Kind() { + case reflect.String: + field.SetString(str) + case reflect.Bool: + switch str { + case "true", "1": + field.SetBool(true) + case "false", "0": + field.SetBool(false) + default: + return wrapYamlBoolConversionError(str) + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if intVal, err := strconv.ParseInt(str, 10, 64); err == nil { + field.SetInt(intVal) + } else { + return fmt.Errorf("cannot convert string '%s' to int: %w", str, err) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if uintVal, err := strconv.ParseUint(str, 10, 64); err == nil { + field.SetUint(uintVal) + } else { + return fmt.Errorf("cannot convert string '%s' to uint: %w", str, err) + } + case reflect.Float32, reflect.Float64: + if floatVal, err := strconv.ParseFloat(str, 64); err == nil { + field.SetFloat(floatVal) + } else { + return fmt.Errorf("cannot convert string '%s' to float: %w", str, err) + } + case reflect.Invalid, reflect.Uintptr, reflect.Complex64, reflect.Complex128, + reflect.Array, reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, + reflect.Ptr, reflect.Slice, reflect.Struct, reflect.UnsafePointer: + return wrapYamlUnsupportedFieldTypeError(field.Type().String()) + default: + return wrapYamlUnsupportedFieldTypeError(field.Type().String()) + } + return nil + } + + // Direct assignment for matching types + if valueReflect.Type() == field.Type() { + field.Set(valueReflect) + return nil + } + + return wrapYamlTypeConversionError(valueReflect.Type().String(), field.Type().String()) +} diff --git a/feeders/yaml_test.go b/feeders/yaml_test.go index f71b688e..9a6d7ae3 100644 --- a/feeders/yaml_test.go +++ b/feeders/yaml_test.go @@ -1,11 +1,28 @@ package feeders import ( + "errors" + "fmt" "os" + "strings" "testing" ) -func TestYamlFeeder_Feed(t *testing.T) { +// Mock logger for testing verbose debug functionality +type mockLogger struct { + messages []string +} + +func (m *mockLogger) Debug(msg string, args ...any) { + formatted := fmt.Sprintf(msg, args...) + m.messages = append(m.messages, formatted) +} + +func (m *mockLogger) getMessages() []string { + return m.messages +} + +func TestYamlFeeder_Feed_BasicStructure(t *testing.T) { tempFile, err := os.CreateTemp("", "test-*.yaml") if err != nil { t.Fatalf("Failed to create temp file: %v", err) @@ -47,3 +64,585 @@ app: t.Errorf("Expected Debug to be true, got false") } } + +func TestYamlFeeder_Feed_PrimitiveTypes(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + yamlContent := ` +stringField: "hello" +intField: 42 +int64Field: 9223372036854775807 +uintField: 123 +floatField: 3.14 +boolField: true +` + if _, err := tempFile.Write([]byte(yamlContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + type Config struct { + StringField string `yaml:"stringField"` + IntField int `yaml:"intField"` + Int64Field int64 `yaml:"int64Field"` + UintField uint `yaml:"uintField"` + FloatField float64 `yaml:"floatField"` + BoolField bool `yaml:"boolField"` + } + + var config Config + feeder := NewYamlFeeder(tempFile.Name()) + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if config.StringField != "hello" { + t.Errorf("Expected StringField to be 'hello', got '%s'", config.StringField) + } + if config.IntField != 42 { + t.Errorf("Expected IntField to be 42, got %d", config.IntField) + } + if config.Int64Field != 9223372036854775807 { + t.Errorf("Expected Int64Field to be 9223372036854775807, got %d", config.Int64Field) + } + if config.UintField != 123 { + t.Errorf("Expected UintField to be 123, got %d", config.UintField) + } + if config.FloatField != 3.14 { + t.Errorf("Expected FloatField to be 3.14, got %f", config.FloatField) + } + if !config.BoolField { + t.Errorf("Expected BoolField to be true, got false") + } +} + +func TestYamlFeeder_Feed_StringConversions(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + yamlContent := ` +intFromString: "42" +floatFromString: "3.14" +boolFromString: "true" +boolFromOne: "1" +boolFromFalse: "false" +boolFromZero: "0" +` + if _, err := tempFile.Write([]byte(yamlContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + type Config struct { + IntFromString int `yaml:"intFromString"` + FloatFromString float64 `yaml:"floatFromString"` + BoolFromString bool `yaml:"boolFromString"` + BoolFromOne bool `yaml:"boolFromOne"` + BoolFromFalse bool `yaml:"boolFromFalse"` + BoolFromZero bool `yaml:"boolFromZero"` + } + + // Test with field tracking enabled to use custom parsing + tracker := NewDefaultFieldTracker() + var config Config + feeder := NewYamlFeeder(tempFile.Name()) + feeder.SetFieldTracker(tracker) + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if config.IntFromString != 42 { + t.Errorf("Expected IntFromString to be 42, got %d", config.IntFromString) + } + if config.FloatFromString != 3.14 { + t.Errorf("Expected FloatFromString to be 3.14, got %f", config.FloatFromString) + } + if !config.BoolFromString { + t.Errorf("Expected BoolFromString to be true, got false") + } + if !config.BoolFromOne { + t.Errorf("Expected BoolFromOne to be true, got false") + } + if config.BoolFromFalse { + t.Errorf("Expected BoolFromFalse to be false, got true") + } + if config.BoolFromZero { + t.Errorf("Expected BoolFromZero to be false, got true") + } +} + +func TestYamlFeeder_Feed_MapFields(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + yamlContent := ` +connections: + primary: + host: "localhost" + port: 5432 + database: "mydb" + secondary: + host: "backup.host" + port: 5433 + database: "backupdb" +stringMap: + key1: "value1" + key2: "value2" +` + if _, err := tempFile.Write([]byte(yamlContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + type DBConnection struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + Database string `yaml:"database"` + } + + type Config struct { + Connections map[string]DBConnection `yaml:"connections"` + StringMap map[string]string `yaml:"stringMap"` + } + + var config Config + feeder := NewYamlFeeder(tempFile.Name()) + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if len(config.Connections) != 2 { + t.Errorf("Expected 2 connections, got %d", len(config.Connections)) + } + if config.Connections["primary"].Host != "localhost" { + t.Errorf("Expected primary host to be 'localhost', got '%s'", config.Connections["primary"].Host) + } + if config.Connections["primary"].Port != 5432 { + t.Errorf("Expected primary port to be 5432, got %d", config.Connections["primary"].Port) + } + if config.Connections["secondary"].Database != "backupdb" { + t.Errorf("Expected secondary database to be 'backupdb', got '%s'", config.Connections["secondary"].Database) + } + + if len(config.StringMap) != 2 { + t.Errorf("Expected 2 string map entries, got %d", len(config.StringMap)) + } + if config.StringMap["key1"] != "value1" { + t.Errorf("Expected key1 to be 'value1', got '%s'", config.StringMap["key1"]) + } +} + +func TestYamlFeeder_Feed_FieldTracking(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + yamlContent := ` +app: + name: TestApp + version: "1.0" +database: + host: localhost + port: 5432 +` + if _, err := tempFile.Write([]byte(yamlContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + type Config struct { + App struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + } `yaml:"app"` + Database struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + } `yaml:"database"` + NotFound string `yaml:"notfound"` + } + + tracker := NewDefaultFieldTracker() + var config Config + feeder := NewYamlFeeder(tempFile.Name()) + feeder.SetFieldTracker(tracker) + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + populations := tracker.GetFieldPopulations() + if len(populations) == 0 { + t.Error("Expected field populations to be recorded") + } + + // Check that we have records for found fields + foundFields := make(map[string]bool) + for _, pop := range populations { + if pop.FoundKey != "" { + foundFields[pop.FieldPath] = true + } + } + + expectedFields := []string{"App.Name", "App.Version", "Database.Host", "Database.Port"} + for _, field := range expectedFields { + if !foundFields[field] { + t.Errorf("Expected field %s to be found and recorded", field) + } + } +} + +func TestYamlFeeder_Feed_VerboseDebug(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + yamlContent := ` +app: + name: TestApp +` + if _, err := tempFile.Write([]byte(yamlContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + type Config struct { + App struct { + Name string `yaml:"name"` + } `yaml:"app"` + } + + logger := &mockLogger{} + var config Config + feeder := NewYamlFeeder(tempFile.Name()) + feeder.SetVerboseDebug(true, logger) + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + messages := logger.getMessages() + if len(messages) == 0 { + t.Error("Expected debug messages to be logged") + } + + // Check for specific debug messages + found := false + for _, msg := range messages { + if strings.Contains(msg, "Starting feed process") { + found = true + break + } + } + if !found { + t.Error("Expected to find 'Starting feed process' in debug messages") + } +} + +func TestYamlFeeder_FeedKey(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + yamlContent := ` +app: + name: TestApp + version: "1.0" +database: + host: localhost + port: 5432 +` + if _, err := tempFile.Write([]byte(yamlContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + type AppConfig struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + } + + var appConfig AppConfig + feeder := NewYamlFeeder(tempFile.Name()) + err = feeder.FeedKey("app", &appConfig) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if appConfig.Name != "TestApp" { + t.Errorf("Expected Name to be 'TestApp', got '%s'", appConfig.Name) + } + if appConfig.Version != "1.0" { + t.Errorf("Expected Version to be '1.0', got '%s'", appConfig.Version) + } +} + +func TestYamlFeeder_FeedKey_NotFound(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + yamlContent := ` +app: + name: TestApp +` + if _, err := tempFile.Write([]byte(yamlContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + type NotFoundConfig struct { + Value string `yaml:"value"` + } + + var config NotFoundConfig + feeder := NewYamlFeeder(tempFile.Name()) + err = feeder.FeedKey("notfound", &config) + if err != nil { + t.Fatalf("Expected no error for missing key, got %v", err) + } + + if config.Value != "" { + t.Errorf("Expected empty value for missing key, got '%s'", config.Value) + } +} + +func TestYamlFeeder_Feed_FileNotFound(t *testing.T) { + feeder := NewYamlFeeder("/nonexistent/file.yaml") + var config struct{} + err := feeder.Feed(&config) + if err == nil { + t.Error("Expected error for nonexistent file") + } +} + +func TestYamlFeeder_Feed_InvalidYaml(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + invalidYaml := ` +app: + name: TestApp + version: "1.0" + invalid: [unclosed array +` + if _, err := tempFile.Write([]byte(invalidYaml)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + type Config struct { + App struct { + Name string `yaml:"name"` + } `yaml:"app"` + } + + var config Config + feeder := NewYamlFeeder(tempFile.Name()) + err = feeder.Feed(&config) + if err == nil { + t.Error("Expected error for invalid YAML") + } +} + +func TestYamlFeeder_Feed_BoolConversionError(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + yamlContent := ` +boolField: "invalid" +` + if _, err := tempFile.Write([]byte(yamlContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + type Config struct { + BoolField bool `yaml:"boolField"` + } + + tracker := NewDefaultFieldTracker() + var config Config + feeder := NewYamlFeeder(tempFile.Name()) + feeder.SetFieldTracker(tracker) + err = feeder.Feed(&config) + if err == nil { + t.Error("Expected error for invalid bool conversion") + } + if !errors.Is(err, ErrYamlBoolConversion) { + t.Errorf("Expected ErrYamlBoolConversion, got %v", err) + } +} + +func TestYamlFeeder_Feed_IntConversionError(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + yamlContent := ` +intField: "not_a_number" +` + if _, err := tempFile.Write([]byte(yamlContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + type Config struct { + IntField int `yaml:"intField"` + } + + tracker := NewDefaultFieldTracker() + var config Config + feeder := NewYamlFeeder(tempFile.Name()) + feeder.SetFieldTracker(tracker) + err = feeder.Feed(&config) + if err == nil { + t.Error("Expected error for invalid int conversion") + } +} + +func TestYamlFeeder_Feed_NoFieldTracker(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + yamlContent := ` +app: + name: TestApp +` + if _, err := tempFile.Write([]byte(yamlContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + type Config struct { + App struct { + Name string `yaml:"name"` + } `yaml:"app"` + } + + var config Config + feeder := NewYamlFeeder(tempFile.Name()) + // Don't set field tracker - should use original behavior + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if config.App.Name != "TestApp" { + t.Errorf("Expected Name to be 'TestApp', got '%s'", config.App.Name) + } +} + +func TestYamlFeeder_Feed_NonStructPointer(t *testing.T) { + tempFile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer os.Remove(tempFile.Name()) + + yamlContent := ` +- item1 +- item2 +` + if _, err := tempFile.Write([]byte(yamlContent)); err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + tempFile.Close() + + tracker := NewDefaultFieldTracker() + var config []string + feeder := NewYamlFeeder(tempFile.Name()) + feeder.SetFieldTracker(tracker) + err = feeder.Feed(&config) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if len(config) != 2 { + t.Errorf("Expected 2 items, got %d", len(config)) + } +} + +func TestYamlFeeder_NewYamlFeeder(t *testing.T) { + filePath := "/test/path.yaml" + feeder := NewYamlFeeder(filePath) + + if feeder == nil { + t.Fatal("Expected feeder to be created, got nil") + } + if feeder.Path != filePath { + t.Errorf("Expected path to be '%s', got '%s'", filePath, feeder.Path) + } + if feeder.verboseDebug { + t.Error("Expected verboseDebug to be false by default") + } + if feeder.logger != nil { + t.Error("Expected logger to be nil by default") + } + if feeder.fieldTracker != nil { + t.Error("Expected fieldTracker to be nil by default") + } +} + +func TestYamlFeeder_SetVerboseDebug(t *testing.T) { + feeder := NewYamlFeeder("/test/path.yaml") + logger := &mockLogger{} + + feeder.SetVerboseDebug(true, logger) + + if !feeder.verboseDebug { + t.Error("Expected verboseDebug to be true") + } + if feeder.logger != logger { + t.Error("Expected logger to be set") + } + + // Check that debug message was logged + messages := logger.getMessages() + if len(messages) == 0 { + t.Error("Expected debug message to be logged") + } +} + +func TestYamlFeeder_SetFieldTracker(t *testing.T) { + feeder := NewYamlFeeder("/test/path.yaml") + tracker := NewDefaultFieldTracker() + + feeder.SetFieldTracker(tracker) + + if feeder.fieldTracker != tracker { + t.Error("Expected fieldTracker to be set") + } +} diff --git a/field_tracker_bridge.go b/field_tracker_bridge.go new file mode 100644 index 00000000..e05c045b --- /dev/null +++ b/field_tracker_bridge.go @@ -0,0 +1,39 @@ +package modular + +import ( + "github.com/GoCodeAlone/modular/feeders" +) + +// FieldTrackerBridge adapts between the main package's FieldTracker interface +// and the feeders package's FieldTracker interface +type FieldTrackerBridge struct { + mainTracker FieldTracker +} + +// NewFieldTrackerBridge creates a new bridge adapter +func NewFieldTrackerBridge(mainTracker FieldTracker) *FieldTrackerBridge { + return &FieldTrackerBridge{ + mainTracker: mainTracker, + } +} + +// RecordFieldPopulation implements the feeders.FieldTracker interface +// by converting feeders.FieldPopulation to the main package's FieldPopulation +func (b *FieldTrackerBridge) RecordFieldPopulation(fp feeders.FieldPopulation) { + // Convert from feeders.FieldPopulation to main package FieldPopulation + mainFP := FieldPopulation{ + FieldPath: fp.FieldPath, + FieldName: fp.FieldName, + FieldType: fp.FieldType, + FeederType: fp.FeederType, + SourceType: fp.SourceType, + SourceKey: fp.SourceKey, + Value: fp.Value, + InstanceKey: fp.InstanceKey, + SearchKeys: fp.SearchKeys, + FoundKey: fp.FoundKey, + } + + // Record to the main tracker + b.mainTracker.RecordFieldPopulation(mainFP) +} diff --git a/go.mod b/go.mod index da1f6f98..a5b0a85a 100644 --- a/go.mod +++ b/go.mod @@ -7,15 +7,12 @@ toolchain go1.24.2 require ( github.com/BurntSushi/toml v1.5.0 github.com/golobby/cast v1.3.3 - github.com/golobby/config/v3 v3.4.2 github.com/stretchr/testify v1.10.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/golobby/dotenv v1.3.2 // indirect - github.com/golobby/env/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/stretchr/objx v0.5.2 // indirect diff --git a/go.sum b/go.sum index 5bb36bda..d0023fc0 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -8,12 +7,6 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golobby/cast v1.3.3 h1:s2Lawb9RMz7YyYf8IrfMQY4IFmA1R/lgfmj97Vc6fig= github.com/golobby/cast v1.3.3/go.mod h1:0oDO5IT84HTXcbLDf1YXuk0xtg/cRDrxhbpWKxwtJCY= -github.com/golobby/config/v3 v3.4.2 h1:oIOSo24mC0A8f93ZTL24NDNw0hZ3Tbb34wc1ckn2CsA= -github.com/golobby/config/v3 v3.4.2/go.mod h1:3go9UVPb3bBNrH7qidd4vd1HbsAAwIYqcQJgGmAa044= -github.com/golobby/dotenv v1.3.2 h1:9vA8XqXXIB3cX/5xQ1CTbOCPegioHtHXIxeFng+uOqQ= -github.com/golobby/dotenv v1.3.2/go.mod h1:9MMVXqzLNluhVxCv3X/DLYBNUb289f05tr+df1+7278= -github.com/golobby/env/v2 v2.2.4 h1:sjdTe+bScPRWUIA1AQH95RHv52jM5Mns2XHwLyEbkzk= -github.com/golobby/env/v2 v2.2.4/go.mod h1:HDJW+dHHwLxkb8FZMjBTBiZUFl1iAA4F9YX15kBC84c= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= diff --git a/user_scenario_test.go b/user_scenario_test.go new file mode 100644 index 00000000..49fd33f5 --- /dev/null +++ b/user_scenario_test.go @@ -0,0 +1,198 @@ +package modular_test + +import ( + "strings" + "testing" + + "github.com/GoCodeAlone/modular" + "github.com/GoCodeAlone/modular/feeders" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// MockLogger for capturing debug output +type MockLogger struct { + mock.Mock +} + +func (m *MockLogger) Debug(msg string, args ...any) { + m.Called(msg, args) +} + +func (m *MockLogger) Error(msg string, args ...any) { + m.Called(msg, args) +} + +func (m *MockLogger) Info(msg string, args ...any) { + m.Called(msg, args) +} + +func (m *MockLogger) Warn(msg string, args ...any) { + m.Called(msg, args) +} + +// TestUserScenario tests the exact scenario the user described: +// Database module DSN configuration not being populated from ENV vars +// in instance-aware scenarios, with verbose logging to show exactly +// what's being looked for. +func TestUserScenario_DatabaseDSNInstanceAware(t *testing.T) { + // This test demonstrates that the user can now get complete visibility + // into field-level configuration population, especially for Database + // module DSN values with instance-aware environment variables. + + // Set up the exact environment variables the user would have + envVars := map[string]string{ + "DB_PRIMARY_DRIVER": "postgres", + "DB_PRIMARY_DSN": "postgres://user:pass@localhost/primary_db", + "DB_PRIMARY_MAX_CONNS": "25", + "DB_SECONDARY_DRIVER": "mysql", + "DB_SECONDARY_DSN": "mysql://user:pass@localhost/secondary_db", + "DB_SECONDARY_MAX_CONNS": "15", + "DB_CACHE_DRIVER": "redis", + "DB_CACHE_DSN": "redis://localhost:6379/0", + "DB_CACHE_MAX_CONNS": "10", + } + + for key, value := range envVars { + t.Setenv(key, value) + } + + // Create a logger that captures all debug output + mockLogger := new(MockLogger) + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + // Create field tracker to get detailed field population information + tracker := modular.NewDefaultFieldTracker() + tracker.SetLogger(mockLogger) + + // Define the Database module configuration structure + // (matching the actual Database module) + type ConnectionConfig struct { + Driver string `env:"DRIVER"` + DSN string `env:"DSN"` // This is the field the user was having trouble with + MaxConns int `env:"MAX_CONNS"` + } + + type DatabaseConfig struct { + Connections map[string]ConnectionConfig `yaml:"connections"` + Default string `yaml:"default"` + } + + // Initialize configuration with multiple database instances + dbConfig := &DatabaseConfig{ + Connections: map[string]ConnectionConfig{ + "primary": {}, + "secondary": {}, + "cache": {}, + }, + Default: "primary", + } + + // Create configuration builder with verbose debugging and field tracking + cfg := modular.NewConfig() + cfg.SetVerboseDebug(true, mockLogger) + cfg.SetFieldTracker(tracker) + + // Add instance-aware environment feeder - this is what enables + // instance-specific environment variable mapping + instanceFeeder := feeders.NewInstanceAwareEnvFeeder(func(instanceKey string) string { + return "DB_" + strings.ToUpper(instanceKey) + "_" + }) + cfg.AddFeeder(instanceFeeder) + + // Add the database configuration + cfg.AddStructKey("database", dbConfig) + + // Feed configuration - this will populate from environment variables + err := cfg.Feed() + require.NoError(t, err) + + // Use FeedInstances to populate the connections map with instance-aware values + err = instanceFeeder.FeedInstances(dbConfig.Connections) + require.NoError(t, err) + + // === VERIFICATION: Configuration was populated correctly === + + // Primary database connection + primaryConn := dbConfig.Connections["primary"] + assert.Equal(t, "postgres", primaryConn.Driver) + assert.Equal(t, "postgres://user:pass@localhost/primary_db", primaryConn.DSN) + assert.Equal(t, 25, primaryConn.MaxConns) + + // Secondary database connection + secondaryConn := dbConfig.Connections["secondary"] + assert.Equal(t, "mysql", secondaryConn.Driver) + assert.Equal(t, "mysql://user:pass@localhost/secondary_db", secondaryConn.DSN) + assert.Equal(t, 15, secondaryConn.MaxConns) + + // Cache database connection + cacheConn := dbConfig.Connections["cache"] + assert.Equal(t, "redis", cacheConn.Driver) + assert.Equal(t, "redis://localhost:6379/0", cacheConn.DSN) + assert.Equal(t, 10, cacheConn.MaxConns) + + // === VERIFICATION: Field tracking provides complete visibility === + + populations := tracker.FieldPopulations + require.GreaterOrEqual(t, len(populations), 9, "Should track 3 fields × 3 instances = 9 populations") + + t.Logf("=== FIELD TRACKING RESULTS ===") + t.Logf("Tracked %d field populations:", len(populations)) + + for i, pop := range populations { + t.Logf(" %d: %s.%s -> %v", i+1, pop.InstanceKey, pop.FieldName, pop.Value) + t.Logf(" Source: %s:%s", pop.SourceType, pop.SourceKey) + t.Logf(" Searched: %v", pop.SearchKeys) + t.Logf(" Found: %s", pop.FoundKey) + t.Logf("") + } + + // Specifically verify the user's problem case: DSN field tracking + primaryDSNPop := findInstanceFieldPopulation(populations, "DSN", "primary") + require.NotNil(t, primaryDSNPop, "Primary DSN field population should be tracked") + assert.Equal(t, "postgres://user:pass@localhost/primary_db", primaryDSNPop.Value) + assert.Equal(t, "env", primaryDSNPop.SourceType) + assert.Equal(t, "DB_PRIMARY_DSN", primaryDSNPop.SourceKey) + assert.Equal(t, "primary", primaryDSNPop.InstanceKey) + assert.Contains(t, primaryDSNPop.SearchKeys, "DB_PRIMARY_DSN") + assert.Equal(t, "DB_PRIMARY_DSN", primaryDSNPop.FoundKey) + + secondaryDSNPop := findInstanceFieldPopulation(populations, "DSN", "secondary") + require.NotNil(t, secondaryDSNPop, "Secondary DSN field population should be tracked") + assert.Equal(t, "mysql://user:pass@localhost/secondary_db", secondaryDSNPop.Value) + assert.Equal(t, "env", secondaryDSNPop.SourceType) + assert.Equal(t, "DB_SECONDARY_DSN", secondaryDSNPop.SourceKey) + assert.Equal(t, "secondary", secondaryDSNPop.InstanceKey) + + cacheDSNPop := findInstanceFieldPopulation(populations, "DSN", "cache") + require.NotNil(t, cacheDSNPop, "Cache DSN field population should be tracked") + assert.Equal(t, "redis://localhost:6379/0", cacheDSNPop.Value) + assert.Equal(t, "env", cacheDSNPop.SourceType) + assert.Equal(t, "DB_CACHE_DSN", cacheDSNPop.SourceKey) + assert.Equal(t, "cache", cacheDSNPop.InstanceKey) + + // === VERIFICATION: Verbose debug logging captured everything === + + // Verify that verbose debug logging was used + mockLogger.AssertCalled(t, "Debug", mock.MatchedBy(func(msg string) bool { + return msg == "Field populated" + }), mock.Anything) + + t.Logf("=== SUCCESS ===") + t.Logf("✅ User's issue resolved:") + t.Logf(" - Database DSN fields populated from instance-aware ENV vars") + t.Logf(" - Complete field-level tracking shows exactly which ENV vars were used") + t.Logf(" - Verbose logging provides step-by-step visibility into the process") + t.Logf(" - Each field shows: source type, source key, search keys, found key, instance") +} + +// Helper function to find a field population by field name and instance key +func findInstanceFieldPopulation(populations []modular.FieldPopulation, fieldName, instanceKey string) *modular.FieldPopulation { + for _, pop := range populations { + if pop.FieldName == fieldName && pop.InstanceKey == instanceKey { + return &pop + } + } + return nil +}