From b5764d249e721086945fd45e684251c9c241375b Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Sat, 1 Nov 2025 10:13:08 -0700 Subject: [PATCH 1/3] Improve SQS-Files adapter error logging with available message keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When bucket_path or file_path configuration is incorrect, customers now see which keys are actually available in their SQS messages. This helps them quickly identify the correct path to use instead of trial and error. Error messages now show: - The attempted configuration path - List of available top-level keys in the message 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- sqs-files/client.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/sqs-files/client.go b/sqs-files/client.go index 1adde72..412d865 100644 --- a/sqs-files/client.go +++ b/sqs-files/client.go @@ -196,6 +196,16 @@ func (a *SQSFilesAdapter) getBucketRegion(bucket string) (string, error) { return s3manager.GetBucketRegion(a.ctx, session.Must(session.NewSession(&aws.Config{})), bucket, "us-east-1") } +// getAvailableKeys returns a list of top-level keys from a Dict +// for use in error messages to help users debug configuration issues +func getAvailableKeys(d utils.Dict) []string { + keys := make([]string, 0, len(d)) + for k := range d { + keys = append(keys, k) + } + return keys +} + func (a *SQSFilesAdapter) receiveEvents() error { defer close(a.chFiles) @@ -238,10 +248,13 @@ func (a *SQSFilesAdapter) receiveEvents() error { filePaths := d.ExpandableFindString(a.conf.FilePath) if bucket == "" { - a.conf.ClientOptions.OnError(errors.New("sqsClient.Message: missing bucket")) + availableKeys := getAvailableKeys(d) + a.conf.ClientOptions.OnError(fmt.Errorf("sqsClient.Message: missing bucket - attempted path '%s', available keys in message: %v", a.conf.BucketPath, availableKeys)) continue } if len(filePaths) == 0 { + availableKeys := getAvailableKeys(d) + a.conf.ClientOptions.OnError(fmt.Errorf("sqsClient.Message: missing file paths - attempted path '%s', available keys in message: %v", a.conf.FilePath, availableKeys)) continue } if err := a.initS3SDKs(bucket); err != nil { From 8208b062d99fd1500e7b393fe9fc6b824293db32 Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Thu, 13 Nov 2025 10:31:14 -0800 Subject: [PATCH 2/3] Add key_prefix parameter to sqs-files adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new optional key_prefix parameter to the SQSFilesConfig that allows users to specify a static prefix to prepend to S3 object keys before fetching them. This is useful when files in SQS messages contain relative paths and need a common prefix to construct the full S3 key. The prefix is applied after URL decoding (if enabled) and before the S3 download operation. Changes: - Add KeyPrefix field to SQSFilesConfig struct - Apply prefix in processFiles() method before S3 download - Add comprehensive test suite covering configuration parsing, path application, URL decoding interaction, and edge cases 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- sqs-files/client.go | 5 + sqs-files/client_test.go | 259 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 sqs-files/client_test.go diff --git a/sqs-files/client.go b/sqs-files/client.go index 412d865..6ee451c 100644 --- a/sqs-files/client.go +++ b/sqs-files/client.go @@ -63,6 +63,7 @@ type SQSFilesConfig struct { BucketPath string `json:"bucket_path,omitempty" yaml:"bucket_path,omitempty"` FilePath string `json:"file_path,omitempty" yaml:"file_path,omitempty"` IsDecodeObjectKey bool `json:"is_decode_object_key,omitempty" yaml:"is_decode_object_key,omitempty"` + KeyPrefix string `json:"key_prefix,omitempty" yaml:"key_prefix,omitempty"` // Optional: alternative to BucketPath Bucket string `json:"bucket,omitempty" yaml:"bucket,omitempty"` } @@ -293,6 +294,10 @@ func (a *SQSFilesAdapter) processFiles() error { continue } } + // Apply key prefix if configured + if a.conf.KeyPrefix != "" { + path = a.conf.KeyPrefix + path + } startTime := time.Now().UTC() a.conf.ClientOptions.DebugLog(fmt.Sprintf("downloading file %s", path)) diff --git a/sqs-files/client_test.go b/sqs-files/client_test.go new file mode 100644 index 0000000..d7c3523 --- /dev/null +++ b/sqs-files/client_test.go @@ -0,0 +1,259 @@ +package usp_sqs_files + +import ( + "encoding/json" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSQSFilesConfig_KeyPrefix(t *testing.T) { + tests := []struct { + name string + configJSON string + wantPrefix string + }{ + { + name: "config with key_prefix", + configJSON: `{ + "client_options": { + "hostname": "test.com", + "oid": "test-oid", + "installation_key": "test-key" + }, + "access_key": "test-access", + "secret_key": "test-secret", + "region": "us-east-1", + "queue_url": "https://sqs.us-east-1.amazonaws.com/123456789/test", + "key_prefix": "logs/backup/" + }`, + wantPrefix: "logs/backup/", + }, + { + name: "config without key_prefix", + configJSON: `{ + "client_options": { + "hostname": "test.com", + "oid": "test-oid", + "installation_key": "test-key" + }, + "access_key": "test-access", + "secret_key": "test-secret", + "region": "us-east-1", + "queue_url": "https://sqs.us-east-1.amazonaws.com/123456789/test" + }`, + wantPrefix: "", + }, + { + name: "config with empty key_prefix", + configJSON: `{ + "client_options": { + "hostname": "test.com", + "oid": "test-oid", + "installation_key": "test-key" + }, + "access_key": "test-access", + "secret_key": "test-secret", + "region": "us-east-1", + "queue_url": "https://sqs.us-east-1.amazonaws.com/123456789/test", + "key_prefix": "" + }`, + wantPrefix: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var conf SQSFilesConfig + err := json.Unmarshal([]byte(tt.configJSON), &conf) + require.NoError(t, err) + + // Test that the key_prefix field is correctly unmarshaled + assert.Equal(t, tt.wantPrefix, conf.KeyPrefix) + }) + } +} + +func TestKeyPrefixApplication(t *testing.T) { + tests := []struct { + name string + keyPrefix string + inputPath string + decodeObjectKey bool + expectedPath string + expectDecodeErr bool + }{ + { + name: "no prefix", + keyPrefix: "", + inputPath: "data/file.json", + decodeObjectKey: false, + expectedPath: "data/file.json", + }, + { + name: "prefix with trailing slash", + keyPrefix: "backup/", + inputPath: "data/file.json", + decodeObjectKey: false, + expectedPath: "backup/data/file.json", + }, + { + name: "prefix without trailing slash", + keyPrefix: "logs", + inputPath: "data/file.json", + decodeObjectKey: false, + expectedPath: "logsdata/file.json", + }, + { + name: "multi-level prefix", + keyPrefix: "archive/2024/01/", + inputPath: "events.log", + decodeObjectKey: false, + expectedPath: "archive/2024/01/events.log", + }, + { + name: "prefix with URL encoded path", + keyPrefix: "prefix/", + inputPath: "path%2Fwith%2Fencoded%2Fslashes.txt", + decodeObjectKey: true, + expectedPath: "prefix/path/with/encoded/slashes.txt", + }, + { + name: "no prefix with URL encoded path", + keyPrefix: "", + inputPath: "some%20file%20name.txt", + decodeObjectKey: true, + expectedPath: "some file name.txt", + }, + { + name: "prefix and decode with spaces", + keyPrefix: "s3-data/", + inputPath: "My%20Documents%2Ffile.pdf", + decodeObjectKey: true, + expectedPath: "s3-data/My Documents/file.pdf", + }, + { + name: "empty path with prefix", + keyPrefix: "prefix/", + inputPath: "", + decodeObjectKey: false, + expectedPath: "prefix/", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Simulate the path processing logic from processFiles + path := tt.inputPath + + // URL decode if configured + if tt.decodeObjectKey { + var err error + path, err = url.QueryUnescape(path) + if tt.expectDecodeErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + } + + // Apply key prefix if configured + if tt.keyPrefix != "" { + path = tt.keyPrefix + path + } + + assert.Equal(t, tt.expectedPath, path) + }) + } +} + +func TestKeyPrefixWithRealConfig(t *testing.T) { + configJSON := `{ + "client_options": { + "hostname": "test.com", + "oid": "test-oid", + "installation_key": "test-key" + }, + "access_key": "test-access", + "secret_key": "test-secret", + "region": "us-east-1", + "queue_url": "https://sqs.us-east-1.amazonaws.com/123456789/test", + "key_prefix": "s3-backup/", + "bucket": "my-bucket" + }` + + var conf SQSFilesConfig + err := json.Unmarshal([]byte(configJSON), &conf) + require.NoError(t, err) + + // Verify key_prefix and bucket are correctly unmarshaled + assert.Equal(t, "s3-backup/", conf.KeyPrefix) + assert.Equal(t, "my-bucket", conf.Bucket) +} + +func TestKeyPrefixYAMLUnmarshal(t *testing.T) { + // Note: This test requires the yaml package if you want to test YAML unmarshaling + // For now, we'll just verify the struct tags are correct by checking JSON + configJSON := `{ + "client_options": { + "hostname": "test.com", + "oid": "test-oid", + "installation_key": "test-key" + }, + "access_key": "test-access", + "secret_key": "test-secret", + "region": "us-east-1", + "queue_url": "https://sqs.us-east-1.amazonaws.com/123456789/test", + "key_prefix": "production/logs/", + "bucket": "my-s3-bucket", + "is_decode_object_key": true + }` + + var conf SQSFilesConfig + err := json.Unmarshal([]byte(configJSON), &conf) + require.NoError(t, err) + + assert.Equal(t, "production/logs/", conf.KeyPrefix) + assert.Equal(t, "my-s3-bucket", conf.Bucket) + assert.True(t, conf.IsDecodeObjectKey) +} + +func TestKeyPrefixEdgeCases(t *testing.T) { + tests := []struct { + name string + prefix string + path string + expected string + }{ + { + name: "special characters in prefix", + prefix: "data-2024_01/", + path: "file.log", + expected: "data-2024_01/file.log", + }, + { + name: "prefix with dots", + prefix: "v1.0/", + path: "config.json", + expected: "v1.0/config.json", + }, + { + name: "very long prefix", + prefix: "level1/level2/level3/level4/level5/", + path: "deep/file.txt", + expected: "level1/level2/level3/level4/level5/deep/file.txt", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := tt.path + if tt.prefix != "" { + path = tt.prefix + path + } + assert.Equal(t, tt.expected, path) + }) + } +} From 17dfae268e6bd78be5817dcd7eba535116d1d223 Mon Sep 17 00:00:00 2001 From: Maxime Lamothe-Brassard Date: Fri, 14 Nov 2025 12:13:59 -0800 Subject: [PATCH 3/3] Delete sqs-files/client_test.go Not useful tests. Super basic anyway. --- sqs-files/client_test.go | 259 --------------------------------------- 1 file changed, 259 deletions(-) delete mode 100644 sqs-files/client_test.go diff --git a/sqs-files/client_test.go b/sqs-files/client_test.go deleted file mode 100644 index d7c3523..0000000 --- a/sqs-files/client_test.go +++ /dev/null @@ -1,259 +0,0 @@ -package usp_sqs_files - -import ( - "encoding/json" - "net/url" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSQSFilesConfig_KeyPrefix(t *testing.T) { - tests := []struct { - name string - configJSON string - wantPrefix string - }{ - { - name: "config with key_prefix", - configJSON: `{ - "client_options": { - "hostname": "test.com", - "oid": "test-oid", - "installation_key": "test-key" - }, - "access_key": "test-access", - "secret_key": "test-secret", - "region": "us-east-1", - "queue_url": "https://sqs.us-east-1.amazonaws.com/123456789/test", - "key_prefix": "logs/backup/" - }`, - wantPrefix: "logs/backup/", - }, - { - name: "config without key_prefix", - configJSON: `{ - "client_options": { - "hostname": "test.com", - "oid": "test-oid", - "installation_key": "test-key" - }, - "access_key": "test-access", - "secret_key": "test-secret", - "region": "us-east-1", - "queue_url": "https://sqs.us-east-1.amazonaws.com/123456789/test" - }`, - wantPrefix: "", - }, - { - name: "config with empty key_prefix", - configJSON: `{ - "client_options": { - "hostname": "test.com", - "oid": "test-oid", - "installation_key": "test-key" - }, - "access_key": "test-access", - "secret_key": "test-secret", - "region": "us-east-1", - "queue_url": "https://sqs.us-east-1.amazonaws.com/123456789/test", - "key_prefix": "" - }`, - wantPrefix: "", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var conf SQSFilesConfig - err := json.Unmarshal([]byte(tt.configJSON), &conf) - require.NoError(t, err) - - // Test that the key_prefix field is correctly unmarshaled - assert.Equal(t, tt.wantPrefix, conf.KeyPrefix) - }) - } -} - -func TestKeyPrefixApplication(t *testing.T) { - tests := []struct { - name string - keyPrefix string - inputPath string - decodeObjectKey bool - expectedPath string - expectDecodeErr bool - }{ - { - name: "no prefix", - keyPrefix: "", - inputPath: "data/file.json", - decodeObjectKey: false, - expectedPath: "data/file.json", - }, - { - name: "prefix with trailing slash", - keyPrefix: "backup/", - inputPath: "data/file.json", - decodeObjectKey: false, - expectedPath: "backup/data/file.json", - }, - { - name: "prefix without trailing slash", - keyPrefix: "logs", - inputPath: "data/file.json", - decodeObjectKey: false, - expectedPath: "logsdata/file.json", - }, - { - name: "multi-level prefix", - keyPrefix: "archive/2024/01/", - inputPath: "events.log", - decodeObjectKey: false, - expectedPath: "archive/2024/01/events.log", - }, - { - name: "prefix with URL encoded path", - keyPrefix: "prefix/", - inputPath: "path%2Fwith%2Fencoded%2Fslashes.txt", - decodeObjectKey: true, - expectedPath: "prefix/path/with/encoded/slashes.txt", - }, - { - name: "no prefix with URL encoded path", - keyPrefix: "", - inputPath: "some%20file%20name.txt", - decodeObjectKey: true, - expectedPath: "some file name.txt", - }, - { - name: "prefix and decode with spaces", - keyPrefix: "s3-data/", - inputPath: "My%20Documents%2Ffile.pdf", - decodeObjectKey: true, - expectedPath: "s3-data/My Documents/file.pdf", - }, - { - name: "empty path with prefix", - keyPrefix: "prefix/", - inputPath: "", - decodeObjectKey: false, - expectedPath: "prefix/", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Simulate the path processing logic from processFiles - path := tt.inputPath - - // URL decode if configured - if tt.decodeObjectKey { - var err error - path, err = url.QueryUnescape(path) - if tt.expectDecodeErr { - assert.Error(t, err) - return - } - require.NoError(t, err) - } - - // Apply key prefix if configured - if tt.keyPrefix != "" { - path = tt.keyPrefix + path - } - - assert.Equal(t, tt.expectedPath, path) - }) - } -} - -func TestKeyPrefixWithRealConfig(t *testing.T) { - configJSON := `{ - "client_options": { - "hostname": "test.com", - "oid": "test-oid", - "installation_key": "test-key" - }, - "access_key": "test-access", - "secret_key": "test-secret", - "region": "us-east-1", - "queue_url": "https://sqs.us-east-1.amazonaws.com/123456789/test", - "key_prefix": "s3-backup/", - "bucket": "my-bucket" - }` - - var conf SQSFilesConfig - err := json.Unmarshal([]byte(configJSON), &conf) - require.NoError(t, err) - - // Verify key_prefix and bucket are correctly unmarshaled - assert.Equal(t, "s3-backup/", conf.KeyPrefix) - assert.Equal(t, "my-bucket", conf.Bucket) -} - -func TestKeyPrefixYAMLUnmarshal(t *testing.T) { - // Note: This test requires the yaml package if you want to test YAML unmarshaling - // For now, we'll just verify the struct tags are correct by checking JSON - configJSON := `{ - "client_options": { - "hostname": "test.com", - "oid": "test-oid", - "installation_key": "test-key" - }, - "access_key": "test-access", - "secret_key": "test-secret", - "region": "us-east-1", - "queue_url": "https://sqs.us-east-1.amazonaws.com/123456789/test", - "key_prefix": "production/logs/", - "bucket": "my-s3-bucket", - "is_decode_object_key": true - }` - - var conf SQSFilesConfig - err := json.Unmarshal([]byte(configJSON), &conf) - require.NoError(t, err) - - assert.Equal(t, "production/logs/", conf.KeyPrefix) - assert.Equal(t, "my-s3-bucket", conf.Bucket) - assert.True(t, conf.IsDecodeObjectKey) -} - -func TestKeyPrefixEdgeCases(t *testing.T) { - tests := []struct { - name string - prefix string - path string - expected string - }{ - { - name: "special characters in prefix", - prefix: "data-2024_01/", - path: "file.log", - expected: "data-2024_01/file.log", - }, - { - name: "prefix with dots", - prefix: "v1.0/", - path: "config.json", - expected: "v1.0/config.json", - }, - { - name: "very long prefix", - prefix: "level1/level2/level3/level4/level5/", - path: "deep/file.txt", - expected: "level1/level2/level3/level4/level5/deep/file.txt", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - path := tt.path - if tt.prefix != "" { - path = tt.prefix + path - } - assert.Equal(t, tt.expected, path) - }) - } -}