Skip to content

Improve SQS-Files adapter error logging with available message keys #239

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
maximelb wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Improve SQS-Files adapter error logging with available message keys #239

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b5764d2
Improve SQS-Files adapter error logging with available message keys
maximelb Nov 1, 2025
9da9f00
Merge branch 'master' into sqs-files-find
maximelb Nov 13, 2025
8208b06
Add key_prefix parameter to sqs-files adapter
maximelb Nov 13, 2025
17dfae2
Delete sqs-files/client_test.go
maximelb Nov 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion sqs-files/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ type SQSFilesConfig struct {
BucketPath string `json:"bucket_path,omitempty" yaml:"bucket_path,omitempty"`
FilePath string `json:"file_path,omitempty" yaml:"file_path,omitempty"`
IsDecodeObjectKey bool `json:"is_decode_object_key,omitempty" yaml:"is_decode_object_key,omitempty"`
KeyPrefix string `json:"key_prefix,omitempty" yaml:"key_prefix,omitempty"`
// Optional: alternative to BucketPath
Bucket string `json:"bucket,omitempty" yaml:"bucket,omitempty"`
}
Expand Down Expand Up @@ -196,6 +197,16 @@ func (a *SQSFilesAdapter) getBucketRegion(bucket string) (string, error) {
return s3manager.GetBucketRegion(a.ctx, session.Must(session.NewSession(&aws.Config{})), bucket, "us-east-1")
}

// getAvailableKeys returns a list of top-level keys from a Dict
// for use in error messages to help users debug configuration issues
func getAvailableKeys(d utils.Dict) []string {
keys := make([]string, 0, len(d))
for k := range d {
keys = append(keys, k)
}
return keys
}

func (a *SQSFilesAdapter) receiveEvents() error {
defer close(a.chFiles)

Expand Down Expand Up @@ -238,10 +249,13 @@ func (a *SQSFilesAdapter) receiveEvents() error {
filePaths := d.ExpandableFindString(a.conf.FilePath)

if bucket == "" {
a.conf.ClientOptions.OnError(errors.New("sqsClient.Message: missing bucket"))
availableKeys := getAvailableKeys(d)
a.conf.ClientOptions.OnError(fmt.Errorf("sqsClient.Message: missing bucket - attempted path '%s', available keys in message: %v", a.conf.BucketPath, availableKeys))
continue
}
if len(filePaths) == 0 {
availableKeys := getAvailableKeys(d)
a.conf.ClientOptions.OnError(fmt.Errorf("sqsClient.Message: missing file paths - attempted path '%s', available keys in message: %v", a.conf.FilePath, availableKeys))
continue
}
if err := a.initS3SDKs(bucket); err != nil {
Expand Down Expand Up @@ -280,6 +294,10 @@ func (a *SQSFilesAdapter) processFiles() error {
continue
}
}
// Apply key prefix if configured
if a.conf.KeyPrefix != "" {
path = a.conf.KeyPrefix + path
}
startTime := time.Now().UTC()
a.conf.ClientOptions.DebugLog(fmt.Sprintf("downloading file %s", path))

Expand Down