From 05d567e3784e0f4c94d26ccfbf54e200a92e87d9 Mon Sep 17 00:00:00 2001 From: razzkumar Date: Fri, 27 Mar 2026 16:42:19 +0545 Subject: [PATCH] feat: support recursive directory secret export Add recursive dir handling for config-backed secret exports, validate recursive usage, cover the behavior with tests, and harden bulk-put counter increments in bash. --- internal/app/config.go | 85 +++++++++++++++++++++++-- internal/app/config_test.go | 123 ++++++++++++++++++++++++++++++++++++ pkg/config/config.go | 8 ++- pkg/config/config_test.go | 11 ++++ scripts/bulk-put.sh | 4 +- 5 files changed, 222 insertions(+), 9 deletions(-) diff --git a/internal/app/config.go b/internal/app/config.go index 5deed53..683ef9b 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -274,24 +274,91 @@ func (a *App) handleFileEntry(cfg *config.Config, secret *config.SecretEntry, kv // handleDirEntry processes a directory entry from the config and saves all keys as individual files func (a *App) handleDirEntry(cfg *config.Config, secret *config.SecretEntry, kvMount, transitMount, encryptionKey string) error { + mount := config.NonEmpty("", cfg.KV.Mount, kvMount) + + if secret.Recursive { + return a.handleDirEntryRecursive(cfg, secret, mount, transitMount, encryptionKey) + } + // Get all data from the Vault path - data, err := a.vaultClient.KVGet(config.NonEmpty("", cfg.KV.Mount, kvMount), secret.Path) + data, err := a.vaultClient.KVGet(mount, secret.Path) if err != nil { return fmt.Errorf("failed to get secrets from path %s: %w", secret.Path, err) } + return a.saveDirKeys(cfg, secret, secret.Path, "", data, transitMount, encryptionKey) +} + +// handleDirEntryRecursive walks the Vault tree under secret.Path using KVList +// and saves every leaf secret into the target directory, preserving the sub-path structure. +func (a *App) handleDirEntryRecursive(cfg *config.Config, secret *config.SecretEntry, mount, transitMount, encryptionKey string) error { + leafPaths, err := a.listRecursive(mount, secret.Path) + if err != nil { + return fmt.Errorf("failed to list secrets recursively under %s: %w", secret.Path, err) + } + + if len(leafPaths) == 0 { + return fmt.Errorf("no secrets found recursively under path %s", secret.Path) + } + + for _, leafPath := range leafPaths { + data, err := a.vaultClient.KVGet(mount, leafPath) + if err != nil { + return fmt.Errorf("failed to get secret at %s: %w", leafPath, err) + } + + // Compute the relative sub-path from the base to build the directory structure + relPath := strings.TrimPrefix(leafPath, secret.Path) + relPath = strings.TrimPrefix(relPath, "/") + + if err := a.saveDirKeys(cfg, secret, leafPath, relPath, data, transitMount, encryptionKey); err != nil { + return err + } + } + + return nil +} + +// listRecursive walks Vault KVList from basePath and returns all leaf (non-directory) paths. +func (a *App) listRecursive(mount, basePath string) ([]string, error) { + entries, err := a.vaultClient.KVList(mount, basePath) + if err != nil { + return nil, err + } + + var leaves []string + for _, entry := range entries { + fullPath := basePath + "/" + strings.TrimSuffix(entry, "/") + if strings.HasSuffix(entry, "/") { + // It's a sub-directory — recurse + subLeaves, err := a.listRecursive(mount, fullPath) + if err != nil { + return nil, err + } + leaves = append(leaves, subLeaves...) + } else { + // It's a leaf secret + leaves = append(leaves, fullPath) + } + } + return leaves, nil +} + +// saveDirKeys extracts keys from Vault data and saves each as a file. +// relPath is the relative sub-path (empty for non-recursive) used to create subdirectories. +func (a *App) saveDirKeys(cfg *config.Config, secret *config.SecretEntry, vaultPath, relPath string, data map[string]interface{}, transitMount, encryptionKey string) error { var keysToSave map[string]interface{} // Handle encrypted multi-value data if utils.IsEncryptedMultiValue(data) { encKeyForDecrypt := config.NonEmpty(encryptionKey, cfg.GetTransitKey(), "") if encKeyForDecrypt == "" { - return fmt.Errorf("encryption key required for encrypted secrets at path %s", secret.Path) + return fmt.Errorf("encryption key required for encrypted secrets at path %s", vaultPath) } decryptedData, err := utils.DecryptMultiValueData(data, a.vaultClient, cfg.GetTransitMount(transitMount), encKeyForDecrypt) if err != nil { - return fmt.Errorf("failed to decrypt secrets from path %s: %w", secret.Path, err) + return fmt.Errorf("failed to decrypt secrets from path %s: %w", vaultPath, err) } keysToSave = decryptedData } else { @@ -308,7 +375,7 @@ func (a *App) handleDirEntry(cfg *config.Config, secret *config.SecretEntry, kvM if len(keysToSave) == 0 { if value, ok := data["value"]; ok { // Use path name as filename - pathParts := strings.Split(secret.Path, "/") + pathParts := strings.Split(vaultPath, "/") keyName := pathParts[len(pathParts)-1] keysToSave[keyName] = value } @@ -316,7 +383,7 @@ func (a *App) handleDirEntry(cfg *config.Config, secret *config.SecretEntry, kvM } if len(keysToSave) == 0 { - return fmt.Errorf("no valid secrets found at path %s", secret.Path) + return fmt.Errorf("no valid secrets found at path %s", vaultPath) } // Save each key as a file in the directory @@ -335,8 +402,14 @@ func (a *App) handleDirEntry(cfg *config.Config, secret *config.SecretEntry, kvM continue // Skip this key as it's handled by its own file configuration } + // Build the file name including the sub-path for recursive entries + fileName := keyName + if relPath != "" { + fileName = filepath.Join(relPath, keyName) + } + // Get directory file configuration for this key - fileConfig, err := cfg.GetDirFileConfig(secret, keyName) + fileConfig, err := cfg.GetDirFileConfig(secret, fileName) if err != nil { return fmt.Errorf("invalid file path for key %s: %w", keyName, err) } diff --git a/internal/app/config_test.go b/internal/app/config_test.go index 3916c96..d847095 100644 --- a/internal/app/config_test.go +++ b/internal/app/config_test.go @@ -375,6 +375,129 @@ secrets: } } +func TestHandleDirEntry_Recursive(t *testing.T) { + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "config.yaml") + outputDir := filepath.Join(tmpDir, "gpg-keys") + + configContent := `version: 1 +kv: + mount: kv +secrets: + - path: creds/gpg + dir: ` + outputDir + ` + recursive: true +` + if err := os.WriteFile(configFile, []byte(configContent), 0644); err != nil { + t.Fatalf("failed to create config file: %v", err) + } + + mock := vault.NewMockClient() + // Set up a tree of secrets under creds/gpg + mock.SetSecret("kv", "creds/gpg/public-key", map[string]interface{}{ + "data": "PUBLIC_KEY_CONTENT", + }) + mock.SetSecret("kv", "creds/gpg/private-key", map[string]interface{}{ + "data": "PRIVATE_KEY_CONTENT", + }) + mock.SetSecret("kv", "creds/gpg/subring/trust", map[string]interface{}{ + "data": "TRUST_CONTENT", + }) + + app := NewWithClient(mock) + + // Capture stderr (GenerateEnvFile prints a message) + oldStderr := os.Stderr + _, w, _ := os.Pipe() + os.Stderr = w + + err := app.GenerateEnvFile(configFile, filepath.Join(tmpDir, "output.env"), "") + + w.Close() + os.Stderr = oldStderr + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Verify KVList was called (recursive listing) + if len(mock.KVListCalls) == 0 { + t.Error("expected KVList to be called for recursive listing") + } + + // Verify files were created with correct content + tests := []struct { + path string + content string + }{ + {filepath.Join(outputDir, "public-key", "data"), "PUBLIC_KEY_CONTENT"}, + {filepath.Join(outputDir, "private-key", "data"), "PRIVATE_KEY_CONTENT"}, + {filepath.Join(outputDir, "subring", "trust", "data"), "TRUST_CONTENT"}, + } + + for _, tt := range tests { + content, err := os.ReadFile(tt.path) + if err != nil { + t.Errorf("failed to read file %s: %v", tt.path, err) + continue + } + if string(content) != tt.content { + t.Errorf("file %s: expected %q, got %q", tt.path, tt.content, string(content)) + } + } +} + +func TestHandleDirEntry_NonRecursive(t *testing.T) { + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "config.yaml") + outputDir := filepath.Join(tmpDir, "certs") + + configContent := `version: 1 +kv: + mount: kv +secrets: + - path: app/certs + dir: ` + outputDir + ` +` + if err := os.WriteFile(configFile, []byte(configContent), 0644); err != nil { + t.Fatalf("failed to create config file: %v", err) + } + + mock := vault.NewMockClient() + mock.SetSecret("kv", "app/certs", map[string]interface{}{ + "tls_crt": "CERT_DATA", + "tls_key": "KEY_DATA", + }) + + app := NewWithClient(mock) + + oldStderr := os.Stderr + _, w, _ := os.Pipe() + os.Stderr = w + + err := app.GenerateEnvFile(configFile, filepath.Join(tmpDir, "output.env"), "") + + w.Close() + os.Stderr = oldStderr + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // KVList should NOT be called for non-recursive + if len(mock.KVListCalls) != 0 { + t.Errorf("expected no KVList calls for non-recursive dir, got %d", len(mock.KVListCalls)) + } + + // Verify files + for _, name := range []string{"tls_crt", "tls_key"} { + path := filepath.Join(outputDir, name) + if _, err := os.Stat(path); os.IsNotExist(err) { + t.Errorf("expected file %s to exist", path) + } + } +} + func TestGetFromConfig(t *testing.T) { tmpDir := t.TempDir() configFile := filepath.Join(tmpDir, "config.yaml") diff --git a/pkg/config/config.go b/pkg/config/config.go index 0b78ad1..1e139f3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -74,7 +74,8 @@ type SecretEntry struct { File *SecretFileConfig `yaml:"file,omitempty"` // Directory configuration - when all keys should be saved as individual files - Dir string `yaml:"dir,omitempty"` // directory path to save all keys as individual files + Dir string `yaml:"dir,omitempty"` // directory path to save all keys as individual files + Recursive bool `yaml:"recursive,omitempty"` // recursively fetch sub-paths (requires dir) } // VaultConfig holds Vault client configuration @@ -584,6 +585,11 @@ func (s *SecretEntry) Validate() error { return fmt.Errorf("cannot specify both 'file' and 'dir'") } + // recursive requires dir + if s.Recursive && s.Dir == "" { + return fmt.Errorf("'recursive' requires 'dir' to be specified") + } + // Validate file mode if specified if s.File != nil && s.File.Mode != "" { if err := validateFileMode(s.File.Mode); err != nil { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 4ca4698..e3f281a 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1003,6 +1003,17 @@ func TestSecretEntry_Validate(t *testing.T) { wantErr: true, errMsg: "file.mode", }, + { + name: "valid recursive dir entry", + entry: SecretEntry{Path: "creds/gpg", Dir: "/tmp/gpg-keys", Recursive: true}, + wantErr: false, + }, + { + name: "recursive without dir", + entry: SecretEntry{Path: "creds/gpg", Recursive: true}, + wantErr: true, + errMsg: "'recursive' requires 'dir'", + }, } for _, tt := range tests { diff --git a/scripts/bulk-put.sh b/scripts/bulk-put.sh index 5d1916d..f8ca2b0 100755 --- a/scripts/bulk-put.sh +++ b/scripts/bulk-put.sh @@ -99,10 +99,10 @@ upload_file() { echo "Uploading: $file → $vault_path (key: $key)" if vlt put "${flags[@]}" --from-file "$file"; then - ((count++)) + ((count++)) || true else echo " FAILED: $file" - ((failed++)) + ((failed++)) || true fi }