TT-17702: Add Lister and Setter logic to Vault and Consul

kv/internal/resolve/refs.go

@@ -12,6 +12,13 @@ type refKey struct {
 	fragment string
 }
 
+// ParseWholeValue is the exported boundary over parseWholeValue for callers
+// outside the resolve engine that must parse a kv:// reference without resolving it.
+func ParseWholeValue(input string) (store, path, fragment string, ok bool, err error) {
+	rk, ok, err := parseWholeValue(input)
+	return rk.store, rk.path, rk.fragment, ok, err
+}
+
 // parseWholeValue parses a whole-value reference of the form
 // "kv://store/path#frag".
 //

kv/internal/store/store.go

@@ -12,8 +12,6 @@ import (
 	"golang.org/x/sync/singleflight"
 )
 
-const defaultProviderTimeout = 5 * time.Second
-
 // SecretStore is an internal decorator that adds caching and singleflight to a Provider.
 type SecretStore struct {
 	name      string
@@ -169,7 +167,7 @@ func NewSecretStore(
 		cache:     cache,
 		sf:        &singleflight.Group{},
 		sfRefresh: &singleflight.Group{},
-		timeout:   defaultProviderTimeout,
+		timeout:   kv.DefaultOperationTimeout,
 	}
 
 	for _, opt := range opts {

kv/internal/store/store_test.go

@@ -82,7 +82,7 @@
 		})
 		require.NoError(t, err)
 		require.NotNil(t, store)
-		require.Equal(t, defaultProviderTimeout, store.timeout)
+		require.Equal(t, kv.DefaultOperationTimeout, store.timeout)
 	})
 
 	t.Run("cache disabled", func(t *testing.T) {
@@ -671,7 +671,7 @@
 // BenchmarkSecretStoreGet measures the per-request cost of a Get through the
 // SecretStore wrapper — the "no measurable per-request latency when values are
 // cache-resident" acceptance criterion.
 func BenchmarkSecretStoreGet(b *testing.B) {
 	ctx := context.Background()
 
 	b.Run("cache-hit", func(b *testing.B) {

kv/provider.go

@@ -42,6 +42,10 @@
 	Conjur ProviderType = "cyberark_conjur"
 )
 
+// DefaultOperationTimeout bounds a single provider Get/Set when neither the store
+// config nor the SecretStore wrapper supplies one.
+const DefaultOperationTimeout = 5 * time.Second
+
 // IsLocal reports whether this provider type resolves secrets from resources
 // available to the local process — environment variables, inline config data,
 // or the filesystem — requiring no network and a literal, reference-free config.
@@ -101,11 +105,17 @@
 	Init(ctx context.Context) error
 }
 
+// Setter is an optional interface for providers that support writing values
+// back to their backend.
+type Setter interface {
+	Set(ctx context.Context, key, value string) error
+}
+
 // Lister is an optional interface for providers that support enumerating
-// keys by prefix. This enables dynamic discovery of available secrets
+// keys & values by prefix. This enables dynamic discovery of available secrets
 // and operational tooling.
 type Lister interface {
-	List(ctx context.Context, prefix string) ([]string, error)
+	List(ctx context.Context, prefix string) (map[string]string, error)
 }
 
 // Closer is an optional interface for providers that need graceful shutdown
@@ -116,7 +126,7 @@
 
 // Standalone is an optional interface for providers that do not need
 // to be combined with caching or singleflight mechanisms.
 type Standalone interface {
 	IsStandalone() bool
 }
 
@@ -126,6 +136,12 @@
 	Timeout() time.Duration
 }
 
+// AsSetter attempts to extract a Setter from a Provider,
+// automatically unwrapping decorators.
+func AsSetter(p Provider) (Setter, bool) {
+	return As[Setter](p)
+}
+
 // AsLister attempts to extract a Lister from a Provider,
 // automatically unwrapping decorators.
 func AsLister(p Provider) (Lister, bool) {
@@ -179,3 +195,13 @@
 
 	return zero, false
 }
+
+// EffectiveTimeout resolves a configured timeout to the value actually used:
+// the configured value when positive, else the default.
+func EffectiveTimeout(configured time.Duration) time.Duration {
+	if configured > 0 {
+		return configured
+	}
+
+	return DefaultOperationTimeout
+}

kv/providers/consul/consul.go

@@ -10,11 +10,13 @@
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/TykTechnologies/storage/kv"
-	"github.com/hashicorp/consul/api"
+	consulsdk "github.com/hashicorp/consul/api"
 )
 
 // Config is the JSON "config" block of a consul store.
@@ -62,7 +64,7 @@
 // JSON, an unparseable wait_time, or TLS settings the client rejects (e.g. an
 // unreadable ca_file). Absent config is valid — an empty blob builds a client
 // against consul's default local agent.
 func NewFactory() kv.ProviderFactory {
 	return func(rawJSON json.RawMessage) (kv.Provider, error) {
 		var conf Config
 
@@ -74,7 +76,7 @@
 			}
 		}
 
-		clientCfg := api.DefaultConfig()
+		clientCfg := consulsdk.DefaultConfig()
 
 		if conf.Address != "" {
 			clientCfg.Address = conf.Address
@@ -89,7 +91,7 @@
 		}
 
 		if conf.HttpAuth.Username != "" || conf.HttpAuth.Password != "" {
-			clientCfg.HttpAuth = &api.HttpBasicAuth{
+			clientCfg.HttpAuth = &consulsdk.HttpBasicAuth{
 				Username: conf.HttpAuth.Username,
 				Password: conf.HttpAuth.Password,
 			}
@@ -112,7 +114,7 @@
 
 		applyTLSConfig(clientCfg, &conf)
 
-		client, err := api.NewClient(clientCfg)
+		client, err := consulsdk.NewClient(clientCfg)
 		if err != nil {
 			return nil, fmt.Errorf("consul: create client: %w", err)
 		}
@@ -121,7 +123,7 @@
 	}
 }
 
-func applyTLSConfig(clientCfg *api.Config, conf *Config) {
+func applyTLSConfig(clientCfg *consulsdk.Config, conf *Config) {
 	tls := conf.TLSConfig
 
 	if tls.Address != "" {
@@ -153,7 +155,7 @@
 type consulProvider struct {
 	// kvClient is consul's KV endpoint, with the resolved Config already baked
 	// into the underlying client.
-	kvClient *api.KV
+	kvClient *consulsdk.KV
 }
 
 // Get reads the value at key and returns it verbatim: no trimming, no key
@@ -164,7 +166,7 @@
 // distinguishes. ctx bounds the request via QueryOptions, so the SecretStore's
 // per-operation deadline is honored.
 func (cp *consulProvider) Get(ctx context.Context, key string) (string, error) {
-	pair, _, err := cp.kvClient.Get(key, (&api.QueryOptions{}).WithContext(ctx))
+	pair, _, err := cp.kvClient.Get(key, (&consulsdk.QueryOptions{}).WithContext(ctx))
 	if err != nil {
 		return "", &kv.StoreUnavailableError{KeyPath: key, Err: err}
 	}
@@ -175,3 +177,56 @@
 
 	return string(pair.Value), nil
 }
+
+// Set writes value verbatim as the raw bytes at key (PUT /v1/kv/<key>), with no
+// key transformation or interpretation.
+// A transport or backend failure returns *kv.StoreUnavailableError.
+func (cp *consulProvider) Set(ctx context.Context, key, value string) error {
+	pair := &consulsdk.KVPair{
+		Key:   key,
+		Value: []byte(value),
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, kv.DefaultOperationTimeout)
+	defer cancel()
+
+	_, err := cp.kvClient.Put(pair, (&consulsdk.WriteOptions{}).WithContext(ctx))
+	if err != nil {
+		return &kv.StoreUnavailableError{KeyPath: key, Err: err}
+	}
+
+	return nil
+}
+
+// List returns every key/value pair under prefix, keyed by the FULL consul key
+// (the caller strips the prefix if it wants relative keys). Consul directory
+// markers — keys ending in "/" — are skipped; they are not real entries.
+//
+// An empty prefix is rejected: consul would treat it as "list the entire KV
+// store", which is never what a reference resolver wants and is an easy footgun.
+// A prefix that matches nothing is not an error — it returns an empty map.
+func (cp *consulProvider) List(ctx context.Context, prefix string) (map[string]string, error) {
+	if prefix == "" {
+		return nil, errors.New("consul: list requires a non-empty prefix")
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, kv.DefaultOperationTimeout)
+	defer cancel()
+
+	pairs, _, err := cp.kvClient.List(prefix, (&consulsdk.QueryOptions{}).WithContext(ctx))
+	if err != nil {
+		return nil, &kv.StoreUnavailableError{KeyPath: prefix, Err: err}
+	}
+
+	out := make(map[string]string, len(pairs))
+
+	for _, p := range pairs {
+		if strings.HasSuffix(p.Key, "/") {
+			continue
+		}
+
+		out[p.Key] = string(p.Value)
+	}
+
+	return out, nil
+}