TT-17102: Add features to support gateway wiring

go.mod

@@ -4,11 +4,14 @@ go 1.25.0
 
 require (
 	github.com/google/go-cmp v0.7.0
+	github.com/hashicorp/consul/api v1.31.2
+	github.com/hashicorp/vault/api v1.23.0
 	github.com/lib/pq v1.10.9
 	github.com/redis/go-redis/v9 v9.11.0
 	github.com/stretchr/testify v1.11.1
 	go.mongodb.org/mongo-driver v1.17.7
 	golang.org/x/oauth2 v0.36.0
+	golang.org/x/sync v0.21.0
 	golang.org/x/text v0.38.0
 	google.golang.org/api v0.287.1
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
@@ -30,19 +33,41 @@ require (
 	cloud.google.com/go/auth v0.20.0 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect
 	github.com/googleapis/gax-go/v2 v2.22.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-hclog v1.6.3 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
+	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.7 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
+	github.com/hashicorp/serf v0.10.1 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
@@ -54,9 +79,10 @@ require (
 	go.opentelemetry.io/otel/metric v1.43.0 // indirect
 	go.opentelemetry.io/otel/trace v1.43.0 // indirect
 	golang.org/x/crypto v0.53.0 // indirect
+	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect
 	golang.org/x/net v0.56.0 // indirect
-	golang.org/x/sync v0.21.0 // indirect
 	golang.org/x/sys v0.46.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect
 	google.golang.org/grpc v1.82.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect

kv/context.go

@@ -0,0 +1,24 @@
+package kv
+
+import "context"
+
+type cacheBypassKey struct{}
+
+// WithCacheBypass returns a context that instructs SecretStore.Get to skip
+// the cache read, fetch from the backend, and re-populate the cache entry.
+// It is context-carried because the call path (Resolver.Resolve →
+// Registry.GetStore → Provider.Get) has fixed signatures with no room for
+// an options parameter.
+func WithCacheBypass(ctx context.Context) context.Context {
+	return context.WithValue(ctx, cacheBypassKey{}, true)
+}
+
+// IsCacheBypassed reports whether context carries the cache-bypass directive.
+func IsCacheBypassed(ctx context.Context) bool {
+	bypassed, ok := ctx.Value(cacheBypassKey{}).(bool)
+	if !ok {
+		return false
+	}
+
+	return bypassed
+}

kv/errors.go

@@ -29,6 +29,10 @@ type KeyNotFoundError struct {
 }
 
 func (e *KeyNotFoundError) Error() string {
+	if e.StoreName == "" {
+		return fmt.Sprintf("key %q not found", e.KeyPath)
+	}
+
 	return fmt.Sprintf("key %q not found in store %q", e.KeyPath, e.StoreName)
 }
 

kv/errors_test.go

@@ -0,0 +1,39 @@
+package kv_test
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/TykTechnologies/storage/kv"
+	"github.com/stretchr/testify/require"
+)
+
+func TestKeyNotFoundError_Error(t *testing.T) {
+	t.Parallel()
+
+	t.Run("omits the store clause when StoreName is empty", func(t *testing.T) {
+		t.Parallel()
+
+		// Standalone providers (e.g. file) cannot know their registry name,
+		// so they leave StoreName empty; the message must still read cleanly.
+		err := &kv.KeyNotFoundError{KeyPath: "db/password"}
+		require.Equal(t, `key "db/password" not found`, err.Error())
+	})
+
+	t.Run("includes the store name when present", func(t *testing.T) {
+		t.Parallel()
+
+		err := &kv.KeyNotFoundError{StoreName: "vault", KeyPath: "db/password"}
+		require.Equal(t, `key "db/password" not found in store "vault"`, err.Error())
+	})
+}
+
+func TestKeyNotFoundError_IsMatchable(t *testing.T) {
+	t.Parallel()
+
+	wrapped := errors.Join(errors.New("context"), &kv.KeyNotFoundError{KeyPath: "k"})
+
+	var notFound *kv.KeyNotFoundError
+	require.ErrorAs(t, wrapped, &notFound)
+	require.Equal(t, "k", notFound.KeyPath)
+}

kv/internal/resolve/lenient_test.go

@@ -0,0 +1,177 @@
+package resolve_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/TykTechnologies/storage/kv"
+	"github.com/TykTechnologies/storage/kv/internal/resolve"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLenientResolve(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		stores  map[string]kv.Provider
+		input   string
+		want    string
+		wantErr error
+	}{
+		{
+			name:  "whole-value reference to absent store returned unchanged",
+			input: "kv://vault/db/creds#password",
+			want:  "kv://vault/db/creds#password",
+		},
+		{
+			name:   "inline token to absent store left in place",
+			stores: map[string]kv.Provider{"env": &mockProvider{value: "myhost.com"}},
+			input:  "https://$kv{env:API_HOST}/$kv{vault:db/creds#password}",
+			want:   "https://myhost.com/$kv{vault:db/creds#password}",
+		},
+		{
+			name:  "multiple inline tokens to absent stores all left in place",
+			input: "$kv{vault:a}/$kv{consul:b}",
+			want:  "$kv{vault:a}/$kv{consul:b}",
+		},
+		{
+			name:   "fully resolvable input resolves exactly as in strict mode",
+			stores: map[string]kv.Provider{"env": &mockProvider{value: "s3cr3t"}},
+			input:  "kv://env/TOKEN",
+			want:   "s3cr3t",
+		},
+		{
+			name:  "plain string without references passes through",
+			input: "no references here",
+			want:  "no references here",
+		},
+		{
+			name: "present store with missing key still errors",
+			stores: map[string]kv.Provider{
+				"env": &mockProvider{err: &kv.KeyNotFoundError{StoreName: "env", KeyPath: "MISSING"}},
+			},
+			input:   "kv://env/MISSING",
+			wantErr: &kv.KeyNotFoundError{},
+		},
+		{
+			name:    "malformed whole-value reference still errors",
+			input:   "kv://no-path-separator",
+			wantErr: resolve.ErrMalformedReference,
+		},
+		{
+			name:    "malformed inline token still errors",
+			input:   "prefix-$kv{missing-store-separator}-suffix",
+			wantErr: resolve.ErrMalformedReference,
+		},
+		{
+			name:   "missing JSON field on a reachable store still errors",
+			stores: map[string]kv.Provider{"env": &mockProvider{value: `{"user":"admin"}`}},
+			input:  "kv://env/creds#missing_field",
+
+			wantErr: resolve.ErrFieldNotFound,
+		},
+		{
+			name: "absent-store token coexists with an error on a reachable store",
+			stores: map[string]kv.Provider{
+				"env": &mockProvider{err: &kv.KeyNotFoundError{StoreName: "env", KeyPath: "A"}},
+			},
+			input:   "$kv{env:A}/$kv{vault:b}",
+			wantErr: &kv.KeyNotFoundError{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			r := resolve.NewResolver(newGetter(tt.stores), resolve.WithLenientMode())
+
+			got, err := r.Resolve(t.Context(), tt.input)
+
+			if tt.wantErr != nil {
+				var keyNotFound *kv.KeyNotFoundError
+				if errors.As(tt.wantErr, &keyNotFound) {
+					require.ErrorAs(t, err, &keyNotFound)
+				} else {
+					require.ErrorIs(t, err, tt.wantErr)
+				}
+
+				return
+			}
+
+			require.NoError(t, err)
+			require.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestLenientResolveAll(t *testing.T) {
+	t.Parallel()
+
+	t.Run("absent-store references pass through, resolvable ones resolve", func(t *testing.T) {
+		t.Parallel()
+
+		r := resolve.NewResolver(
+			newGetter(map[string]kv.Provider{"env": &mockProvider{value: "hvs.secret-token"}}),
+			resolve.WithLenientMode(),
+		)
+
+		doc := []byte(`{
+			"token": "kv://env/VAULT_TOKEN",
+			"credentials": "kv://vault/aws-creds",
+			"url": "https://$kv{env:VAULT_TOKEN}@$kv{consul:host}"
+		}`)
+
+		got, err := r.ResolveAll(context.Background(), doc)
+		require.NoError(t, err)
+		require.JSONEq(t, `{
+			"token": "hvs.secret-token",
+			"credentials": "kv://vault/aws-creds",
+			"url": "https://hvs.secret-token@$kv{consul:host}"
+		}`, string(got))
+	})
+
+	t.Run("malformed reference inside the document still errors", func(t *testing.T) {
+		t.Parallel()
+
+		r := resolve.NewResolver(newGetter(nil), resolve.WithLenientMode())
+
+		_, err := r.ResolveAll(context.Background(), []byte(`{"bad": "kv://no-path-separator"}`))
+		require.ErrorIs(t, err, resolve.ErrMalformedReference)
+	})
+
+	t.Run("invalid JSON still errors", func(t *testing.T) {
+		t.Parallel()
+
+		r := resolve.NewResolver(newGetter(nil), resolve.WithLenientMode())
+
+		_, err := r.ResolveAll(context.Background(), []byte(`{not json`))
+		require.ErrorIs(t, err, resolve.ErrInvalidJSON)
+	})
+}
+
+// Strict mode is the default and must be completely unaffected by the
+// existence of the lenient option.
+func TestStrictModeRemainsDefault(t *testing.T) {
+	t.Parallel()
+
+	t.Run("whole-value reference to absent store errors", func(t *testing.T) {
+		t.Parallel()
+
+		r := resolve.NewResolver(newGetter(nil))
+
+		_, err := r.Resolve(context.Background(), "kv://vault/db/creds")
+		require.ErrorIs(t, err, kv.ErrStoreNotFound)
+	})
+
+	t.Run("inline token to absent store errors", func(t *testing.T) {
+		t.Parallel()
+
+		r := resolve.NewResolver(newGetter(nil))
+
+		_, err := r.Resolve(context.Background(), "https://$kv{vault:host}/v1")
+		require.ErrorIs(t, err, kv.ErrStoreNotFound)
+	})
+}