diff --git a/go.mod b/go.mod index c323d464..7de72339 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,14 @@ go 1.25.0 require ( github.com/google/go-cmp v0.7.0 + github.com/hashicorp/consul/api v1.31.2 + github.com/hashicorp/vault/api v1.23.0 github.com/lib/pq v1.10.9 github.com/redis/go-redis/v9 v9.11.0 github.com/stretchr/testify v1.11.1 go.mongodb.org/mongo-driver v1.17.7 golang.org/x/oauth2 v0.36.0 + golang.org/x/sync v0.21.0 golang.org/x/text v0.38.0 google.golang.org/api v0.287.1 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 @@ -30,19 +33,41 @@ require ( cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect + github.com/armon/go-metrics v0.4.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/hcl v1.0.1-vault-7 // indirect + github.com/hashicorp/serf v0.10.1 // indirect github.com/klauspost/compress v1.16.7 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/montanaflynn/stats v0.7.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect @@ -54,9 +79,10 @@ require ( go.opentelemetry.io/otel/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect golang.org/x/crypto v0.53.0 // indirect + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect golang.org/x/net v0.56.0 // indirect - golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.46.0 // indirect + golang.org/x/time v0.15.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect google.golang.org/grpc v1.82.0 // indirect google.golang.org/protobuf v1.36.11 // indirect diff --git a/go.sum b/go.sum index a2ff4abe..306a03fb 100644 --- a/go.sum +++ b/go.sum @@ -4,33 +4,78 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/TykTechnologies/gorm v1.20.7-0.20260417144542-d6d1a2e12d9d h1:wePfwBWzjkC+lc/jEQsvcbQ3Rt0u+7MIj3jXWPnU6Qo= github.com/TykTechnologies/gorm v1.20.7-0.20260417144542-d6d1a2e12d9d/go.mod h1:hz0d/E0QBTYarOnYtdcNnBWN/NYxVMP7nZNDT6E/fFM= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= +github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -39,6 +84,60 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/hashicorp/consul/api v1.31.2 h1:NicObVJHcCmyOIl7Z9iHPvvFrocgTYo9cITSGg0/7pw= +github.com/hashicorp/consul/api v1.31.2/go.mod h1:Z8YgY0eVPukT/17ejW+l+C7zJmKwgPHtjU1q16v/Y40= +github.com/hashicorp/consul/sdk v0.16.1 h1:V8TxTnImoPD5cj0U9Spl0TUxcytjcbbJeADFF07KdHg= +github.com/hashicorp/consul/sdk v0.16.1/go.mod h1:fSXvwxB2hmh1FMZCNl6PwX0Q/1wdWtHJcZ7Ea5tns0s= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= +github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= +github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= +github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= +github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= +github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= +github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A= +github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko= github.com/helloeave/json v1.15.3 h1:roUxUEGhsSvhuhi80c4qmLiW633d5uf0mkzUGzBMfX8= github.com/helloeave/json v1.15.3/go.mod h1:uTHhuUsgnrpm9cc7Gi3tfIUwgf1dq/7+uLfpUFLBFEQ= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -57,8 +156,13 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -70,27 +174,89 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/redis/go-redis/v9 v9.11.0 h1:E3S08Gl/nJNn5vkxd2i78wZxWAPNZgUNTp8WIJUAiIs= github.com/redis/go-redis/v9 v9.11.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -118,38 +284,71 @@ go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfC go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -158,9 +357,11 @@ golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.287.1 h1:LiyJx32VU3cwQfLchn/513qKhc25hq0pEANYJoWNnnI= @@ -175,13 +376,19 @@ google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/kv/context.go b/kv/context.go new file mode 100644 index 00000000..85bb6337 --- /dev/null +++ b/kv/context.go @@ -0,0 +1,24 @@ +package kv + +import "context" + +type cacheBypassKey struct{} + +// WithCacheBypass returns a context that instructs SecretStore.Get to skip +// the cache read, fetch from the backend, and re-populate the cache entry. +// It is context-carried because the call path (Resolver.Resolve → +// Registry.GetStore → Provider.Get) has fixed signatures with no room for +// an options parameter. +func WithCacheBypass(ctx context.Context) context.Context { + return context.WithValue(ctx, cacheBypassKey{}, true) +} + +// IsCacheBypassed reports whether context carries the cache-bypass directive. +func IsCacheBypassed(ctx context.Context) bool { + bypassed, ok := ctx.Value(cacheBypassKey{}).(bool) + if !ok { + return false + } + + return bypassed +} diff --git a/kv/errors.go b/kv/errors.go index 8baf8e2d..77749029 100644 --- a/kv/errors.go +++ b/kv/errors.go @@ -29,6 +29,10 @@ type KeyNotFoundError struct { } func (e *KeyNotFoundError) Error() string { + if e.StoreName == "" { + return fmt.Sprintf("key %q not found", e.KeyPath) + } + return fmt.Sprintf("key %q not found in store %q", e.KeyPath, e.StoreName) } diff --git a/kv/errors_test.go b/kv/errors_test.go new file mode 100644 index 00000000..8d40a07d --- /dev/null +++ b/kv/errors_test.go @@ -0,0 +1,39 @@ +package kv_test + +import ( + "errors" + "testing" + + "github.com/TykTechnologies/storage/kv" + "github.com/stretchr/testify/require" +) + +func TestKeyNotFoundError_Error(t *testing.T) { + t.Parallel() + + t.Run("omits the store clause when StoreName is empty", func(t *testing.T) { + t.Parallel() + + // Standalone providers (e.g. file) cannot know their registry name, + // so they leave StoreName empty; the message must still read cleanly. + err := &kv.KeyNotFoundError{KeyPath: "db/password"} + require.Equal(t, `key "db/password" not found`, err.Error()) + }) + + t.Run("includes the store name when present", func(t *testing.T) { + t.Parallel() + + err := &kv.KeyNotFoundError{StoreName: "vault", KeyPath: "db/password"} + require.Equal(t, `key "db/password" not found in store "vault"`, err.Error()) + }) +} + +func TestKeyNotFoundError_IsMatchable(t *testing.T) { + t.Parallel() + + wrapped := errors.Join(errors.New("context"), &kv.KeyNotFoundError{KeyPath: "k"}) + + var notFound *kv.KeyNotFoundError + require.ErrorAs(t, wrapped, ¬Found) + require.Equal(t, "k", notFound.KeyPath) +} diff --git a/kv/internal/resolve/lenient_test.go b/kv/internal/resolve/lenient_test.go new file mode 100644 index 00000000..aabd278c --- /dev/null +++ b/kv/internal/resolve/lenient_test.go @@ -0,0 +1,177 @@ +package resolve_test + +import ( + "context" + "errors" + "testing" + + "github.com/TykTechnologies/storage/kv" + "github.com/TykTechnologies/storage/kv/internal/resolve" + "github.com/stretchr/testify/require" +) + +func TestLenientResolve(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + stores map[string]kv.Provider + input string + want string + wantErr error + }{ + { + name: "whole-value reference to absent store returned unchanged", + input: "kv://vault/db/creds#password", + want: "kv://vault/db/creds#password", + }, + { + name: "inline token to absent store left in place", + stores: map[string]kv.Provider{"env": &mockProvider{value: "myhost.com"}}, + input: "https://$kv{env:API_HOST}/$kv{vault:db/creds#password}", + want: "https://myhost.com/$kv{vault:db/creds#password}", + }, + { + name: "multiple inline tokens to absent stores all left in place", + input: "$kv{vault:a}/$kv{consul:b}", + want: "$kv{vault:a}/$kv{consul:b}", + }, + { + name: "fully resolvable input resolves exactly as in strict mode", + stores: map[string]kv.Provider{"env": &mockProvider{value: "s3cr3t"}}, + input: "kv://env/TOKEN", + want: "s3cr3t", + }, + { + name: "plain string without references passes through", + input: "no references here", + want: "no references here", + }, + { + name: "present store with missing key still errors", + stores: map[string]kv.Provider{ + "env": &mockProvider{err: &kv.KeyNotFoundError{StoreName: "env", KeyPath: "MISSING"}}, + }, + input: "kv://env/MISSING", + wantErr: &kv.KeyNotFoundError{}, + }, + { + name: "malformed whole-value reference still errors", + input: "kv://no-path-separator", + wantErr: resolve.ErrMalformedReference, + }, + { + name: "malformed inline token still errors", + input: "prefix-$kv{missing-store-separator}-suffix", + wantErr: resolve.ErrMalformedReference, + }, + { + name: "missing JSON field on a reachable store still errors", + stores: map[string]kv.Provider{"env": &mockProvider{value: `{"user":"admin"}`}}, + input: "kv://env/creds#missing_field", + + wantErr: resolve.ErrFieldNotFound, + }, + { + name: "absent-store token coexists with an error on a reachable store", + stores: map[string]kv.Provider{ + "env": &mockProvider{err: &kv.KeyNotFoundError{StoreName: "env", KeyPath: "A"}}, + }, + input: "$kv{env:A}/$kv{vault:b}", + wantErr: &kv.KeyNotFoundError{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + r := resolve.NewResolver(newGetter(tt.stores), resolve.WithLenientMode()) + + got, err := r.Resolve(t.Context(), tt.input) + + if tt.wantErr != nil { + var keyNotFound *kv.KeyNotFoundError + if errors.As(tt.wantErr, &keyNotFound) { + require.ErrorAs(t, err, &keyNotFound) + } else { + require.ErrorIs(t, err, tt.wantErr) + } + + return + } + + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestLenientResolveAll(t *testing.T) { + t.Parallel() + + t.Run("absent-store references pass through, resolvable ones resolve", func(t *testing.T) { + t.Parallel() + + r := resolve.NewResolver( + newGetter(map[string]kv.Provider{"env": &mockProvider{value: "hvs.secret-token"}}), + resolve.WithLenientMode(), + ) + + doc := []byte(`{ + "token": "kv://env/VAULT_TOKEN", + "credentials": "kv://vault/aws-creds", + "url": "https://$kv{env:VAULT_TOKEN}@$kv{consul:host}" + }`) + + got, err := r.ResolveAll(context.Background(), doc) + require.NoError(t, err) + require.JSONEq(t, `{ + "token": "hvs.secret-token", + "credentials": "kv://vault/aws-creds", + "url": "https://hvs.secret-token@$kv{consul:host}" + }`, string(got)) + }) + + t.Run("malformed reference inside the document still errors", func(t *testing.T) { + t.Parallel() + + r := resolve.NewResolver(newGetter(nil), resolve.WithLenientMode()) + + _, err := r.ResolveAll(context.Background(), []byte(`{"bad": "kv://no-path-separator"}`)) + require.ErrorIs(t, err, resolve.ErrMalformedReference) + }) + + t.Run("invalid JSON still errors", func(t *testing.T) { + t.Parallel() + + r := resolve.NewResolver(newGetter(nil), resolve.WithLenientMode()) + + _, err := r.ResolveAll(context.Background(), []byte(`{not json`)) + require.ErrorIs(t, err, resolve.ErrInvalidJSON) + }) +} + +// Strict mode is the default and must be completely unaffected by the +// existence of the lenient option. +func TestStrictModeRemainsDefault(t *testing.T) { + t.Parallel() + + t.Run("whole-value reference to absent store errors", func(t *testing.T) { + t.Parallel() + + r := resolve.NewResolver(newGetter(nil)) + + _, err := r.Resolve(context.Background(), "kv://vault/db/creds") + require.ErrorIs(t, err, kv.ErrStoreNotFound) + }) + + t.Run("inline token to absent store errors", func(t *testing.T) { + t.Parallel() + + r := resolve.NewResolver(newGetter(nil)) + + _, err := r.Resolve(context.Background(), "https://$kv{vault:host}/v1") + require.ErrorIs(t, err, kv.ErrStoreNotFound) + }) +} diff --git a/kv/internal/resolve/resolver.go b/kv/internal/resolve/resolver.go index b166508f..2b4a8a00 100644 --- a/kv/internal/resolve/resolver.go +++ b/kv/internal/resolve/resolver.go @@ -14,10 +14,31 @@ import ( type Resolver struct { registry kv.StoreGetter + lenient bool } type Option func(*Resolver) +// WithLenientMode makes the resolver leave any reference that targets an +// unknown store (kv.ErrStoreNotFound) unchanged, instead of failing. +// +// It exists for exactly one caller: Phase 1 of the registry bootstrap, +// which resolves store configs before the remote stores they may reference +// have been initialized. At that point the absent stores are precisely the +// remote ones, and their references must pass through verbatim so the +// caller's later strict pass can resolve them. +// +// Lenient mode tolerates absent stores only: malformed references, +// reachable stores with missing keys, and missing JSON fields still fail, +// so config typos are never silently masked. Deliberately not exposed +// through the public kv/resolver facade — exposing it would invite callers +// to suppress resolution errors wholesale. +func WithLenientMode() Option { + return func(r *Resolver) { + r.lenient = true + } +} + func NewResolver(registry kv.StoreGetter, opts ...Option) *Resolver { r := &Resolver{registry: registry} for _, opt := range opts { @@ -31,66 +52,49 @@ var inlineRe = regexp.MustCompile(`\$kv\{([^}]+)\}`) func (r *Resolver) Resolve(ctx context.Context, input string) (string, error) { if strings.HasPrefix(input, "kv://") { - trimmed := strings.TrimPrefix(input, "kv://") - - slashIdx := strings.IndexByte(trimmed, '/') - if slashIdx < 0 { - return "", fmt.Errorf( - "%w: missing path separator in %q", - ErrMalformedReference, - input, - ) - } + return r.resolveURI(ctx, input) + } - storeName := trimmed[:slashIdx] - rest := trimmed[slashIdx+1:] - path, fragment, _ := strings.Cut(rest, "#") + return r.resolveInline(ctx, input) +} - if storeName == "" || path == "" { - return "", fmt.Errorf( - "%w: empty store name or path in %q", - ErrMalformedReference, - input, - ) - } +// resolveURI resolves a whole-value "kv://store/path#fragment" reference. +func (r *Resolver) resolveURI(ctx context.Context, input string) (string, error) { + body := strings.TrimPrefix(input, "kv://") - return r.fetchAndExtract(ctx, storeName, path, fragment) + storeName, path, fragment, err := parseReference(body, '/', input) + if err != nil { + return "", err } - var resolveErrs []error - result := inlineRe.ReplaceAllStringFunc(input, func(match string) string { - // strip "$kv{" prefix and "}" suffix - inner := match[4 : len(match)-1] - - colonIdx := strings.IndexByte(inner, ':') - if colonIdx < 0 { - resolveErrs = append(resolveErrs, fmt.Errorf( - "%w: missing store separator in %q", - ErrMalformedReference, - match, - )) - - return match - } + res, err := r.fetchAndExtract(ctx, storeName, path, fragment) + if r.lenient && errors.Is(err, kv.ErrStoreNotFound) { + return input, nil + } - storeName := inner[:colonIdx] - rest := inner[colonIdx+1:] - path, fragment, _ := strings.Cut(rest, "#") + return res, err +} - if storeName == "" || path == "" { - resolveErrs = append(resolveErrs, fmt.Errorf( - "%w: empty store name or path in %q", - ErrMalformedReference, - match, - )) +// resolveInline resolves every embedded "$kv{store:path#fragment}" token in a +// larger string, accumulating errors so a single call reports all failures. +func (r *Resolver) resolveInline(ctx context.Context, input string) (string, error) { + // The token regex requires a closing brace, so an unclosed "$kv{" can + // never match — without this check a typo'd reference would silently pass + // through as a literal value. + if unclosedInlineToken(input) >= 0 { + return "", fmt.Errorf( + "%w: unclosed $kv{ reference in %q", + ErrMalformedReference, + input, + ) + } - return match - } + var resolveErrs []error - val, err := r.fetchAndExtract(ctx, storeName, path, fragment) + result := inlineRe.ReplaceAllStringFunc(input, func(match string) string { + val, err := r.resolveInlineToken(ctx, match) if err != nil { resolveErrs = append(resolveErrs, err) - return match } return val @@ -103,6 +107,53 @@ func (r *Resolver) Resolve(ctx context.Context, input string) (string, error) { return result, nil } +func (r *Resolver) resolveInlineToken(ctx context.Context, match string) (string, error) { + // strip "$kv{" prefix and "}" suffix + inner := match[4 : len(match)-1] + + storeName, path, fragment, err := parseReference(inner, ':', match) + if err != nil { + return match, err + } + + val, err := r.fetchAndExtract(ctx, storeName, path, fragment) + if err != nil { + if r.lenient && errors.Is(err, kv.ErrStoreNotFound) { + return match, nil + } + + return match, err + } + + return val, nil +} + +// parseReference splits "storepath#fragment" into its parts. raw is the +// original reference text, used only for error messages. +func parseReference(body string, sep byte, raw string) (storeName, path, fragment string, err error) { + sepIdx := strings.IndexByte(body, sep) + if sepIdx < 0 { + return "", "", "", fmt.Errorf( + "%w: missing store/path separator in %q", + ErrMalformedReference, + raw, + ) + } + + storeName = body[:sepIdx] + path, fragment, _ = strings.Cut(body[sepIdx+1:], "#") + + if storeName == "" || path == "" { + return "", "", "", fmt.Errorf( + "%w: empty store name or path in %q", + ErrMalformedReference, + raw, + ) + } + + return storeName, path, fragment, nil +} + func (r *Resolver) ResolveAll(ctx context.Context, rawJSON []byte) ([]byte, error) { // Fast path: skip unmarshal/remarshal entirely when no KV syntax is present, // preserving the original bytes and avoiding unnecessary allocations. @@ -163,6 +214,32 @@ func (r *Resolver) fetchAndExtract(ctx context.Context, storeName, path, fragmen return extractJSONPointer(raw, fragment) } +// unclosedInlineToken returns the index of the first "$kv{" occurrence in +// input that is not the start of a well-formed $kv{...} token, or -1 when +// every occurrence is properly closed. +func unclosedInlineToken(input string) int { + starts := make(map[int]struct{}) + for _, m := range inlineRe.FindAllStringIndex(input, -1) { + starts[m[0]] = struct{}{} + } + + offset := 0 + + for { + i := strings.Index(input[offset:], "$kv{") + if i < 0 { + return -1 + } + + abs := offset + i + if _, ok := starts[abs]; !ok { + return abs + } + + offset = abs + len("$kv{") + } +} + func (r *Resolver) walkAndResolve(ctx context.Context, node any) (any, error) { switch v := node.(type) { case string: diff --git a/kv/internal/resolve/resolver_test.go b/kv/internal/resolve/resolver_test.go index 7a23dfec..20bb99b1 100644 --- a/kv/internal/resolve/resolver_test.go +++ b/kv/internal/resolve/resolver_test.go @@ -3,6 +3,7 @@ package resolve_test import ( "context" "encoding/json" + "fmt" "testing" "github.com/TykTechnologies/storage/kv" @@ -203,6 +204,28 @@ func TestResolve(t *testing.T) { input: "kv://vault/$kv{env:SUFFIX}", want: "resolved", }, + { + name: "malformed $kv{} unclosed token", + input: "$kv{unclosed", + wantErr: resolve.ErrMalformedReference, + }, + { + name: "malformed $kv{} unclosed token after a valid token", + stores: map[string]kv.Provider{"env": &mockProvider{value: "v"}}, + input: "$kv{env:KEY} and then $kv{oops", + wantErr: resolve.ErrMalformedReference, + }, + { + name: "malformed $kv{} unclosed token inside a larger string", + input: "https://$kv{unclosed/path", + wantErr: resolve.ErrMalformedReference, + }, + { + name: "resolved value containing an unclosed marker is not an error", + stores: map[string]kv.Provider{"vault": &mockProvider{value: "literal-$kv{-inside"}}, + input: "x-$kv{vault:secret}-y", + want: "x-literal-$kv{-inside-y", + }, } for _, tc := range tests { @@ -629,3 +652,54 @@ func BenchmarkResolve_ThreeInlineTokensWithFragment(b *testing.B) { } } } + +// BenchmarkResolveAll measures the whole-document resolution cost the caller +// pays at startup as a function of reference count. Providers are +// in-memory mocks, so the numbers isolate the library's own overhead +// — parse, walk, substitute, re-serialize — from backend latency. +func BenchmarkResolveAll(b *testing.B) { + getter := newGetter(map[string]kv.Provider{ + "env": &mockProvider{value: "resolved-value"}, + "vault": &mockProvider{value: `{"username":"admin","password":"hunter2"}`}, + }) + r := resolve.NewResolver(getter) + ctx := context.Background() + + for _, n := range []int{20, 50, 100} { + doc := buildBenchDoc(b, n) + + b.Run(fmt.Sprintf("refs=%d", n), func(b *testing.B) { + for b.Loop() { + if _, err := r.ResolveAll(ctx, doc); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func buildBenchDoc(b *testing.B, n int) []byte { + b.Helper() + + fields := make(map[string]any, n*2) + + for i := 0; i < n; i++ { + key := fmt.Sprintf("field_%d", i) + + switch i % 3 { + case 0: + fields[key] = "kv://env/SOME_KEY" + case 1: + fields[key] = fmt.Sprintf("https://$kv{env:HOST_%d}/v1", i) + case 2: + fields[key] = "kv://vault/db/creds#password" + } + + fields[fmt.Sprintf("plain_%d", i)] = "no reference here" + } + + doc, err := json.Marshal(map[string]any{"config": fields}) + require.NoError(b, err) + + return doc +} diff --git a/kv/internal/store/cache_bypass_test.go b/kv/internal/store/cache_bypass_test.go new file mode 100644 index 00000000..14c3a1ea --- /dev/null +++ b/kv/internal/store/cache_bypass_test.go @@ -0,0 +1,231 @@ +package store + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/TykTechnologies/storage/kv" + "github.com/stretchr/testify/require" +) + +func sequencedProvider() *mockProvider { + var n int32 + + return &mockProvider{ + mockGetFunc: func(_ context.Context, _ string) (string, error) { + return fmt.Sprintf("v%d", atomic.AddInt32(&n, 1)), nil + }, + } +} + +func TestGetWithCacheBypass(t *testing.T) { + t.Parallel() + + enabledCache := kv.CacheConfig{Enabled: true, TTL: "1m"} + + t.Run("bypass skips cache read and returns fresh value", func(t *testing.T) { + t.Parallel() + + provider := sequencedProvider() + store := newTestStore(t, provider, enabledCache) + + first, err := store.Get(t.Context(), "db/password") + require.NoError(t, err) + require.Equal(t, "v1", first) + + cached, err := store.Get(t.Context(), "db/password") + require.NoError(t, err) + require.Equal(t, "v1", cached) + require.Equal(t, int32(1), provider.calls.Load(), "second Get must be served from cache") + + fresh, err := store.Get(kv.WithCacheBypass(t.Context()), "db/password") + require.NoError(t, err) + require.Equal(t, "v2", fresh) + require.Equal(t, int32(2), provider.calls.Load(), "bypass Get must hit the provider") + }) + + t.Run("bypass re-populates the cache entry", func(t *testing.T) { + t.Parallel() + + provider := sequencedProvider() + store := newTestStore(t, provider, enabledCache) + + _, err := store.Get(t.Context(), "db/password") // v1 cached + require.NoError(t, err) + + fresh, err := store.Get(kv.WithCacheBypass(t.Context()), "db/password") + require.NoError(t, err) + require.Equal(t, "v2", fresh) + + after, err := store.Get(t.Context(), "db/password") + require.NoError(t, err) + require.Equal(t, "v2", after) + require.Equal(t, int32(2), provider.calls.Load(), "post-bypass Get must be served from the re-populated cache") + }) + + t.Run("bypass on a never-cached key behaves like a normal miss", func(t *testing.T) { + t.Parallel() + + provider := sequencedProvider() + store := newTestStore(t, provider, enabledCache) + + fresh, err := store.Get(kv.WithCacheBypass(t.Context()), "db/password") + require.NoError(t, err) + require.Equal(t, "v1", fresh) + require.Equal(t, int32(1), provider.calls.Load()) + + cached, err := store.Get(t.Context(), "db/password") + require.NoError(t, err) + require.Equal(t, "v1", cached) + require.Equal(t, int32(1), provider.calls.Load(), "value fetched via bypass must be cached") + }) + + t.Run("bypass skips a cached negative entry", func(t *testing.T) { + t.Parallel() + + var n int32 + provider := &mockProvider{ + mockGetFunc: func(_ context.Context, _ string) (string, error) { + if atomic.AddInt32(&n, 1) == 1 { + return "", &kv.KeyNotFoundError{StoreName: "test", KeyPath: "db/password"} + } + + return "recovered", nil + }, + } + store := newTestStore(t, provider, enabledCache) + + _, err := store.Get(t.Context(), "db/password") + require.Error(t, err) + + _, err = store.Get(t.Context(), "db/password") + require.Error(t, err) + require.Equal(t, int32(1), provider.calls.Load(), "error must be served from negative cache") + + fresh, err := store.Get(kv.WithCacheBypass(t.Context()), "db/password") + require.NoError(t, err) + require.Equal(t, "recovered", fresh) + + cached, err := store.Get(t.Context(), "db/password") + require.NoError(t, err) + require.Equal(t, "recovered", cached) + require.Equal(t, int32(2), provider.calls.Load()) + }) + + t.Run("provider error during bypass propagates instead of stale value", func(t *testing.T) { + t.Parallel() + + var n int32 + provider := &mockProvider{ + mockGetFunc: func(_ context.Context, _ string) (string, error) { + if atomic.AddInt32(&n, 1) == 1 { + return "v1", nil + } + + return "", &kv.StoreUnavailableError{ + StoreName: "test", + KeyPath: "db/password", + Err: fmt.Errorf("backend down"), + } + }, + } + store := newTestStore(t, provider, enabledCache) + + _, err := store.Get(t.Context(), "db/password") + require.NoError(t, err) + + _, err = store.Get(kv.WithCacheBypass(t.Context()), "db/password") + require.Error( + t, + err, + "bypass asked for a fresh value; a stale fallback would hide rotation failures", + ) + + var unavailable *kv.StoreUnavailableError + require.ErrorAs(t, err, &unavailable) + }) + + t.Run("bypass with disabled cache behaves like a plain Get", func(t *testing.T) { + t.Parallel() + + provider := sequencedProvider() + store := newTestStore(t, provider, kv.CacheConfig{Enabled: false}) + + got, err := store.Get(kv.WithCacheBypass(t.Context()), "db/password") + require.NoError(t, err) + require.Equal(t, "v1", got) + + got, err = store.Get(kv.WithCacheBypass(t.Context()), "db/password") + require.NoError(t, err) + require.Equal(t, "v2", got) + }) + + t.Run("context without marker keeps existing cache behavior", func(t *testing.T) { + t.Parallel() + + provider := sequencedProvider() + store := newTestStore(t, provider, enabledCache) + + for range 5 { + got, err := store.Get(t.Context(), "db/password") + require.NoError(t, err) + require.Equal(t, "v1", got) + } + + require.Equal(t, int32(1), provider.calls.Load(), "plain Gets must keep serving from cache") + }) +} + +func TestGetWithCacheBypassConcurrency(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var n int32 + provider := &mockProvider{ + delay: time.Second, + mockGetFunc: func(_ context.Context, _ string) (string, error) { + return fmt.Sprintf("v%d", atomic.AddInt32(&n, 1)), nil + }, + } + store := newTestStore(t, provider, kv.CacheConfig{Enabled: true, TTL: "1m"}) + ctx := context.Background() + + seeded, err := store.Get(ctx, "db/password") + require.NoError(t, err) + require.Equal(t, "v1", seeded) + + var wg sync.WaitGroup + + // Bypass Gets block on the provider's delay; they must dedupe into + // one fetch via singleflight. + for range 5 { + wg.Go(func() { + got, err := store.Get(kv.WithCacheBypass(ctx), "db/password") + require.NoError(t, err) + require.Equal(t, "v2", got) + }) + } + + // Non-bypass Gets run while the bypass fetch is in flight: they are + // served the still-valid cached value without blocking. + for range 5 { + wg.Go(func() { + got, err := store.Get(ctx, "db/password") + require.NoError(t, err) + require.Equal(t, "v1", got) + }) + } + + wg.Wait() + + require.Equal(t, int32(2), provider.calls.Load(), "5 concurrent bypass Gets must collapse into one provider fetch") + + after, err := store.Get(ctx, "db/password") + require.NoError(t, err) + require.Equal(t, "v2", after) + require.Equal(t, int32(2), provider.calls.Load()) + }) +} diff --git a/kv/internal/store/store.go b/kv/internal/store/store.go index 59e6ead0..649622cf 100644 --- a/kv/internal/store/store.go +++ b/kv/internal/store/store.go @@ -12,8 +12,6 @@ import ( "golang.org/x/sync/singleflight" ) -const defaultProviderTimeout = 5 * time.Second - // SecretStore is an internal decorator that adds caching and singleflight to a Provider. type SecretStore struct { name string @@ -35,7 +33,7 @@ func (s *SecretStore) Get(ctx context.Context, path string) (string, error) { } val, exists, needsRefresh, err := s.cache.Get(path) - if exists { + if exists && !kv.IsCacheBypassed(ctx) { // Fail fast on cached errors if err != nil { return "", err @@ -155,12 +153,12 @@ func NewSecretStore( opts ...Option, ) (*SecretStore, error) { if provider == nil { - return nil, fmt.Errorf("failed to create a secret store with name %q: provider cannot be nil", name) + return nil, fmt.Errorf("secret store %q: provider cannot be nil", name) } cache, err := cache.NewCache(cacheConfig) if err != nil { - return nil, fmt.Errorf("failed to create secret store: %w", err) + return nil, fmt.Errorf("secret store %q: %w", name, err) } s := &SecretStore{ @@ -169,7 +167,7 @@ func NewSecretStore( cache: cache, sf: &singleflight.Group{}, sfRefresh: &singleflight.Group{}, - timeout: defaultProviderTimeout, + timeout: kv.DefaultOperationTimeout, } for _, opt := range opts { diff --git a/kv/internal/store/store_test.go b/kv/internal/store/store_test.go index f33616b0..b9dbd20a 100644 --- a/kv/internal/store/store_test.go +++ b/kv/internal/store/store_test.go @@ -62,7 +62,7 @@ func TestNewSecretStore(t *testing.T) { }) require.Error(t, err) require.Nil(t, store) - require.Contains(t, err.Error(), "failed to create secret store") + require.Contains(t, err.Error(), `secret store "test"`) }) t.Run("negative TTL", func(t *testing.T) { @@ -82,7 +82,7 @@ func TestNewSecretStore(t *testing.T) { }) require.NoError(t, err) require.NotNil(t, store) - require.Equal(t, defaultProviderTimeout, store.timeout) + require.Equal(t, kv.DefaultOperationTimeout, store.timeout) }) t.Run("cache disabled", func(t *testing.T) { @@ -667,3 +667,47 @@ func newTestStore(t *testing.T, provider kv.Provider, cfg kv.CacheConfig) *Secre return store } + +// BenchmarkSecretStoreGet measures the per-request cost of a Get through the +// SecretStore wrapper — the "no measurable per-request latency when values are +// cache-resident" acceptance criterion. +func BenchmarkSecretStoreGet(b *testing.B) { + ctx := context.Background() + + b.Run("cache-hit", func(b *testing.B) { + s, err := NewSecretStore("bench", &mockProvider{}, kv.CacheConfig{Enabled: true, TTL: "1h"}) + if err != nil { + b.Fatal(err) + } + defer s.Close(ctx) + + // Warm the cache so every measured Get is a hit. + if _, err := s.Get(ctx, "path"); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + + for b.Loop() { + if _, err := s.Get(ctx, "path"); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("cache-disabled", func(b *testing.B) { + s, err := NewSecretStore("bench", &mockProvider{}, kv.CacheConfig{Enabled: false}) + if err != nil { + b.Fatal(err) + } + defer s.Close(ctx) + + b.ResetTimer() + + for b.Loop() { + if _, err := s.Get(ctx, "path"); err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/kv/provider.go b/kv/provider.go index b7d94e10..2f6398bd 100644 --- a/kv/provider.go +++ b/kv/provider.go @@ -18,15 +18,15 @@ const ( // Inline resolves secrets from plain text in the configuration. Inline ProviderType = "inline" + // File resolves secrets from files on the local filesystem. + File ProviderType = "file" + // Vault resolves secrets from HashiCorp Vault. Vault ProviderType = "hashicorp_vault" // Consul resolves secrets from HashiCorp Consul. Consul ProviderType = "hashicorp_consul" - // K8s resolves secrets from Kubernetes Secrets mounted as files. - K8s ProviderType = "k8s_files" - // --- Enterprise Edition (EE) Providers --- // AWS resolves secrets from AWS Secrets Manager. @@ -42,6 +42,22 @@ const ( Conjur ProviderType = "cyberark_conjur" ) +// DefaultOperationTimeout bounds a single provider Get/Set when neither the store +// config nor the SecretStore wrapper supplies one. +const DefaultOperationTimeout = 5 * time.Second + +// IsLocal reports whether this provider type resolves secrets from resources +// available to the local process — environment variables, inline config data, +// or the filesystem — requiring no network and a literal, reference-free config. +func (t ProviderType) IsLocal() bool { + switch t { + case Env, Inline, File: + return true + default: + return false + } +} + // KeyValueRetriever defines the core read capability for retrieving values by key. type KeyValueRetriever interface { Get(ctx context.Context, key string) (string, error) @@ -69,6 +85,18 @@ type StoreGetter interface { // // The factory pattern allows the registry to create providers dynamically // without compile-time dependencies on specific provider implementations. +// +// Config validation convention. A factory returns an error only for config that +// is present but invalid — a value that can never produce correct behavior +// (malformed JSON, or a security-critical field set to an unusable value, e.g. +// a relative path where an absolute one is required). It must NOT error for +// merely-absent optional config: an unset value is a valid state, so the factory +// builds a provider that runs in its default or disabled mode (e.g. an env +// provider with no prefix, or a file provider with no base_path that then +// rejects every Get). Whether to create a store at all for an unconfigured +// feature is the caller's decision, not the factory's. The effect: configuration +// mistakes surface once, at construction, while unused features stay quiet +// instead of failing on every Get. type ProviderFactory func(config json.RawMessage) (Provider, error) // Initializer is an optional interface for providers that require network @@ -155,3 +183,13 @@ func As[T any](p Provider) (T, bool) { return zero, false } + +// EffectiveTimeout resolves a configured timeout to the value actually used: +// the configured value when positive, else the default. +func EffectiveTimeout(configured time.Duration) time.Duration { + if configured > 0 { + return configured + } + + return DefaultOperationTimeout +} diff --git a/kv/provider_test.go b/kv/provider_test.go index f72b3320..a36e4ff5 100644 --- a/kv/provider_test.go +++ b/kv/provider_test.go @@ -1,9 +1,10 @@ -package kv +package kv_test import ( "context" "testing" + "github.com/TykTechnologies/storage/kv" "github.com/stretchr/testify/require" ) @@ -13,7 +14,7 @@ func (m *mockProvider) Get(ctx context.Context, path string) (string, error) { return "", nil } -func (m *mockProvider) Unwrap() Provider { +func (m *mockProvider) Unwrap() kv.Provider { return m } @@ -21,7 +22,36 @@ func TestAs_CircularDependencyWontFail(t *testing.T) { t.Parallel() m := &mockProvider{} - _, ok := As[Closer](m) + _, ok := kv.As[kv.Closer](m) require.False(t, ok) } + +func TestProviderType_IsLocal(t *testing.T) { + t.Parallel() + + local := []kv.ProviderType{ + kv.Env, + kv.Inline, + kv.File, + } + + remote := []kv.ProviderType{ + kv.Vault, + kv.Consul, + kv.AWS, + kv.GCP, + kv.Azure, + kv.Conjur, + "unknown_provider", + "", + } + + for _, pt := range local { + require.Truef(t, pt.IsLocal(), "%q must initialize in Phase 1", pt) + } + + for _, pt := range remote { + require.Falsef(t, pt.IsLocal(), "%q must NOT be treated as a Phase 1 local store", pt) + } +} diff --git a/kv/providers/consul/config.go b/kv/providers/consul/config.go new file mode 100644 index 00000000..323e805d --- /dev/null +++ b/kv/providers/consul/config.go @@ -0,0 +1,168 @@ +package consul + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/TykTechnologies/storage/kv" + consulsdk "github.com/hashicorp/consul/api" +) + +// Config is the JSON "config" block of a consul store. +type Config struct { + // Address of the consul agent as host:port; the scheme comes from Scheme. + Address string `json:"address"` + + // Scheme is the URI scheme ("http" or "https"). Defaults to "http". + Scheme string `json:"scheme"` + + // Datacenter to query. Empty uses the agent's default datacenter. + Datacenter string `json:"datacenter"` + + // HttpAuth holds optional HTTP Basic Auth credentials. + HttpAuth struct { + Username string `json:"username"` + Password string `json:"password"` + } `json:"http_auth"` + + // WaitTime is consul's blocking-query (watch) timeout, as a Go duration + // string such as "5s". It is NOT a per-operation timeout — our plain reads + // never block on it — which is why the provider does not implement Timeouter. + // Empty or "0s" leaves the agent default. + WaitTime string `json:"wait_time"` + + // Token is used to provide a per-request ACL token + // which overrides the agent's default token. + Token string `json:"token"` + + // TLSConfig configures TLS for https connections. + TLSConfig struct { + Address string `json:"address"` + CAFile string `json:"ca_file"` + CAPath string `json:"ca_path"` + CertFile string `json:"cert_file"` + KeyFile string `json:"key_file"` + InsecureSkipVerify bool `json:"insecure_skip_verify"` + } `json:"tls_config"` +} + +// NewFactory returns a kv.ProviderFactory for HashiCorp Consul stores. +// +// The factory parses Config and builds a consul API client, but performs no +// network I/O. It errors only for config that is present but unusable: malformed +// JSON, an unparseable wait_time, or TLS settings the client rejects (e.g. an +// unreadable ca_file). Absent config is valid — an empty blob builds a client +// against consul's default local agent. +func NewFactory() kv.ProviderFactory { + return func(raw json.RawMessage) (kv.Provider, error) { + var conf Config + if err := parseConfig(raw, &conf); err != nil { + return nil, err + } + + clientCfg, err := conf.clientConfig() + if err != nil { + return nil, err + } + + client, err := consulsdk.NewClient(clientCfg) + if err != nil { + return nil, fmt.Errorf("consul: create client: %w", err) + } + + return &consulProvider{kvClient: client.KV()}, nil + } +} + +func parseConfig(raw json.RawMessage, conf *Config) error { + if len(raw) == 0 { + return nil + } + + if err := json.Unmarshal(raw, conf); err != nil { + return fmt.Errorf("consul: invalid config: %w", err) + } + + return nil +} + +func (conf *Config) clientConfig() (*consulsdk.Config, error) { + clientCfg := consulsdk.DefaultConfig() + + if conf.Address != "" { + clientCfg.Address = conf.Address + } + + if conf.Scheme != "" { + clientCfg.Scheme = conf.Scheme + } + + if conf.Datacenter != "" { + clientCfg.Datacenter = conf.Datacenter + } + + if conf.Token != "" { + clientCfg.Token = conf.Token + } + + if conf.HttpAuth.Username != "" || conf.HttpAuth.Password != "" { + clientCfg.HttpAuth = &consulsdk.HttpBasicAuth{ + Username: conf.HttpAuth.Username, + Password: conf.HttpAuth.Password, + } + } + + if err := conf.applyWaitTime(clientCfg); err != nil { + return nil, err + } + + conf.applyTLSConfig(clientCfg) + + return clientCfg, nil +} + +func (conf *Config) applyWaitTime(clientCfg *consulsdk.Config) error { + if conf.WaitTime == "" { + return nil + } + + waitTime, err := time.ParseDuration(conf.WaitTime) + if err != nil { + return fmt.Errorf("consul: invalid wait_time %q: %w", conf.WaitTime, err) + } + + if waitTime > 0 { + clientCfg.WaitTime = waitTime + } + + return nil +} + +func (conf *Config) applyTLSConfig(clientCfg *consulsdk.Config) { + tls := conf.TLSConfig + + if tls.Address != "" { + clientCfg.TLSConfig.Address = tls.Address + } + + if tls.CAFile != "" { + clientCfg.TLSConfig.CAFile = tls.CAFile + } + + if tls.CAPath != "" { + clientCfg.TLSConfig.CAPath = tls.CAPath + } + + if tls.CertFile != "" { + clientCfg.TLSConfig.CertFile = tls.CertFile + } + + if tls.KeyFile != "" { + clientCfg.TLSConfig.KeyFile = tls.KeyFile + } + + if tls.InsecureSkipVerify { + clientCfg.TLSConfig.InsecureSkipVerify = true + } +} diff --git a/kv/providers/consul/consul.go b/kv/providers/consul/consul.go new file mode 100644 index 00000000..6418900d --- /dev/null +++ b/kv/providers/consul/consul.go @@ -0,0 +1,42 @@ +// Package consul provides a kv.Provider backed by HashiCorp Consul's KV store. +// It performs direct per-key reads (GET /v1/kv/) and returns each key's +// value verbatim, leaving "#field" extraction to the resolver. +// +// Consul is a remote provider: it is not Standalone, so the registry wraps it in +// the caching / singleflight SecretStore. The client is built without any +// network I/O; the connection is established lazily on the first Get. +package consul + +import ( + "context" + + "github.com/TykTechnologies/storage/kv" + consulsdk "github.com/hashicorp/consul/api" +) + +// consulProvider is a kv.Provider backed by a configured consul KV client. +type consulProvider struct { + // kvClient is consul's KV endpoint, with the resolved Config already baked + // into the underlying client. + kvClient *consulsdk.KV +} + +// Get reads the value at key and returns it verbatim: no trimming, no key +// transformation, no interpretation ("#field" extraction is the resolver's job). +// +// A missing key returns *kv.KeyNotFoundError and a transport/backend failure +// returns *kv.StoreUnavailableError — the two error types the negative cache +// distinguishes. ctx bounds the request via QueryOptions, so the SecretStore's +// per-operation deadline is honored. +func (cp *consulProvider) Get(ctx context.Context, key string) (string, error) { + pair, _, err := cp.kvClient.Get(key, (&consulsdk.QueryOptions{}).WithContext(ctx)) + if err != nil { + return "", &kv.StoreUnavailableError{KeyPath: key, Err: err} + } + + if pair == nil { + return "", &kv.KeyNotFoundError{KeyPath: key} + } + + return string(pair.Value), nil +} diff --git a/kv/providers/consul/consul_test.go b/kv/providers/consul/consul_test.go new file mode 100644 index 00000000..97a2db91 --- /dev/null +++ b/kv/providers/consul/consul_test.go @@ -0,0 +1,401 @@ +package consul_test + +import ( + "context" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/TykTechnologies/storage/kv" + "github.com/TykTechnologies/storage/kv/providers/consul" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// How these tests work: +// +// The provider is exercised through the real github.com/hashicorp/consul/api +// client against an in-process HTTP server (consulStub) that mimics consul's KV +// HTTP API. Each test points Config.Address at the stub, so the client builds +// genuine consul requests and the stub returns canned KV responses. This covers +// the actual client wiring — URL/path construction, base64 value decoding, +// 404→not-found, basic-auth headers — rather than a mock of it, while staying +// hermetic and millisecond-fast. +// + +// clearConsulEnv blanks the CONSUL_* environment that consulapi.DefaultConfig +// reads, so client construction is hermetic and reproducible. +func clearConsulEnv(t *testing.T) { + t.Helper() + + for _, k := range []string{ + "CONSUL_HTTP_ADDR", "CONSUL_HTTP_TOKEN", "CONSUL_HTTP_TOKEN_FILE", + "CONSUL_HTTP_AUTH", "CONSUL_HTTP_SSL", "CONSUL_HTTP_SSL_VERIFY", + "CONSUL_CACERT", "CONSUL_CAPATH", "CONSUL_CLIENT_CERT", + "CONSUL_CLIENT_KEY", "CONSUL_TLS_SERVER_NAME", "CONSUL_NAMESPACE", + } { + t.Setenv(k, "") + } +} + +// consulStub is an httptest server that records the requests it receives and +// delegates response construction to a per-test handler. +type consulStub struct { + url string + mu sync.Mutex + got []string + auth []string +} + +// requests returns a copy of the recorded "METHOD /path" entries. +func (s *consulStub) requests() []string { + s.mu.Lock() + defer s.mu.Unlock() + + return append([]string(nil), s.got...) +} + +// lastAuth returns the Authorization header of the most recent request. +func (s *consulStub) lastAuth() string { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.auth) == 0 { + return "" + } + + return s.auth[len(s.auth)-1] +} + +func newConsulStub(t *testing.T, handler http.HandlerFunc) *consulStub { + t.Helper() + + s := &consulStub{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.got = append(s.got, r.Method+" "+r.URL.Path) + s.auth = append(s.auth, r.Header.Get("Authorization")) + s.mu.Unlock() + + handler(w, r) + })) + t.Cleanup(srv.Close) + + s.url = srv.URL + + return s +} + +// writeConsulValue writes the wire shape consul's KV API returns for a present +// key: a JSON array of one KVPair whose Value is base64-encoded. Go marshals a +// []byte field to base64, exactly as consul does and consulapi expects. +func writeConsulValue(w http.ResponseWriter, key, value string) { + w.Header().Set("Content-Type", "application/json") + + //nolint:errcheck + _ = json.NewEncoder(w).Encode([]struct { + Key string `json:"Key"` + Value []byte `json:"Value"` + }{ + {Key: key, Value: []byte(value)}, + }) +} + +// addrOf strips the scheme so the stub URL is usable as a consul Config.Address +// (host:port), matching how the gateway promotes "consul.internal:8500". +func addrOf(url string) string { + return url[len("http://"):] +} + +// newConsulProvider builds the provider through its factory, exactly as the +// registry would, with a hermetic environment. +func newConsulProvider(t *testing.T, cfg *consul.Config) kv.Provider { + t.Helper() + + clearConsulEnv(t) + + raw, err := json.Marshal(cfg) + require.NoError(t, err) + + p, err := consul.NewFactory()(raw) + require.NoError(t, err) + require.NotNil(t, p) + + return p +} + +func mustJSON(t *testing.T, v any) json.RawMessage { + t.Helper() + + b, err := json.Marshal(v) + require.NoError(t, err) + + return b +} + +func TestNewFactory(t *testing.T) { + tests := []struct { + name string + config string + wantErr bool + wantErrContains string + }{ + { + name: "valid full config", + config: `{"address":"127.0.0.1:8500","scheme":"http","datacenter":"dc1","token":"root","wait_time":"5s"}`, + }, + { + name: "address only", + config: `{"address":"consul.internal:8500"}`, + }, + { + name: "empty object is permissive", + config: `{}`, + }, + { + name: "empty bytes is permissive", + config: ``, + }, + { + name: "nil is permissive", + config: "\x00null-sentinel", // replaced with nil below + }, + + { + name: "wait_time 0s (unset duration) is accepted", + config: `{"wait_time":"0s"}`, + }, + { + name: "invalid json", + config: `{not json`, + wantErr: true, + wantErrContains: "consul", + }, + { + name: "invalid wait_time (not a duration string)", + config: `{"wait_time":"5x"}`, + wantErr: true, + wantErrContains: "consul", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clearConsulEnv(t) + + raw := json.RawMessage(tt.config) + if tt.config == "\x00null-sentinel" { + raw = nil + } + + p, err := consul.NewFactory()(raw) + + if tt.wantErr { + require.Error(t, err) + require.Nil(t, p) + + if tt.wantErrContains != "" { + require.ErrorContains(t, err, tt.wantErrContains) + } + + return + } + + require.NoError(t, err) + require.NotNil(t, p) + }) + } +} + +func TestNewFactory_InvalidTLSCAFileErrors(t *testing.T) { + clearConsulEnv(t) + + var cfg consul.Config + cfg.Address = "127.0.0.1:8500" + cfg.TLSConfig.CAFile = "/nonexistent/ca-does-not-exist.pem" + + p, err := consul.NewFactory()(mustJSON(t, cfg)) + require.Error(t, err) + require.Nil(t, p) + require.ErrorContains(t, err, "consul") +} + +func TestProvider_IsNotStandaloner(t *testing.T) { + p := newConsulProvider(t, &consul.Config{}) + + s, ok := kv.AsStandaloner(p) + require.False(t, ok && s.IsStandalone(), + "consul must not be standalone (the registry must wrap it in the cache)") +} + +func TestProvider_DoesNotExposeTimeouter(t *testing.T) { + p := newConsulProvider(t, &consul.Config{WaitTime: "30s"}) + + _, ok := kv.AsTimeouter(p) + require.False(t, ok, + "consul must not expose Timeouter; wait_time is a watch timeout, not a per-op timeout") +} + +func TestGet_ReadsValue(t *testing.T) { + tests := []struct { + name string + key string + value string + wantPath string + }{ + { + name: "single-segment key", + key: "mykey", + value: "myvalue", + wantPath: "GET /v1/kv/mykey", + }, + { + name: "multi-segment key preserved verbatim (no transform)", + key: "services/redis/host", + value: "cache01.internal", + wantPath: "GET /v1/kv/services/redis/host", + }, + { + name: "value returned byte-exact with no trailing-newline trim", + key: "raw", + value: "line\n", + wantPath: "GET /v1/kv/raw", + }, + { + name: "binary-safe value (embedded NUL survives base64 round-trip)", + key: "bin", + value: "a\x00b", + wantPath: "GET /v1/kv/bin", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stub := newConsulStub(t, func(w http.ResponseWriter, _ *http.Request) { + writeConsulValue(w, tt.key, tt.value) + }) + + p := newConsulProvider(t, &consul.Config{Address: addrOf(stub.url)}) + + got, err := p.Get(t.Context(), tt.key) + require.NoError(t, err) + + assert.Equal(t, tt.value, got) + assert.Equal(t, []string{tt.wantPath}, stub.requests()) + }) + } +} + +func TestGet_MissingKeyReturnsKeyNotFound(t *testing.T) { + stub := newConsulStub(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + + p := newConsulProvider(t, &consul.Config{Address: addrOf(stub.url)}) + + _, err := p.Get(t.Context(), "services/absent") + + var notFound *kv.KeyNotFoundError + require.ErrorAs(t, err, ¬Found, + "a missing key must map to *kv.KeyNotFoundError for negative_ttl_not_found caching") + require.Equal(t, "services/absent", notFound.KeyPath) +} + +func TestGet_BackendErrorReturnsStoreUnavailable(t *testing.T) { + stub := newConsulStub(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }) + + p := newConsulProvider(t, &consul.Config{Address: addrOf(stub.url)}) + + _, err := p.Get(t.Context(), "services/redis") + + var unavailable *kv.StoreUnavailableError + require.ErrorAs(t, err, &unavailable, + "a backend failure must map to *kv.StoreUnavailableError for negative_ttl_transient caching") +} + +func TestGet_UsesQueryContextForBasicAuth(t *testing.T) { + stub := newConsulStub(t, func(w http.ResponseWriter, _ *http.Request) { + writeConsulValue(w, "k", "v") + }) + + var cfg consul.Config + cfg.Address = addrOf(stub.url) + cfg.HttpAuth.Username = "user" + cfg.HttpAuth.Password = "pass" + + p := newConsulProvider(t, &cfg) + + got, err := p.Get(t.Context(), "k") + require.NoError(t, err) + require.Equal(t, "v", got) + + want := "Basic " + base64.StdEncoding.EncodeToString([]byte("user:pass")) + require.Equal(t, want, stub.lastAuth()) +} + +func TestGet_PropagatesContextCancellation(t *testing.T) { + stub := newConsulStub(t, func(w http.ResponseWriter, _ *http.Request) { + writeConsulValue(w, "k", "v") + }) + + p := newConsulProvider(t, &consul.Config{Address: addrOf(stub.url)}) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + _, err := p.Get(ctx, "k") + require.Error(t, err) + require.ErrorIs(t, err, context.Canceled) +} + +func TestGet_HonorsContextDeadline(t *testing.T) { + stub := newConsulStub(t, func(w http.ResponseWriter, _ *http.Request) { + // Sleep past the caller's deadline so the request is aborted in flight. + // The provider must attach ctx via QueryOptions.WithContext for the + // SecretStore's per-op deadline to actually bound the call. + time.Sleep(200 * time.Millisecond) + writeConsulValue(w, "k", "v") + }) + + p := newConsulProvider(t, &consul.Config{Address: addrOf(stub.url)}) + + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + + _, err := p.Get(ctx, "k") + require.Error(t, err) + require.ErrorIs(t, err, context.DeadlineExceeded) +} + +func TestBackwardCompatParity_ConsulGet(t *testing.T) { + t.Run("key exists -> value returned", func(t *testing.T) { + stub := newConsulStub(t, func(w http.ResponseWriter, _ *http.Request) { + writeConsulValue(w, "tyk-apis/my_service_url", "https://upstream.internal") + }) + + p := newConsulProvider(t, &consul.Config{Address: addrOf(stub.url)}) + + got, err := p.Get(t.Context(), "tyk-apis/my_service_url") + require.NoError(t, err) + require.Equal(t, "https://upstream.internal", got) + }) + + t.Run("key absent -> not found", func(t *testing.T) { + stub := newConsulStub(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + + p := newConsulProvider(t, &consul.Config{Address: addrOf(stub.url)}) + + _, err := p.Get(t.Context(), "tyk-apis/missing") + + var notFound *kv.KeyNotFoundError + require.ErrorAs(t, err, ¬Found) + }) +} diff --git a/kv/providers/env/env.go b/kv/providers/env/env.go new file mode 100644 index 00000000..20cc879e --- /dev/null +++ b/kv/providers/env/env.go @@ -0,0 +1,107 @@ +// Package env provides a KV provider that reads secrets from the process +// environment. It is a Standalone provider (no caching): environment lookups +// are in-process and cheap, and a rotated value is visible on the next read. +// +// A lookup name is built as Prefix + key, with the key optionally uppercased +// (see Config). +// +// Security: the Prefix is a confinement boundary. Every key is read as +// Prefix+key, so a reference can only reach variables the operator deliberately +// placed under that prefix — never arbitrary process variables (cloud +// credentials, tokens, PATH, …). This matters when references come from a +// less-trusted source than the host. An empty prefix removes that boundary, +// so it is rejected by default: every Get returns ErrPrefixRequired. +// Setting AllowNoPrefix opts back into reading process env directly, +// and is only safe when every reference source is as trusted as the host. +package env + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + "github.com/TykTechnologies/storage/kv" +) + +// ErrPrefixRequired is returned by Get when the provider has an empty prefix and +// AllowNoPrefix is not set. +var ErrPrefixRequired = errors.New("env: prefix is required") + +// Config is the env provider's configuration. +type Config struct { + // Prefix is prepended literally to the (optionally uppercased) key before the + // environment lookup; it is never uppercased itself. By default it is + // required: it is the store's security boundary, so a provider with an empty + // prefix rejects every Get with ErrPrefixRequired unless AllowNoPrefix is set. + Prefix string `json:"prefix"` + + // Uppercase, when true, uppercases the key — not the prefix — before lookup, + // so Get("my_key") with prefix "TYK_SECRET_" reads "TYK_SECRET_MY_KEY". + Uppercase bool `json:"uppercase"` + + // AllowNoPrefix permits an empty prefix, reading process environment + // variables directly with no confinement. It defaults to false so a forgotten + // prefix stays fail-closed rather than silently exposing every process + // variable. + AllowNoPrefix bool `json:"allow_no_prefix"` +} + +// NewFactory returns a ProviderFactory for environment-backed stores. +// +// An empty or absent config still builds a provider, so the store can be +// registered unconditionally. +func NewFactory() kv.ProviderFactory { + return func(config json.RawMessage) (kv.Provider, error) { + if len(config) == 0 { + return &envProvider{}, nil + } + + var cfg Config + + if err := json.Unmarshal(config, &cfg); err != nil { + return nil, fmt.Errorf("env: invalid config: %w", err) + } + + return &envProvider{ + prefix: cfg.Prefix, + uppercase: cfg.Uppercase, + allowNoPrefix: cfg.AllowNoPrefix, + }, nil + } +} + +type envProvider struct { + prefix string + uppercase bool + allowNoPrefix bool +} + +// Get reads the environment variable named Prefix + (uppercased key if +// Uppercase) and returns its value. +// +// If the prefix is empty and AllowNoPrefix is not set it returns +// ErrPrefixRequired for every key — the prefix guard is checked first, so even +// an empty key is rejected before any lookup. Otherwise the result mirrors +// os.Getenv exactly: a missing variable and a variable set to "" are +// indistinguishable, and both return ("", nil); an empty key reads +// os.Getenv(Prefix) and likewise returns no error. +func (ep *envProvider) Get(_ context.Context, key string) (string, error) { + if ep.prefix == "" && !ep.allowNoPrefix { + return "", ErrPrefixRequired + } + + if ep.uppercase { + key = strings.ToUpper(key) + } + + return os.Getenv(ep.prefix + key), nil +} + +// IsStandalone reports that the provider needs no cache wrapper: environment +// reads are in-process and cheap, and there is nothing to refresh. +func (ep *envProvider) IsStandalone() bool { + return true +} diff --git a/kv/providers/env/env_test.go b/kv/providers/env/env_test.go new file mode 100644 index 00000000..50bf4070 --- /dev/null +++ b/kv/providers/env/env_test.go @@ -0,0 +1,200 @@ +package env_test + +import ( + "encoding/json" + "testing" + + "github.com/TykTechnologies/storage/kv" + "github.com/TykTechnologies/storage/kv/providers/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newProvider builds the provider through its factory with the given config, +// exactly as the registry would. +func newProvider(t *testing.T, cfg env.Config) kv.Provider { + t.Helper() + + raw, err := json.Marshal(cfg) + require.NoError(t, err) + + p, err := env.NewFactory()(raw) + require.NoError(t, err) + require.NotNil(t, p) + + return p +} + +func TestNewFactory(t *testing.T) { + t.Parallel() + + t.Run("valid config builds a provider", func(t *testing.T) { + t.Parallel() + + p, err := env.NewFactory()(json.RawMessage(`{"prefix":"TYK_SECRET_","uppercase":true}`)) + require.NoError(t, err) + require.NotNil(t, p) + }) + + t.Run("empty or absent config builds a provider (env refs rejected until prefix is set)", func(t *testing.T) { + t.Parallel() + + for _, cfg := range []json.RawMessage{nil, {}, json.RawMessage(`{}`)} { + p, err := env.NewFactory()(cfg) + require.NoError(t, err, "config %q", string(cfg)) + require.NotNil(t, p) + } + }) + + t.Run("invalid JSON config errors", func(t *testing.T) { + t.Parallel() + + _, err := env.NewFactory()(json.RawMessage(`{not json`)) + require.Error(t, err) + }) +} + +func TestProviderIsStandaloner(t *testing.T) { + t.Parallel() + + p := newProvider(t, env.Config{}) + + standalone, ok := kv.AsStandaloner(p) + require.True(t, ok, "env provider must implement Standalone") + require.True(t, standalone.IsStandalone(), + "env reads are in-process and cheap — no cache wrapper") +} + +func TestProviderGet(t *testing.T) { + t.Run("legacy compat: prefix + uppercase reads TYK_SECRET_", func(t *testing.T) { + t.Setenv("TYK_SECRET_MY_KEY", "value") + + got, err := newProvider(t, env.Config{Prefix: "TYK_SECRET_", Uppercase: true}). + Get(t.Context(), "my_key") + require.NoError(t, err) + assert.Equal(t, "value", got) + }) + + t.Run("unset variable returns empty string and no error", func(t *testing.T) { + got, err := newProvider(t, env.Config{Prefix: "TYK_SECRET_", Uppercase: true}). + Get(t.Context(), "definitely_not_set_var") + require.NoError(t, err) + assert.Equal(t, "", got) + }) + + t.Run("missing variable never returns kv.KeyNotFoundError", func(t *testing.T) { + _, err := newProvider(t, env.Config{Prefix: "TYK_SECRET_", Uppercase: true}). + Get(t.Context(), "definitely_not_set_var") + require.NoError(t, err) + + var notFound *kv.KeyNotFoundError + require.NotErrorAs(t, err, ¬Found, "env must never report not-found") + }) + + t.Run("variable set to empty string returns empty string and no error", func(t *testing.T) { + t.Setenv("TYK_SECRET_EMPTY_KEY", "") + + got, err := newProvider(t, env.Config{Prefix: "TYK_SECRET_", Uppercase: true}). + Get(t.Context(), "empty_key") + require.NoError(t, err) + assert.Equal(t, "", got) + }) + + t.Run("uppercase false uses the key as-is (no case folding)", func(t *testing.T) { + t.Setenv("TYK_SECRET_FOO", "bar") + + p := newProvider(t, env.Config{Prefix: "TYK_SECRET_", Uppercase: false}) + + got, err := p.Get(t.Context(), "FOO") + require.NoError(t, err) + assert.Equal(t, "bar", got) + + got, err = p.Get(t.Context(), "foo") + require.NoError(t, err) + assert.Equal(t, "", got, "no case folding: lowercase key must not match TYK_SECRET_FOO") + }) + + t.Run("empty prefix is rejected by default: every key errors with ErrPrefixRequired", func(t *testing.T) { + t.Setenv("BARE_KEY", "bare-value") + + // AllowNoPrefix defaults to false — the fail-closed guard. + p := newProvider(t, env.Config{Prefix: "", Uppercase: true}) + + for _, key := range []string{"bare_key", "BARE_KEY", ""} { + _, err := p.Get(t.Context(), key) + require.ErrorIs(t, err, env.ErrPrefixRequired, "key %q", key) + } + }) + + t.Run("allow_no_prefix reads raw process env when the prefix is empty", func(t *testing.T) { + t.Setenv("BARE_KEY", "bare-value") + + got, err := newProvider(t, env.Config{Prefix: "", Uppercase: true, AllowNoPrefix: true}). + Get(t.Context(), "bare_key") + require.NoError(t, err) + assert.Equal(t, "bare-value", got) + }) + + t.Run("allow_no_prefix with uppercase false uses the key verbatim", func(t *testing.T) { + t.Setenv("raw_lower", "x") + + got, err := newProvider(t, env.Config{AllowNoPrefix: true}). + Get(t.Context(), "raw_lower") + require.NoError(t, err) + assert.Equal(t, "x", got) + }) + + t.Run("allow_no_prefix still returns empty string and no error for a missing var", func(t *testing.T) { + got, err := newProvider(t, env.Config{AllowNoPrefix: true, Uppercase: true}). + Get(t.Context(), "definitely_not_set_var") + require.NoError(t, err) + assert.Equal(t, "", got) + }) + + t.Run("allow_no_prefix is inert when a prefix is set: the prefix still confines", func(t *testing.T) { + t.Setenv("TYK_SECRET_K", "confined") + t.Setenv("K", "bare") + + got, err := newProvider(t, env.Config{Prefix: "TYK_SECRET_", Uppercase: true, AllowNoPrefix: true}). + Get(t.Context(), "k") + require.NoError(t, err) + assert.Equal(t, "confined", got, + "AllowNoPrefix must not disable a configured prefix") + }) + + t.Run("allow_no_prefix json tag is wired to the raw config", func(t *testing.T) { + t.Setenv("RAW_TAG_KEY", "ok") + + p, err := env.NewFactory()(json.RawMessage(`{"allow_no_prefix":true,"uppercase":true}`)) + require.NoError(t, err) + + got, err := p.Get(t.Context(), "raw_tag_key") + require.NoError(t, err) + assert.Equal(t, "ok", got) + }) + + t.Run("prefix is literal: uppercase never applies to the prefix", func(t *testing.T) { + t.Setenv("tyk_K", "lit") + + got, err := newProvider(t, env.Config{Prefix: "tyk_", Uppercase: true}). + Get(t.Context(), "k") + require.NoError(t, err) + assert.Equal(t, "lit", got) + }) + + t.Run("value is returned byte-exact (no trailing-newline trim)", func(t *testing.T) { + t.Setenv("TYK_SECRET_RAW", "line\n") + + got, err := newProvider(t, env.Config{Prefix: "TYK_SECRET_", Uppercase: true}). + Get(t.Context(), "raw") + require.NoError(t, err) + assert.Equal(t, "line\n", got) + }) + + t.Run("empty key with a prefix set returns empty string and no error (no key guard)", func(t *testing.T) { + got, err := newProvider(t, env.Config{Prefix: "TYK_SECRET_", Uppercase: true}). + Get(t.Context(), "") + require.NoError(t, err) + assert.Equal(t, "", got) + }) +} diff --git a/kv/providers/file/errors.go b/kv/providers/file/errors.go new file mode 100644 index 00000000..2c3f782e --- /dev/null +++ b/kv/providers/file/errors.go @@ -0,0 +1,12 @@ +package file + +import "errors" + +var ( + ErrBasePathRequired = errors.New("file: base_path required") + ErrBasePathNotAbsolute = errors.New("file: base_path must be an absolute path") + ErrAbsoluteRejected = errors.New("file: absolute path rejected") + ErrTraversal = errors.New("file: path traversal detected") + ErrSymlinkEscape = errors.New("file: symlink escapes base_path") + ErrEmptyKey = errors.New("file: key must not be empty") +) diff --git a/kv/providers/file/file.go b/kv/providers/file/file.go new file mode 100644 index 00000000..b064d9b8 --- /dev/null +++ b/kv/providers/file/file.go @@ -0,0 +1,149 @@ +// Package file provides a KV provider that reads secrets from the local +// filesystem — plain files and Kubernetes Secrets mounted as files. It is a +// Standalone provider (no caching) so rotated secrets are visible on the next read. +package file + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/TykTechnologies/storage/kv" +) + +// Config is the file provider's configuration. +type Config struct { + // BasePath is the mandatory security boundary for file references. + // + // It must be an absolute path: a relative value is rejected at construction + // so the boundary's location is explicit and never depends on the process + // working directory. When set, keys must be relative paths that resolve + // within this directory — absolute keys and ".." traversal are rejected, + // and symlinks that escape the directory after resolution are rejected. + // + // When empty, the provider resolves nothing: every key is rejected, so file + // references are effectively disabled until a base_path is configured. + BasePath string `json:"base_path"` +} + +// NewFactory returns a ProviderFactory for filesystem-backed stores. +// +// An empty or absent config is accepted and yields a provider with no +// base_path; that provider rejects every Get (see Config.BasePath). This lets +// callers register the store unconditionally and treat "no base_path" as +// "file references disabled" rather than a construction error. +func NewFactory() kv.ProviderFactory { + return func(config json.RawMessage) (kv.Provider, error) { + if len(config) == 0 { + return &fileProvider{}, nil + } + + var cfg Config + + if err := json.Unmarshal(config, &cfg); err != nil { + return nil, fmt.Errorf("file: invalid config: %w", err) + } + + if cfg.BasePath != "" && !filepath.IsAbs(cfg.BasePath) { + return nil, fmt.Errorf("%w: %q", ErrBasePathNotAbsolute, cfg.BasePath) + } + + return &fileProvider{basePath: cfg.BasePath}, nil + } +} + +type fileProvider struct { + basePath string +} + +// IsStandalone reports that the provider needs no cache wrapper: filesystem +// reads are cheap. +func (fp *fileProvider) IsStandalone() bool { + return true +} + +// Get reads the file addressed by key and returns its contents with trailing +// newlines trimmed. key must be a relative path confined to base_path (see +// Config.BasePath). A missing file returns *kv.KeyNotFoundError. +func (fp *fileProvider) Get(ctx context.Context, key string) (string, error) { + path, err := resolveKeyPath(fp.basePath, key) + if err != nil { + return "", err + } + + // Resolve K8s AtomicWriter symlinks (e.g. ..data -> ..2024_01_01_00_00_00). + resolved, err := filepath.EvalSymlinks(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", &kv.KeyNotFoundError{KeyPath: key} + } + + return "", fmt.Errorf("file: cannot resolve path %q: %w", path, err) + } + + // Re-verify after symlink resolution: a symlink inside basePath can point + // outside (symlink escape). + canonicalBase, err := filepath.EvalSymlinks(fp.basePath) + // EvalSymlinks failure here requires a race (basePath symlink broken between + // resolving the file path above and this call). Not worth a flaky test. + if err != nil { + return "", fmt.Errorf("file: cannot resolve base_path %q: %w", fp.basePath, err) + } + + if !confined(canonicalBase, resolved) { + return "", fmt.Errorf("%w: key %q resolved to %q", ErrSymlinkEscape, key, resolved) + } + + data, err := os.ReadFile(resolved) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", &kv.KeyNotFoundError{KeyPath: key} + } + + return "", fmt.Errorf("file: cannot read file %q: %w", resolved, err) + } + + return strings.TrimRight(string(data), "\r\n"), nil +} + +// resolveKeyPath applies the base_path boundary policy and returns the +// candidate file path. +func resolveKeyPath(basePath, key string) (string, error) { + if basePath == "" { + return "", fmt.Errorf( + "%w: key %q (set base_path)", + ErrBasePathRequired, + key, + ) + } + + if key == "" { + return "", ErrEmptyKey + } + + if filepath.IsAbs(key) { + return "", fmt.Errorf( + "%w: %q (use a path relative to base_path)", + ErrAbsoluteRejected, + key, + ) + } + + joined := filepath.Join(basePath, key) + if !confined(basePath, joined) { + return "", fmt.Errorf("%w: key %q", ErrTraversal, key) + } + + return joined, nil +} + +// confined reports whether target resolves to a location within base, +// using lexical analysis only. +func confined(base, target string) bool { + rel, err := filepath.Rel(base, target) + return err == nil && filepath.IsLocal(rel) +} diff --git a/kv/providers/file/file_internal_test.go b/kv/providers/file/file_internal_test.go new file mode 100644 index 00000000..d759fa6a --- /dev/null +++ b/kv/providers/file/file_internal_test.go @@ -0,0 +1,35 @@ +package file + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfined(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + base string + target string + want bool + }{ + {"direct child", "/base", "/base/secret", true}, + {"nested child", "/base", "/base/sub/dir/secret", true}, + {"base itself", "/base", "/base", true}, + {"cleaned dotdot stays inside", "/base", "/base/sub/../secret", true}, + {"parent escape", "/base", "/base/../secret", false}, + {"sibling escape", "/base", "/other", false}, + {"prefix confusion is not confinement", "/base", "/base-evil", false}, + {"absolute outside", "/base", "/etc/passwd", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tc.want, confined(tc.base, tc.target)) + }) + } +} diff --git a/kv/providers/file/file_test.go b/kv/providers/file/file_test.go new file mode 100644 index 00000000..266ca88e --- /dev/null +++ b/kv/providers/file/file_test.go @@ -0,0 +1,251 @@ +package file_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/TykTechnologies/storage/kv" + "github.com/TykTechnologies/storage/kv/providers/file" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newProvider builds the provider through its factory with the given base_path, +// exactly as the registry would. +func newProvider(t *testing.T, basePath string) kv.Provider { + t.Helper() + + cfg, err := json.Marshal(file.Config{BasePath: basePath}) + require.NoError(t, err) + + p, err := file.NewFactory()(cfg) + require.NoError(t, err) + require.NotNil(t, p) + + return p +} + +func TestNewFactory(t *testing.T) { + t.Parallel() + + t.Run("valid config builds a provider", func(t *testing.T) { + t.Parallel() + + p, err := file.NewFactory()(json.RawMessage(`{"base_path":"/etc/tyk/secrets"}`)) + require.NoError(t, err) + require.NotNil(t, p) + }) + + t.Run("empty config builds a provider (file refs rejected until base_path is set)", func(t *testing.T) { + t.Parallel() + + for _, cfg := range []json.RawMessage{nil, {}, json.RawMessage(`{}`)} { + p, err := file.NewFactory()(cfg) + require.NoError(t, err, "config %q", string(cfg)) + require.NotNil(t, p) + } + }) + + t.Run("invalid JSON config errors", func(t *testing.T) { + t.Parallel() + + _, err := file.NewFactory()(json.RawMessage(`{not json`)) + require.Error(t, err) + }) + + t.Run("rejects a relative base_path", func(t *testing.T) { + t.Parallel() + + _, err := file.NewFactory()(json.RawMessage(`{"base_path":"secrets"}`)) + require.ErrorIs(t, err, file.ErrBasePathNotAbsolute) + }) +} + +func TestProviderIsStandaloner(t *testing.T) { + t.Parallel() + + p := newProvider(t, "") + + standalone, ok := kv.AsStandaloner(p) + require.True(t, ok, "file provider must implement Standalone") + require.True(t, standalone.IsStandalone(), + "file reads are cheap and must reflect rotation immediately — no cache wrapper") +} + +func TestProviderGet(t *testing.T) { + t.Parallel() + + t.Run("strips a trailing LF newline", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "secret.txt"), []byte("my-secret-value\n"), 0o600)) + + got, err := newProvider(t, dir).Get(t.Context(), "secret.txt") + require.NoError(t, err) + assert.Equal(t, "my-secret-value", got) + }) + + t.Run("strips a trailing CRLF newline", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "secret.txt"), []byte("my-secret-value\r\n"), 0o600)) + + got, err := newProvider(t, dir).Get(t.Context(), "secret.txt") + require.NoError(t, err) + assert.Equal(t, "my-secret-value", got) + }) + + t.Run("preserves internal newlines, trims only the trailing one", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + pem := "-----BEGIN CERTIFICATE-----\nMIIBkTCB+wIJ\n-----END CERTIFICATE-----\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "cert.pem"), []byte(pem), 0o600)) + + got, err := newProvider(t, dir).Get(t.Context(), "cert.pem") + require.NoError(t, err) + assert.Equal(t, "-----BEGIN CERTIFICATE-----\nMIIBkTCB+wIJ\n-----END CERTIFICATE-----", got) + }) + + t.Run("missing relative key under base_path returns kv.KeyNotFoundError", func(t *testing.T) { + t.Parallel() + + // Cross-provider consistency: an absent key is a not-found, detectable + // via errors.As regardless of which provider produced it. + dir := t.TempDir() // exists, but contains no "absent" file + _, err := newProvider(t, dir).Get(t.Context(), "absent") + + var notFound *kv.KeyNotFoundError + require.ErrorAs(t, err, ¬Found) + }) + + t.Run("base_path is mandatory: every key is rejected when it is empty", func(t *testing.T) { + t.Parallel() + + for _, key := range []string{"my-cert", "/etc/passwd"} { + _, err := newProvider(t, "").Get(t.Context(), key) + require.ErrorIs(t, err, file.ErrBasePathRequired, "key %q", key) + } + }) + + t.Run("rejects an empty key", func(t *testing.T) { + t.Parallel() + + _, err := newProvider(t, t.TempDir()).Get(t.Context(), "") + require.ErrorIs(t, err, file.ErrEmptyKey) + }) + + t.Run("resolves a relative key under base_path", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "api-key"), []byte("the-api-key"), 0o600)) + + got, err := newProvider(t, dir).Get(t.Context(), "api-key") + require.NoError(t, err) + assert.Equal(t, "the-api-key", got) + }) + + t.Run("rejects an absolute path", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + f := filepath.Join(dir, "secret") + require.NoError(t, os.WriteFile(f, []byte("abs-value"), 0o600)) + + _, err := newProvider(t, "/some/other/base").Get(t.Context(), f) + require.ErrorIs(t, err, file.ErrAbsoluteRejected) + }) + + t.Run("rejects an absolute path even when it points inside base_path", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + f := filepath.Join(dir, "secret") + require.NoError(t, os.WriteFile(f, []byte("abs-value"), 0o600)) + + _, err := newProvider(t, dir).Get(t.Context(), f) + require.ErrorIs(t, err, file.ErrAbsoluteRejected) + }) + + t.Run("rejects dotdot traversal when base_path is set", func(t *testing.T) { + t.Parallel() + + _, err := newProvider(t, t.TempDir()).Get(t.Context(), "../etc/passwd") + require.ErrorIs(t, err, file.ErrTraversal) + }) + + t.Run("rejects embedded dotdot traversal", func(t *testing.T) { + t.Parallel() + + _, err := newProvider(t, t.TempDir()).Get(t.Context(), "subdir/../../etc/passwd") + require.ErrorIs(t, err, file.ErrTraversal) + }) + + t.Run("rejects a symlink that escapes base_path after EvalSymlinks", func(t *testing.T) { + t.Parallel() + + base := t.TempDir() + target := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(target, "passwd"), []byte("root:x:0:0"), 0o600)) + // A symlink inside base/ pointing outside to target/passwd. + require.NoError(t, os.Symlink(filepath.Join(target, "passwd"), filepath.Join(base, "evil-link"))) + + _, err := newProvider(t, base).Get(t.Context(), "evil-link") + require.ErrorIs(t, err, file.ErrSymlinkEscape) + }) + + t.Run("follows K8s AtomicWriter symlinks", func(t *testing.T) { + t.Parallel() + + // Simulate a K8s secret mount: + // /..2024_01_01_00_00_00/my-key (actual data) + // /..data -> ..2024_01_01_00_00_00 + // /my-key -> ..data/my-key + dir := t.TempDir() + dataDir := filepath.Join(dir, "..2024_01_01_00_00_00") + require.NoError(t, os.Mkdir(dataDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dataDir, "my-key"), []byte("secret-from-k8s"), 0o600)) + require.NoError(t, os.Symlink("..2024_01_01_00_00_00", filepath.Join(dir, "..data"))) + require.NoError(t, os.Symlink("..data/my-key", filepath.Join(dir, "my-key"))) + + got, err := newProvider(t, dir).Get(t.Context(), "my-key") + require.NoError(t, err) + assert.Equal(t, "secret-from-k8s", got) + }) +} + +// TestProviderGet_PicksUpRotation proves the standalone (no-cache) contract: +// after a K8s AtomicWriter symlink swap, the very next Get returns the new +// value with no invalidation step. +func TestProviderGet_PicksUpRotation(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + v1 := filepath.Join(dir, "..2024_01_01_00_00_00") + require.NoError(t, os.Mkdir(v1, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(v1, "my-key"), []byte("old-secret"), 0o600)) + require.NoError(t, os.Symlink("..2024_01_01_00_00_00", filepath.Join(dir, "..data"))) + require.NoError(t, os.Symlink("..data/my-key", filepath.Join(dir, "my-key"))) + + provider := newProvider(t, dir) + + got, err := provider.Get(t.Context(), "my-key") + require.NoError(t, err) + require.Equal(t, "old-secret", got) + + // Rotate: new version dir, atomic swap of the ..data symlink. + v2 := filepath.Join(dir, "..2024_06_01_00_00_00") + require.NoError(t, os.Mkdir(v2, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(v2, "my-key"), []byte("new-secret"), 0o600)) + require.NoError(t, os.Remove(filepath.Join(dir, "..data"))) + require.NoError(t, os.Symlink("..2024_06_01_00_00_00", filepath.Join(dir, "..data"))) + + got, err = provider.Get(t.Context(), "my-key") + require.NoError(t, err) + require.Equal(t, "new-secret", got, "standalone provider must reflect rotation on the next read") +} diff --git a/kv/providers/inline/inline.go b/kv/providers/inline/inline.go new file mode 100644 index 00000000..e38248e2 --- /dev/null +++ b/kv/providers/inline/inline.go @@ -0,0 +1,72 @@ +// Package inline provides a KV provider that serves secrets from a literal +// key/value map embedded in the configuration. It is a Standalone provider (no +// caching): lookups are an in-memory map read, with nothing to refresh. +package inline + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/TykTechnologies/storage/kv" +) + +// ErrEmptyKey is returned by Get for an empty key. +var ErrEmptyKey = errors.New("inline: key must not be empty") + +// Config is the inline provider's configuration. +type Config struct { + // Data holds literal key/value secrets. + Data map[string]string `json:"data"` +} + +// NewFactory returns a ProviderFactory for inline stores. +// +// An empty or absent config is valid and yields a provider with no data: the +// store registers unconditionally and every Get is not-found until data is +// supplied. Invalid JSON returns an error. +func NewFactory() kv.ProviderFactory { + return func(config json.RawMessage) (kv.Provider, error) { + if len(config) == 0 { + return &inlineProvider{}, nil + } + + var cfg Config + if err := json.Unmarshal(config, &cfg); err != nil { + return nil, fmt.Errorf("inline: invalid config: %w", err) + } + + return &inlineProvider{data: cfg.Data}, nil + } +} + +// inlineProvider serves secrets from a map built once at construction and never +// mutated, so concurrent Get reads are safe without locking. +type inlineProvider struct { + data map[string]string +} + +// Get returns the value stored under key. +// +// An empty key returns ErrEmptyKey. A key that is absent from the map returns +// *kv.KeyNotFoundError. The lookup is exact: no prefix, no case folding. +// A key present with value "" is distinct from a missing key and returns ("", nil). +func (ip *inlineProvider) Get(_ context.Context, key string) (string, error) { + if key == "" { + return "", ErrEmptyKey + } + + v, ok := ip.data[key] + if !ok { + return "", &kv.KeyNotFoundError{KeyPath: key} + } + + return v, nil +} + +// IsStandalone reports that the provider needs no cache wrapper: inline data is +// in-memory and literal, and there is nothing to refresh. +func (ip *inlineProvider) IsStandalone() bool { + return true +} diff --git a/kv/providers/inline/inline_test.go b/kv/providers/inline/inline_test.go new file mode 100644 index 00000000..10bab84e --- /dev/null +++ b/kv/providers/inline/inline_test.go @@ -0,0 +1,140 @@ +package inline_test + +import ( + "encoding/json" + "testing" + + "github.com/TykTechnologies/storage/kv" + "github.com/TykTechnologies/storage/kv/providers/inline" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newProvider builds the provider through its factory with the given config, +// exactly as the registry would. +func newProvider(t *testing.T, cfg inline.Config) kv.Provider { + t.Helper() + + raw, err := json.Marshal(cfg) + require.NoError(t, err) + + p, err := inline.NewFactory()(raw) + require.NoError(t, err) + require.NotNil(t, p) + + return p +} + +func TestNewFactory(t *testing.T) { + t.Parallel() + + t.Run("valid config builds a provider", func(t *testing.T) { + t.Parallel() + + p, err := inline.NewFactory()(json.RawMessage(`{"data":{"token":"hvs.xxx"}}`)) + require.NoError(t, err) + require.NotNil(t, p) + }) + + t.Run("empty or absent config builds a provider with an empty map", func(t *testing.T) { + t.Parallel() + + for _, cfg := range []json.RawMessage{nil, {}, json.RawMessage(`{}`)} { + p, err := inline.NewFactory()(cfg) + require.NoError(t, err, "config %q", string(cfg)) + require.NotNil(t, p) + } + }) + + t.Run("invalid JSON config errors", func(t *testing.T) { + t.Parallel() + + _, err := inline.NewFactory()(json.RawMessage(`{not json`)) + require.Error(t, err) + }) +} + +func TestProviderIsStandalone(t *testing.T) { + t.Parallel() + + p := newProvider(t, inline.Config{}) + + standalone, ok := kv.AsStandaloner(p) + require.True(t, ok, "inline provider must implement Standalone") + require.True(t, standalone.IsStandalone(), + "inline data is in-memory and literal — no cache wrapper") +} + +func TestProviderGet(t *testing.T) { + t.Parallel() + + t.Run("returns the stored value for a present key", func(t *testing.T) { + t.Parallel() + + got, err := newProvider(t, inline.Config{Data: map[string]string{"token": "hvs.xxx"}}). + Get(t.Context(), "token") + require.NoError(t, err) + assert.Equal(t, "hvs.xxx", got) + }) + + t.Run("a key present with an empty value returns empty string and no error", func(t *testing.T) { + t.Parallel() + + got, err := newProvider(t, inline.Config{Data: map[string]string{"blank": ""}}). + Get(t.Context(), "blank") + require.NoError(t, err) + assert.Equal(t, "", got) + }) + + t.Run("a missing key returns kv.KeyNotFoundError", func(t *testing.T) { + t.Parallel() + + _, err := newProvider(t, inline.Config{Data: map[string]string{"present": "v"}}). + Get(t.Context(), "absent") + + var notFound *kv.KeyNotFoundError + require.ErrorAs(t, err, ¬Found) + assert.Equal(t, "absent", notFound.KeyPath) + }) + + t.Run("keys match exactly: no case folding", func(t *testing.T) { + t.Parallel() + + p := newProvider(t, inline.Config{Data: map[string]string{"Foo": "bar"}}) + + got, err := p.Get(t.Context(), "Foo") + require.NoError(t, err) + assert.Equal(t, "bar", got) + + _, err = p.Get(t.Context(), "foo") + var notFound *kv.KeyNotFoundError + require.ErrorAs(t, err, ¬Found, "lookup is case-sensitive, no transformation") + }) + + t.Run("values are returned verbatim: a kv:// string is not resolved", func(t *testing.T) { + t.Parallel() + + got, err := newProvider(t, inline.Config{Data: map[string]string{"ref": "kv://vault/secret#field"}}). + Get(t.Context(), "ref") + require.NoError(t, err) + assert.Equal(t, "kv://vault/secret#field", got) + }) + + t.Run("nil/empty data map yields not-found for any key without panicking", func(t *testing.T) { + t.Parallel() + + for _, cfg := range []inline.Config{{}, {Data: map[string]string{}}} { + _, err := newProvider(t, cfg).Get(t.Context(), "anything") + var notFound *kv.KeyNotFoundError + require.ErrorAs(t, err, ¬Found) + } + }) + + t.Run("empty key errors", func(t *testing.T) { + t.Parallel() + + _, err := newProvider(t, inline.Config{Data: map[string]string{"present": "v"}}). + Get(t.Context(), "") + require.ErrorIs(t, err, inline.ErrEmptyKey) + }) +} diff --git a/kv/providers/vault/config.go b/kv/providers/vault/config.go new file mode 100644 index 00000000..f026eecf --- /dev/null +++ b/kv/providers/vault/config.go @@ -0,0 +1,163 @@ +package vault + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/TykTechnologies/storage/kv" + vaultsdk "github.com/hashicorp/vault/api" +) + +// Config is the JSON "config" block of a vault store. +type Config struct { + // Address is the Vault server URL, e.g. "https://vault.example.com:8200". + // Optional: when empty the Vault client default is used (the VAULT_ADDR + // environment variable, otherwise https://127.0.0.1:8200). + Address string `json:"address"` + + // AgentAddress is the URL of a local Vault Agent, e.g. "http://127.0.0.1:8100". + // When set, the client routes requests through the agent instead of Address. + // A token is still required. + AgentAddress string `json:"agent_address"` + + // MaxRetries caps how many times the client retries a request after a + // server (5xx) error. Applied only when > 0; otherwise the Vault client + // default is kept. + MaxRetries int `json:"max_retries"` + + // Timeout bounds each Vault request. It is a Go duration string such as + // "5s" or "500ms"; an empty value means "unset", leaving the SecretStore to + // apply its own default. + Timeout string `json:"timeout"` + + // Token authenticates requests to Vault. Required. + Token string `json:"token"` + + // KVVersion selects the KV secrets engine version. Any value other than 1 + // means v2 (the default): secrets live under "/data/" and are + // wrapped in a "data" envelope. 1 selects v1, where the path is used as-is. + KVVersion int `json:"kv_version"` + + // MountPath is the path the KV secrets engine is mounted at, e.g. "secret" + // or a nested "tenants/a/kv". It is OPTIONAL and only affects KV v2. + // + // The key passed to Get is always the full logical path under this mount + // (it must start with MountPath). When set, the provider inserts the v2 + // "/data/" segment immediately after MountPath instead of assuming the mount + // is the first path segment — which is what makes nested mounts work. A key + // that is not under MountPath is rejected. + // + // When empty, the provider falls back to the legacy behavior of injecting + // "/data" after the first segment, so existing single-segment-mount configs + // are unaffected. Ignored for KV v1 (which has no data segment). + MountPath string `json:"mount_path"` +} + +// NewFactory returns a kv.ProviderFactory for HashiCorp Vault stores. +// +// The factory parses the provider Config and constructs a Vault API client, but +// performs no network I/O: the connection to Vault is established lazily on the +// first Get. It returns an error only for config that is present but unusable: +// - malformed JSON, +// - an unparseable timeout (must be a Go duration string, e.g. "5s"), +// - a missing token. Vault has no usable zero value, so a token is required +// even when agent_address is set. +// +// The resulting provider is remote: it is NOT Standalone and exposes its timeout +// via the Timeouter interface, so the registry wraps it in the caching / +// singleflight SecretStore and bounds each Get with the configured timeout. +func NewFactory() kv.ProviderFactory { + return func(raw json.RawMessage) (kv.Provider, error) { + var conf Config + if err := parseConfig(raw, &conf); err != nil { + return nil, err + } + + if err := conf.validate(); err != nil { + return nil, err + } + + timeout, err := conf.parsedTimeout() + if err != nil { + return nil, err + } + + client, err := conf.newClient(timeout) + if err != nil { + return nil, err + } + + client.SetToken(conf.Token) + + return &vaultProvider{ + client: client, + timeout: timeout, + kvv2: conf.KVVersion != 1, + // trim trailing slash(es) so "tenants/a/kv/" and "tenants/a/kv" behave the same + mountPath: strings.TrimRight(conf.MountPath, "/"), + }, nil + } +} + +func parseConfig(raw json.RawMessage, conf *Config) error { + if len(raw) == 0 { + return errors.New("vault: config is missing") + } + + if err := json.Unmarshal(raw, conf); err != nil { + return fmt.Errorf("vault: invalid config: %w", err) + } + + return nil +} + +func (conf *Config) validate() error { + if conf.Token == "" { + return errors.New("vault: token is required") + } + + return nil +} + +func (conf *Config) parsedTimeout() (time.Duration, error) { + if conf.Timeout == "" { + return 0, nil + } + + d, err := time.ParseDuration(conf.Timeout) + if err != nil { + return 0, fmt.Errorf("vault: invalid timeout %q: %w", conf.Timeout, err) + } + + return d, nil +} + +func (conf *Config) newClient(timeout time.Duration) (*vaultsdk.Client, error) { + defaultCfg := vaultsdk.DefaultConfig() + + if conf.Address != "" { + defaultCfg.Address = conf.Address + } + + if conf.AgentAddress != "" { + defaultCfg.AgentAddress = conf.AgentAddress + } + + if conf.MaxRetries > 0 { + defaultCfg.MaxRetries = conf.MaxRetries + } + + if timeout > 0 { + defaultCfg.Timeout = timeout + } + + client, err := vaultsdk.NewClient(defaultCfg) + if err != nil { + return nil, fmt.Errorf("vault: create client: %w", err) + } + + return client, nil +} diff --git a/kv/providers/vault/vault.go b/kv/providers/vault/vault.go new file mode 100644 index 00000000..92f47f75 --- /dev/null +++ b/kv/providers/vault/vault.go @@ -0,0 +1,121 @@ +// Package vault provides a kv.Provider backed by HashiCorp Vault. It reads +// secrets from Vault's KV engine (v1 or v2) and returns each secret's data map +// serialized as JSON, leaving field selection to the resolver's "#field" syntax. +// +// Unlike the local providers (env, file, inline), vault is remote: it is not +// Standalone and exposes a Timeouter, so the registry wraps it in the caching / +// singleflight SecretStore. The Vault client is created without any network I/O; +// the connection is established lazily on the first Get. +package vault + +import ( + "context" + "encoding/json" + "fmt" + "path" + "strings" + "time" + + "github.com/TykTechnologies/storage/kv" + vaultsdk "github.com/hashicorp/vault/api" +) + +// vaultProvider is a kv.Provider backed by a configured Vault API client. +type vaultProvider struct { + // client is the Vault API client. The resolved Config (address, token, + // retries, timeout) is already baked into it at construction. + client *vaultsdk.Client + + // timeout is the parsed Config.Timeout, surfaced via Timeout() so the + // SecretStore wrapper can bound each Get with it. 0 means "unset", letting + // the store use its own default. + timeout time.Duration + + // kvv2 selects the read path: when true, Get injects "/data" after the + // mount and unwraps the v2 "data" envelope; when false it reads the path + // as-is (KV v1). + kvv2 bool + + // mountPath is the normalized (trailing slash trimmed) Config.MountPath. When + // non-empty it overrides the legacy first-segment "/data" injection for KV v2 + // and confines keys to that mount. + mountPath string +} + +// Get reads the secret at key and returns its data map serialized as JSON. +// Field selection (the "#field" fragment) is the resolver's responsibility, so +// Get returns the whole secret, never a single value. When mount_path is +// configured, key must be the full logical path under that mount. +// +// A missing secret returns *kv.KeyNotFoundError; a backend or transport failure +// returns *kv.StoreUnavailableError. +func (vp *vaultProvider) Get(ctx context.Context, key string) (string, error) { + apiPath, err := vp.physicalPath(key) + if err != nil { + return "", err + } + + secret, err := vp.client.Logical().ReadWithContext(ctx, apiPath) + if err != nil { + return "", &kv.StoreUnavailableError{KeyPath: key, Err: err} + } + + if secret == nil { + return "", &kv.KeyNotFoundError{KeyPath: key} + } + + data := secret.Data + + if vp.kvv2 { + var ok bool + // KV v2 wraps the secret in an inner "data" field. Its absence means the + // path holds no v2 secret, which we treat as not-found. + data, ok = data["data"].(map[string]any) + if !ok { + return "", &kv.KeyNotFoundError{KeyPath: key} + } + } + + b, err := json.Marshal(data) + if err != nil { + return "", fmt.Errorf("vault: encode secret %q: %w", key, err) + } + + return string(b), nil +} + +func (vp *vaultProvider) Timeout() time.Duration { + return vp.timeout +} + +// physicalPath maps the caller's logical key to the physical path the Vault HTTP +// API expects. For KV v1 the logical and physical paths are identical. For KV v2 +// the engine stores secrets under "/data/", so the "/data/" segment +// is inserted after the mount: +// +// - With mount_path set, immediately after that (possibly multi-segment) mount. +// key must be the full logical path under the mount; otherwise it is rejected. +// The check is on the path-segment boundary (mountPath + "/"), so a mount of +// "secret" does not match an unrelated "secrets/..." key. +// - With mount_path empty, after the first path segment — the legacy assumption +// that the mount is a single top-level segment. The injection is intentionally +// blind: a segment legitimately named "data" must not be special-cased. +func (vp *vaultProvider) physicalPath(key string) (string, error) { + if !vp.kvv2 { + return key, nil + } + + if vp.mountPath != "" { + clean := path.Clean(key) + if !strings.HasPrefix(clean, vp.mountPath+"/") { + return "", fmt.Errorf("vault: key %q is not under mount_path %q", key, vp.mountPath) + } + + return vp.mountPath + "/data" + strings.TrimPrefix(clean, vp.mountPath), nil + } + + splitted := strings.Split(key, "/") + splitted[0] += "/data" + + return strings.Join(splitted, "/"), nil +} diff --git a/kv/providers/vault/vault_test.go b/kv/providers/vault/vault_test.go new file mode 100644 index 00000000..22708b39 --- /dev/null +++ b/kv/providers/vault/vault_test.go @@ -0,0 +1,540 @@ +package vault_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/TykTechnologies/storage/kv" + "github.com/TykTechnologies/storage/kv/providers/vault" + "github.com/TykTechnologies/storage/kv/registry" + "github.com/TykTechnologies/storage/kv/resolver" + "github.com/stretchr/testify/require" +) + +// How these tests work: +// +// The provider is exercised through the real github.com/hashicorp/vault/api +// client against an in-process HTTP server (vaultStub) that stands in for Vault. +// Each test points Config.Address (or AgentAddress) at the stub's URL, so the +// client builds genuine Vault requests and the stub returns canned KV responses. +// +// This means the tests cover the actual client wiring — URL/path construction, +// the KV v2 "/data" injection (asserted via the request path the stub records), +// token handling, and response parsing — rather than a mock of it, while staying +// hermetic and millisecond-fast. +// +// Note on t.Parallel: every test clears the VAULT_* environment via clearVaultEnv +// (t.Setenv) so a developer's shell can't perturb client construction. t.Setenv is +// incompatible with t.Parallel by design, so these tests are intentionally serial +// — the suite runs in well under a second, so there is nothing to gain from +// parallelism anyway. + +// clearVaultEnv blanks the VAULT_* environment so client construction is +// hermetic. +func clearVaultEnv(t *testing.T) { + t.Helper() + + for _, k := range []string{ + "VAULT_ADDR", "VAULT_AGENT_ADDR", "VAULT_TOKEN", + "VAULT_CACERT", "VAULT_CAPATH", "VAULT_CLIENT_CERT", + "VAULT_CLIENT_KEY", "VAULT_SKIP_VERIFY", "VAULT_TLS_SERVER_NAME", + "VAULT_MAX_RETRIES", "VAULT_CLIENT_TIMEOUT", "VAULT_RATE_LIMIT", + } { + t.Setenv(k, "") + } +} + +// vaultStub is an httptest server that records the requests it receives and +// delegates response construction to a per-test handler. +type vaultStub struct { + url string + mu sync.Mutex + got []string +} + +// requests returns a copy of the recorded "METHOD /path" entries. +func (s *vaultStub) requests() []string { + s.mu.Lock() + defer s.mu.Unlock() + + return append([]string(nil), s.got...) +} + +func newVaultStub(t *testing.T, handler http.HandlerFunc) *vaultStub { + t.Helper() + + s := &vaultStub{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.got = append(s.got, r.Method+" "+r.URL.Path) + s.mu.Unlock() + + handler(w, r) + })) + t.Cleanup(srv.Close) + + s.url = srv.URL + + return s +} + +func writeJSON(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + + if body != nil { + //nolint:errcheck + _ = json.NewEncoder(w).Encode(body) + } +} + +// kvv2Envelope wraps secret data the way Vault's KV v2 engine does on the wire: +// {"data": {"data": , "metadata": {...}}}. +func kvv2Envelope(data map[string]any) map[string]any { + return map[string]any{ + "data": map[string]any{ + "data": data, + "metadata": map[string]any{"version": 1}, + }, + } +} + +// kvv1Envelope wraps secret data the way Vault's KV v1 engine does: +// {"data": }. +func kvv1Envelope(data map[string]any) map[string]any { + return map[string]any{"data": data} +} + +// newVaultProvider builds the provider through its factory, exactly as the +// registry would, with a hermetic environment. +func newVaultProvider(t *testing.T, cfg *vault.Config) kv.Provider { + t.Helper() + + clearVaultEnv(t) + + raw, err := json.Marshal(cfg) + require.NoError(t, err) + + p, err := vault.NewFactory()(raw) + require.NoError(t, err) + require.NotNil(t, p) + + return p +} + +func mustJSON(t *testing.T, v any) json.RawMessage { + t.Helper() + + b, err := json.Marshal(v) + require.NoError(t, err) + + return b +} + +func TestNewFactory(t *testing.T) { + tests := []struct { + name string + config string + wantErr bool + wantErrContains string + }{ + { + name: "valid full config", + config: `{"address":"http://vault.test:8200","token":"root","kv_version":2}`, + }, + { + name: "token only (address is optional)", + config: `{"token":"root"}`, + }, + { + name: "agent_address with token", + config: `{"agent_address":"http://127.0.0.1:8100","token":"root"}`, + }, + + // Vault has no usable zero value: a token is required even in agent mode, + // so every tokenless config is rejected. + { + name: "empty bytes", + config: ``, + wantErr: true, + }, + { + name: "empty object", + config: `{}`, + wantErr: true, + }, + { + name: "address without token", + config: `{"address":"http://vault.test:8200"}`, + wantErr: true, + }, + { + name: "agent_address without token", + config: `{"agent_address":"http://127.0.0.1:8100"}`, + wantErr: true, + }, + { + name: "address and agent_address without token", + config: `{"address":"http://vault.test:8200","agent_address":"http://127.0.0.1:8100"}`, + wantErr: true, + }, + + // Present-but-invalid config. The factory namespaces these with "vault:". + { + name: "invalid json", + config: `{not json`, + wantErr: true, + wantErrContains: "vault", + }, + { + name: "invalid timeout (not a duration string)", + config: `{"token":"root","timeout":"5x"}`, + wantErr: true, + wantErrContains: "vault", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clearVaultEnv(t) + + p, err := vault.NewFactory()(json.RawMessage(tt.config)) + + if tt.wantErr { + require.Error(t, err) + require.Nil(t, p) + + if tt.wantErrContains != "" { + require.ErrorContains(t, err, tt.wantErrContains) + } + + return + } + + require.NoError(t, err) + require.NotNil(t, p) + }) + } +} + +func TestProvider_ReportsConfiguredTimeout(t *testing.T) { + p := newVaultProvider(t, &vault.Config{Token: "root", Timeout: "7s"}) + + to, ok := kv.AsTimeouter(p) + require.True(t, ok, "vault must expose Timeouter so the registry can bound its operations") + require.Equal(t, 7*time.Second, to.Timeout()) +} + +func TestProvider_TimeoutUnsetReportsZero(t *testing.T) { + p := newVaultProvider(t, &vault.Config{Token: "root"}) + + to, ok := kv.AsTimeouter(p) + require.True(t, ok) + require.Zero(t, to.Timeout(), "an unset timeout must report 0 so the store applies its own default") +} + +func TestProvider_IsNotStandaloner(t *testing.T) { + p := newVaultProvider(t, &vault.Config{Token: "root"}) + + // Vault is remote and must be wrapped in the registry's cache/singleflight + // decorator, so it must NOT report itself standaloner. + s, ok := kv.AsStandaloner(p) + require.False(t, ok && s.IsStandalone(), + "vault must not be standalone (the registry must wrap it in the cache)") +} + +func TestGet_ReadsSecret(t *testing.T) { + tests := []struct { + name string + key string + kvVersion int + secret map[string]any + wantPath string + wantJSON string + }{ + { + name: "kv2 multi-segment path returns the whole data map", + key: "secret/myapp/config", + kvVersion: 2, + secret: map[string]any{"api_key": "abc123", "username": "bob"}, + wantPath: "GET /v1/secret/data/myapp/config", + wantJSON: `{"api_key":"abc123","username":"bob"}`, + }, + { + name: "kv2 default version (0) injects /data", + key: "secret/myapp/config", + kvVersion: 0, + secret: map[string]any{"api_key": "abc123"}, + wantPath: "GET /v1/secret/data/myapp/config", + wantJSON: `{"api_key":"abc123"}`, + }, + { + name: "kv2 single-segment path injects /data after the mount", + key: "mysecret", + kvVersion: 2, + secret: map[string]any{"api_key": "abc123"}, + wantPath: "GET /v1/mysecret/data", + wantJSON: `{"api_key":"abc123"}`, + }, + { + name: "kv1 reads the path as-is with no unwrap", + key: "secret/myapp", + kvVersion: 1, + secret: map[string]any{"api_key": "abc123"}, + wantPath: "GET /v1/secret/myapp", + wantJSON: `{"api_key":"abc123"}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stub := newVaultStub(t, func(w http.ResponseWriter, _ *http.Request) { + if tt.kvVersion == 1 { + writeJSON(w, http.StatusOK, kvv1Envelope(tt.secret)) + return + } + + writeJSON(w, http.StatusOK, kvv2Envelope(tt.secret)) + }) + + p := newVaultProvider(t, &vault.Config{ + Address: stub.url, + Token: "root", + KVVersion: tt.kvVersion, + }) + + got, err := p.Get(t.Context(), tt.key) + require.NoError(t, err) + require.JSONEq(t, tt.wantJSON, got) + require.Equal(t, []string{tt.wantPath}, stub.requests()) + }) + } +} + +func TestGet_MountPath(t *testing.T) { + tests := []struct { + name string + mountPath string + key string + kvVersion int + wantPath string + wantErr bool + }{ + { + name: "kv2 nested mount injects /data after the configured mount", + mountPath: "tenants/a/kv", + key: "tenants/a/kv/myapp/config", + kvVersion: 2, + wantPath: "GET /v1/tenants/a/kv/data/myapp/config", + }, + { + name: "kv2 single-segment mount matches the legacy result", + mountPath: "secret", + key: "secret/myapp", + kvVersion: 2, + wantPath: "GET /v1/secret/data/myapp", + }, + { + name: "kv2 trailing slash on mount_path is normalized", + mountPath: "tenants/a/kv/", + key: "tenants/a/kv/myapp", + kvVersion: 2, + wantPath: "GET /v1/tenants/a/kv/data/myapp", + }, + { + name: "kv1 ignores mount_path and reads the path as-is", + mountPath: "secret", + key: "secret/myapp", + kvVersion: 1, + wantPath: "GET /v1/secret/myapp", + }, + { + name: "kv2 key outside the mount is rejected before any request", + mountPath: "tenants/a/kv", + key: "other/secret", + kvVersion: 2, + wantErr: true, + }, + { + name: "kv2 key sharing a string prefix but not under the mount is rejected", + mountPath: "secret", + key: "secrets/myapp", + kvVersion: 2, + wantErr: true, + }, + { + name: "kv2 key using ../ to escape the mount is rejected", + mountPath: "tenants/a/kv", + key: "tenants/a/kv/../../b/kv/secret", + kvVersion: 2, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stub := newVaultStub(t, func(w http.ResponseWriter, _ *http.Request) { + if tt.kvVersion == 1 { + writeJSON(w, http.StatusOK, kvv1Envelope(map[string]any{"k": "v"})) + return + } + + writeJSON(w, http.StatusOK, kvv2Envelope(map[string]any{"k": "v"})) + }) + + p := newVaultProvider(t, &vault.Config{ + Address: stub.url, + Token: "root", + KVVersion: tt.kvVersion, + MountPath: tt.mountPath, + }) + + _, err := p.Get(t.Context(), tt.key) + + if tt.wantErr { + require.Error(t, err) + require.Empty(t, stub.requests(), + "a key outside mount_path must be rejected before any Vault request") + + return + } + + require.NoError(t, err) + require.Equal(t, []string{tt.wantPath}, stub.requests()) + }) + } +} + +func TestGet_ReturnsCompactJSONWithoutTrailingNewline(t *testing.T) { + stub := newVaultStub(t, func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, kvv2Envelope(map[string]any{"api_key": "abc123"})) + }) + + p := newVaultProvider(t, &vault.Config{Address: stub.url, Token: "root", KVVersion: 2}) + + got, err := p.Get(t.Context(), "secret/myapp/config") + require.NoError(t, err) + require.Equal(t, `{"api_key":"abc123"}`, got) +} + +func TestGet_SecretNotFoundReturnsKeyNotFound(t *testing.T) { + stub := newVaultStub(t, func(w http.ResponseWriter, _ *http.Request) { + // Empty-body 404: vault's client returns (nil, nil) for a missing path. + w.WriteHeader(http.StatusNotFound) + }) + + p := newVaultProvider(t, &vault.Config{Address: stub.url, Token: "root", KVVersion: 2}) + + _, err := p.Get(t.Context(), "secret/missing") + + var notFound *kv.KeyNotFoundError + require.ErrorAs(t, err, ¬Found, + "a missing secret must map to *kv.KeyNotFoundError for negative_ttl_not_found caching") +} + +func TestGet_KVv2MissingDataEnvelopeReturnsKeyNotFound(t *testing.T) { + stub := newVaultStub(t, func(w http.ResponseWriter, _ *http.Request) { + // 200, but the inner "data" key is absent (only metadata) → the KVv2 + // data.data unwrap fails. + writeJSON(w, http.StatusOK, map[string]any{ + "data": map[string]any{"metadata": map[string]any{"version": 1}}, + }) + }) + + p := newVaultProvider(t, &vault.Config{Address: stub.url, Token: "root", KVVersion: 2}) + + _, err := p.Get(t.Context(), "secret/myapp/config") + + var notFound *kv.KeyNotFoundError + require.ErrorAs(t, err, ¬Found) +} + +func TestGet_BackendErrorReturnsStoreUnavailable(t *testing.T) { + stub := newVaultStub(t, func(w http.ResponseWriter, _ *http.Request) { + // A 200 is not retried by the vault client, so an unparseable body fails + // fast and deterministically exercises the transport/parse error branch + // (avoids the multi-second retry backoff a 5xx would incur). + w.WriteHeader(http.StatusOK) + + _, err := w.Write([]byte("{ this is not valid vault json")) + if err != nil { + t.Error(err) + } + }) + + p := newVaultProvider(t, &vault.Config{Address: stub.url, Token: "root", KVVersion: 2}) + + _, err := p.Get(t.Context(), "secret/myapp/config") + + var unavailable *kv.StoreUnavailableError + require.ErrorAs(t, err, &unavailable, + "a backend failure must map to *kv.StoreUnavailableError for negative_ttl_transient caching") +} + +func TestGet_PropagatesContextCancellation(t *testing.T) { + stub := newVaultStub(t, func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, kvv2Envelope(map[string]any{"api_key": "abc123"})) + }) + + p := newVaultProvider(t, &vault.Config{Address: stub.url, Token: "root", KVVersion: 2}) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + _, err := p.Get(ctx, "secret/myapp/config") + require.ErrorIs(t, err, context.Canceled) +} + +func TestGet_AgentModeRoutesToAgentAddress(t *testing.T) { + agent := newVaultStub(t, func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, kvv2Envelope(map[string]any{"api_key": "abc123"})) + }) + + // agent_address (with a token, per the legacy contract) must route requests + // to the agent, not the default server address. + p := newVaultProvider(t, &vault.Config{AgentAddress: agent.url, Token: "root", KVVersion: 2}) + + got, err := p.Get(t.Context(), "secret/myapp/config") + require.NoError(t, err) + require.JSONEq(t, `{"api_key":"abc123"}`, got) + require.NotEmpty(t, agent.requests(), "the request must be routed to the Vault agent address") +} + +func TestResolver_ExtractsFieldFromVaultSecret(t *testing.T) { + clearVaultEnv(t) + + stub := newVaultStub(t, func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, kvv2Envelope(map[string]any{ + "api_key": "abc123", + "password": "s3cr3t", + })) + }) + + r := registry.NewRegistry() + require.NoError(t, r.Add(kv.Vault, vault.NewFactory())) + + ctx := context.Background() + require.NoError(t, r.InitStores(ctx, &kv.Config{ + Stores: map[string]kv.StoreConfig{ + "vault": { + Type: kv.Vault, + Config: mustJSON(t, vault.Config{Address: stub.url, Token: "root", KVVersion: 2}), + }, + }, + })) + t.Cleanup(func() { _ = r.Close(ctx) }) + + res := resolver.NewResolver(r) + + // End-to-end: the provider returns the whole secret as JSON, the resolver + // extracts the #field via JSON pointer. This pins the core contract change + // (field extraction lives in the resolver, not the provider). + got, err := res.Resolve(ctx, "kv://vault/secret/myapp/config#api_key") + require.NoError(t, err) + require.Equal(t, "abc123", got) +} diff --git a/kv/registry/from_config.go b/kv/registry/from_config.go new file mode 100644 index 00000000..46adb5cb --- /dev/null +++ b/kv/registry/from_config.go @@ -0,0 +1,214 @@ +package registry + +import ( + "context" + "encoding/json" + "fmt" + "maps" + + "github.com/TykTechnologies/storage/kv" + "github.com/TykTechnologies/storage/kv/internal/resolve" +) + +// initOptions collects the inputs to NewFromConfig that aren't the raw +// config document. +type initOptions struct { + factories map[kv.ProviderType]kv.ProviderFactory + defaultStores map[string]kv.StoreConfig + logger kv.Logger +} + +// InitOption configures NewFromConfig. +type InitOption func(*initOptions) + +// WithFactories injects additional provider factories (enterprise or custom) +// on top of the OSS defaults. A nil map is a no-op — the OSS build path. +func WithFactories(f map[kv.ProviderType]kv.ProviderFactory) InitOption { + return func(o *initOptions) { + o.factories = f + } +} + +// WithDefaultStores supplies store definitions merged with LOWER precedence +// than the kv.stores block in rawConfig — a store defined in both places +// uses the rawConfig definition. +func WithDefaultStores(s map[string]kv.StoreConfig) InitOption { + return func(o *initOptions) { + o.defaultStores = s + } +} + +// WithInitLogger sets the logger that receives warnings emitted during +// registry initialization. +func WithInitLogger(l kv.Logger) InitOption { + return func(o *initOptions) { + o.logger = l + } +} + +// NewFromConfig initializes a Registry from the kv section of rawConfig. +// Internally it runs a two-phase process so that store credentials (e.g. a +// Vault token) can themselves live in an env or inline store and be +// referenced from the config. +// +// Referenceability rules: +// +// - A store config may reference any local-type store by name — +// wherever it is defined (WithDefaultStores or kv.stores) — because all +// local-type stores initialize before store configs are resolved. +// - A store config may never reference a remote store (vault, consul, +// aws, ...): remote stores initialize concurrently, so this is an +// unresolvable bootstrap cycle. Such references reach the provider +// factory as literals and fail there, subject to the store's +// required flag. +// - Values inside an inline store's data map are always literal; a +// kv:// reference there is never resolved. +// +// NewFromConfig does not resolve the rest of the document — the single +// strict ResolveAll pass over the full config is owned by the caller. +// A nil or empty rawConfig is valid: the kv-section parse is skipped and +// the registry is built from WithDefaultStores alone. +// +// The caller owns the returned registry, including Close on shutdown. +func NewFromConfig( + ctx context.Context, + rawConfig []byte, + opts ...InitOption, +) (*Registry, error) { + // A store's own config can contain kv:// references — e.g. a Vault store + // whose token lives in an env var: {"token": "kv://env/VAULT_TOKEN"}. + // Resolving that requires the env store to already exist, which forces a + // two-phase init: + // + // Phase 1 — stand up the local-type stores (env, inline, file: literal + // config, no network) in a throwaway registry, and use them to resolve + // references inside the OTHER store configs. + // Phase 2 — build the real registry and open every store, now that all + // configs are resolved. Local stores re-init cheaply here; remote + // stores open their connections for the first time. + options := &initOptions{} + for _, opt := range opts { + opt(options) + } + + var config struct { + KV kv.Config `json:"kv"` + } + + if len(rawConfig) > 0 { + if err := json.Unmarshal(rawConfig, &config); err != nil { + return nil, fmt.Errorf("kv: parse config: %w", err) + } + } + + merged := make(map[string]kv.StoreConfig, len(options.defaultStores)+len(config.KV.Stores)) + maps.Copy(merged, options.defaultStores) + maps.Copy(merged, config.KV.Stores) + + // Phase 1: resolve references in store configs against local stores. + if err := resolveStoreConfigReferences(ctx, merged, options.factories, options.logger); err != nil { + return nil, err + } + + // Phase 2: build the registry and initialize all stores. + full, err := newRegistryWithFactories(options.factories, options.logger) + if err != nil { + return nil, err + } + + if len(merged) > 0 { + err := full.InitStores(ctx, &kv.Config{Stores: merged, Cache: config.KV.Cache}) + if err != nil { + return nil, fmt.Errorf("kv: initialize stores: %w", err) + } + } + + return full, nil +} + +// resolveStoreConfigReferences is Phase 1. It initializes the local-type stores +// from merged into a TEMPORARY registry, then resolves kv:// references found +// inside the remaining (remote) store configs against them, rewriting merged in +// place. The temporary registry is discarded; Phase 2 rebuilds everything. +// +// Resolution is lenient: a reference to a store absent from the temporary +// registry — i.e. a remote store, which only exists after Phase 2 — is left +// untouched instead of failing. It reaches the provider factory as a literal in +// Phase 2 and fails there if the store was required. This is why a remote store +// config cannot reference another remote store. +func resolveStoreConfigReferences( + ctx context.Context, + merged map[string]kv.StoreConfig, + factories map[kv.ProviderType]kv.ProviderFactory, + logger kv.Logger, +) error { + locals := make(map[string]kv.StoreConfig) + var hasRemotes bool + + for name, storeCfg := range merged { + if storeCfg.Type.IsLocal() { + locals[name] = storeCfg + } else { + hasRemotes = true + } + } + + // Only remote configs can carry references worth resolving; local configs + // are literal. If there are none, skip the bootstrap entirely. + if !hasRemotes { + return nil + } + + bootstrap, err := newRegistryWithFactories(factories, logger) + if err != nil { + return err + } + + defer func() { + _ = bootstrap.Close(context.WithoutCancel(ctx)) + }() + + if len(locals) > 0 { + if err := bootstrap.InitStores(ctx, &kv.Config{Stores: locals}); err != nil { + return fmt.Errorf("kv: initialize bootstrap local stores: %w", err) + } + } + + lenient := resolve.NewResolver(bootstrap, resolve.WithLenientMode()) + + for name, storeCfg := range merged { + // Local store configs are literal and are initialized from the raw parse in + // Phase 1 (env/inline/file alike), so there is nothing to resolve here. + // Skip them, and skip stores with no config blob (ResolveAll(nil) errors). + if storeCfg.Type.IsLocal() || len(storeCfg.Config) == 0 { + continue + } + + resolved, err := lenient.ResolveAll(ctx, storeCfg.Config) + if err != nil { + return fmt.Errorf("kv: resolve config of store %q: %w", name, err) + } + + storeCfg.Config = resolved + merged[name] = storeCfg + } + + return nil +} + +func newRegistryWithFactories(factories map[kv.ProviderType]kv.ProviderFactory, logger kv.Logger) (*Registry, error) { + var rOpts []Option + if logger != nil { + rOpts = append(rOpts, WithLogger(logger)) + } + + r := NewDefaultRegistry(rOpts...) + + for providerType, factory := range factories { + if err := r.set(providerType, factory); err != nil { + return nil, fmt.Errorf("kv: register factory: %w", err) + } + } + + return r, nil +} diff --git a/kv/registry/from_config_test.go b/kv/registry/from_config_test.go new file mode 100644 index 00000000..bd6c870c --- /dev/null +++ b/kv/registry/from_config_test.go @@ -0,0 +1,820 @@ +package registry_test + +import ( + "context" + "encoding/json" + "errors" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/TykTechnologies/storage/kv" + "github.com/TykTechnologies/storage/kv/registry" + "github.com/TykTechnologies/storage/kv/resolver" + "github.com/stretchr/testify/require" +) + +// fakeProvider serves a fixed data map. A nil map means every key resolves +// to "value". +type fakeProvider struct { + data map[string]string + initFunc func(ctx context.Context) error + calls atomic.Int32 + closed atomic.Bool +} + +func (p *fakeProvider) Get(_ context.Context, key string) (string, error) { + p.calls.Add(1) + + if p.data == nil { + return "value", nil + } + + v, ok := p.data[key] + if !ok { + return "", &kv.KeyNotFoundError{StoreName: "fake", KeyPath: key} + } + + return v, nil +} + +func (p *fakeProvider) Init(ctx context.Context) error { + if p.initFunc != nil { + return p.initFunc(ctx) + } + + return nil +} + +func (p *fakeProvider) Close(_ context.Context) error { + p.closed.Store(true) + return nil +} + +// configRecorder captures every config a factory receives. +type configRecorder struct { + mu sync.Mutex + configs []json.RawMessage +} + +func (r *configRecorder) record(cfg json.RawMessage) { + r.mu.Lock() + defer r.mu.Unlock() + + r.configs = append(r.configs, cfg) +} + +func (r *configRecorder) all() []json.RawMessage { + r.mu.Lock() + defer r.mu.Unlock() + + return append([]json.RawMessage(nil), r.configs...) +} + +// single asserts the factory was called exactly once and returns the config. +func (r *configRecorder) single(t *testing.T) json.RawMessage { + t.Helper() + + configs := r.all() + require.Len(t, configs, 1) + + return configs[0] +} + +func recordingFactory(rec *configRecorder, p kv.Provider, err error) kv.ProviderFactory { + return func(cfg json.RawMessage) (kv.Provider, error) { + if rec != nil { + rec.record(cfg) + } + + if err != nil { + return nil, err + } + + return p, nil + } +} + +func dataFactory(rec *configRecorder) kv.ProviderFactory { + return func(cfg json.RawMessage) (kv.Provider, error) { + if rec != nil { + rec.record(cfg) + } + + var parsed struct { + Data map[string]string `json:"data"` + } + + if err := json.Unmarshal(cfg, &parsed); err != nil { + return nil, err + } + + return &fakeProvider{data: parsed.Data}, nil + } +} + +func tokenOf(t *testing.T, cfg json.RawMessage) string { + t.Helper() + + var parsed struct { + Token string `json:"token"` + } + require.NoError(t, json.Unmarshal(cfg, &parsed)) + + return parsed.Token +} + +func newRegistry(t *testing.T, rawConfig []byte, opts ...registry.InitOption) *registry.Registry { + t.Helper() + + reg, err := registry.NewFromConfig(t.Context(), rawConfig, opts...) + require.NoError(t, err) + require.NotNil(t, reg) + t.Cleanup(func() { + _ = reg.Close(context.WithoutCancel(t.Context())) + }) + + return reg +} + +type mockLogger struct { + debugCalls atomic.Int32 + warnCalls atomic.Int32 + errorCalls atomic.Int32 +} + +func (l *mockLogger) Debug(_ string, _ map[string]any) { + l.debugCalls.Add(1) +} + +func (l *mockLogger) Warn(_ string, _ map[string]any) { + l.warnCalls.Add(1) +} + +func (l *mockLogger) Error(_ string, _ map[string]any) { + l.errorCalls.Add(1) +} + +func TestNewFromConfigInitializesStoresFromKVSection(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "listen_port": 8080, + "kv": { + "stores": { + "my-values": {"type": "inline", "config": {"data": {"greeting": "hello"}}}, + "vault": {"type": "hashicorp_vault", "config": {"token": "literal-token"}} + } + } + }`) + + reg := newRegistry(t, doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Inline: dataFactory(nil), + kv.Vault: recordingFactory(nil, &fakeProvider{}, nil), + })) + + inline, err := reg.GetStore("my-values") + require.NoError(t, err) + + got, err := inline.Get(t.Context(), "greeting") + require.NoError(t, err) + require.Equal(t, "hello", got) + + _, err = reg.GetStore("vault") + require.NoError(t, err) + + _, err = reg.GetStore("unknown") + require.ErrorIs(t, err, kv.ErrStoreNotFound) +} + +func TestNewFromConfigPhase1Resolution(t *testing.T) { + t.Parallel() + + t.Run("remote config referencing an env store is resolved before the factory runs", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "env": {"type": "env", "config": {"data": {"VAULT_TOKEN": "hvs.from-env"}}}, + "vault": {"type": "hashicorp_vault", "config": {"token": "kv://env/VAULT_TOKEN"}} + } + } + }`) + + vaultRec := &configRecorder{} + newRegistry(t, doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Env: dataFactory(nil), + kv.Vault: recordingFactory(vaultRec, &fakeProvider{}, nil), + })) + + require.Equal(t, "hvs.from-env", tokenOf(t, vaultRec.single(t))) + }) + + t.Run("remote config referencing a user-defined inline store is resolved", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "my-values": {"type": "inline", "config": {"data": {"vault_token": "hvs.from-inline"}}}, + "vault": {"type": "hashicorp_vault", "config": {"token": "kv://my-values/vault_token"}} + } + } + }`) + + vaultRec := &configRecorder{} + newRegistry(t, doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Inline: dataFactory(nil), + kv.Vault: recordingFactory(vaultRec, &fakeProvider{}, nil), + })) + + require.Equal(t, "hvs.from-inline", tokenOf(t, vaultRec.single(t))) + }) + + t.Run("remote config referencing a file store is resolved", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "vault": {"type": "hashicorp_vault", "config": {"token": "kv://file/vault_token"}} + } + } + }`) + + vaultRec := &configRecorder{} + newRegistry( + t, doc, + registry.WithDefaultStores(map[string]kv.StoreConfig{ + "file": {Type: kv.File, Required: true, Config: json.RawMessage(`{"base_path": "/etc/some"}`)}, + }), + registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Vault: recordingFactory(vaultRec, &fakeProvider{}, nil), + kv.File: recordingFactory(nil, &fakeProvider{data: map[string]string{ + "vault_token": "random_vault_token", + }}, nil), + }), + ) + + require.Equal(t, "random_vault_token", tokenOf(t, vaultRec.single(t))) + }) + + t.Run("references inside WithDefaultStores configs are resolved too", func(t *testing.T) { + t.Parallel() + + // The promoted-legacy-store shape: the default vault store's token + // references an env store defined in the user's kv.stores block. + doc := []byte(`{ + "kv": { + "stores": { + "env": {"type": "env", "config": {"data": {"VAULT_TOKEN": "hvs.for-default"}}} + } + } + }`) + + vaultRec := &configRecorder{} + newRegistry( + t, doc, + registry.WithDefaultStores(map[string]kv.StoreConfig{ + "vault": {Type: kv.Vault, Required: true, Config: json.RawMessage(`{"token": "kv://env/VAULT_TOKEN"}`)}, + }), + registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Env: dataFactory(nil), + kv.Vault: recordingFactory(vaultRec, &fakeProvider{}, nil), + }), + ) + + require.Equal(t, "hvs.for-default", tokenOf(t, vaultRec.single(t))) + }) + + t.Run("inline store data values are never resolved", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "env": {"type": "env", "config": {"data": {"REAL": "resolved-value"}}}, + "my-values": {"type": "inline", "config": {"data": {"keep": "kv://env/REAL"}}} + } + } + }`) + + inlineRec := &configRecorder{} + reg := newRegistry(t, doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Env: dataFactory(nil), + kv.Inline: dataFactory(inlineRec), + })) + + // The literal must survive end to end: in every config the inline + // factory ever received, and through Get on the initialized store. + for _, cfg := range inlineRec.all() { + var parsed struct { + Data map[string]string `json:"data"` + } + require.NoError(t, json.Unmarshal(cfg, &parsed)) + require.Equal(t, "kv://env/REAL", parsed.Data["keep"]) + } + + store, err := reg.GetStore("my-values") + require.NoError(t, err) + + got, err := store.Get(t.Context(), "keep") + require.NoError(t, err) + require.Equal(t, "kv://env/REAL", got) + }) +} + +func TestNewFromConfigUnresolvableReferences(t *testing.T) { + t.Parallel() + + // errsOnUnresolved simulates a real remote provider rejecting a config + // whose credential is still a kv:// literal. + errsOnUnresolved := func(rec *configRecorder) kv.ProviderFactory { + return func(cfg json.RawMessage) (kv.Provider, error) { + rec.record(cfg) + + var parsed struct { + Token string `json:"token"` + } + + if err := json.Unmarshal(cfg, &parsed); err != nil { + return nil, err + } + + if len(parsed.Token) >= 5 && parsed.Token[:5] == "kv://" { + return nil, errors.New("invalid token: looks like an unresolved KV reference") + } + + return &fakeProvider{}, nil + } + } + + t.Run("remote-to-remote reference passes through and fails a required store", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "consul": {"type": "hashicorp_consul", "config": {"token": "literal"}}, + "vault": {"type": "hashicorp_vault", "required": true, "config": {"token": "kv://consul/token"}} + } + } + }`) + + vaultRec := &configRecorder{} + reg, err := registry.NewFromConfig(t.Context(), doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Consul: recordingFactory(nil, &fakeProvider{}, nil), + kv.Vault: errsOnUnresolved(vaultRec), + })) + require.Error(t, err) + require.Nil(t, reg) + + // Phase 1 must have left the remote-to-remote reference verbatim. + require.Equal(t, "kv://consul/token", tokenOf(t, vaultRec.single(t))) + }) + + t.Run("optional store with unresolvable reference is skipped, others initialize", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "consul": {"type": "hashicorp_consul", "config": {"token": "literal"}}, + "vault": {"type": "hashicorp_vault", "required": false, "config": {"token": "kv://absent-store/token"}} + } + } + }`) + + vaultRec := &configRecorder{} + reg := newRegistry(t, doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Consul: recordingFactory(nil, &fakeProvider{}, nil), + kv.Vault: errsOnUnresolved(vaultRec), + })) + + require.Equal(t, "kv://absent-store/token", tokenOf(t, vaultRec.single(t))) + + _, err := reg.GetStore("vault") + require.ErrorIs(t, err, kv.ErrStoreNotFound) + + _, err = reg.GetStore("consul") + require.NoError(t, err) + }) + + t.Run("malformed reference in a store config fails fast, remote factory never runs", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "vault": {"type": "hashicorp_vault", "config": {"token": "kv://no-path-separator"}} + } + } + }`) + + vaultRec := &configRecorder{} + reg, err := registry.NewFromConfig(t.Context(), doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Vault: recordingFactory(vaultRec, &fakeProvider{}, nil), + })) + require.ErrorIs(t, err, resolver.ErrMalformedReference) + require.Nil(t, reg) + require.Empty(t, vaultRec.all(), "Phase 1 must fail before any remote factory is invoked") + }) +} + +func TestNewFromConfigDefaultStoresMerge(t *testing.T) { + t.Parallel() + + t.Run("nil rawConfig builds the registry from defaults alone", func(t *testing.T) { + t.Parallel() + + reg := newRegistry( + t, nil, + registry.WithDefaultStores(map[string]kv.StoreConfig{ + "env": {Type: kv.Env, Required: true, Config: json.RawMessage(`{"data": {"KEY": "from-default"}}`)}, + }), + registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Env: dataFactory(nil), + }), + ) + + store, err := reg.GetStore("env") + require.NoError(t, err) + + got, err := store.Get(t.Context(), "KEY") + require.NoError(t, err) + require.Equal(t, "from-default", got) + }) + + t.Run("kv.stores definition wins over a same-named default store", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "vault": {"type": "hashicorp_vault", "config": {"token": "from-config"}} + } + } + }`) + + vaultRec := &configRecorder{} + newRegistry( + t, doc, + registry.WithDefaultStores(map[string]kv.StoreConfig{ + "vault": {Type: kv.Vault, Config: json.RawMessage(`{"token": "from-defaults"}`)}, + }), + registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Vault: recordingFactory(vaultRec, &fakeProvider{}, nil), + }), + ) + + require.Equal(t, "from-config", tokenOf(t, vaultRec.single(t)), + "user-defined kv.stores entry must override the promoted default") + }) +} + +func TestNewFromConfigDocumentHandling(t *testing.T) { + t.Parallel() + + t.Run("references outside the kv section are ignored", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "secret": "kv://not-a-store/key", + "broken": "kv://no-path-separator", + "kv": { + "stores": { + "my-values": {"type": "inline", "config": {"data": {"k": "v"}}} + } + } + }`) + + reg := newRegistry(t, doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Inline: dataFactory(nil), + })) + + _, err := reg.GetStore("my-values") + require.NoError(t, err) + }) + + t.Run("empty and kv-less documents build a defaults-only registry", func(t *testing.T) { + t.Parallel() + + for _, doc := range [][]byte{ + nil, + {}, + []byte(`{}`), + []byte(`{"listen_port": 8080}`), + []byte(`{"kv": {}}`), + } { + reg := newRegistry( + t, doc, + registry.WithDefaultStores(map[string]kv.StoreConfig{ + "env": {Type: kv.Env, Config: json.RawMessage(`{"data": {}}`)}, + }), + registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Env: dataFactory(nil), + }), + ) + + _, err := reg.GetStore("env") + require.NoError(t, err, "doc: %q", doc) + } + }) + + t.Run("no options and nil config still yield a usable empty registry", func(t *testing.T) { + t.Parallel() + + reg, err := registry.NewFromConfig(t.Context(), nil) + require.NoError(t, err) + require.NotNil(t, reg) + + _, err = reg.GetStore("anything") + require.ErrorIs(t, err, kv.ErrStoreNotFound) + }) + + t.Run("nil WithFactories map is accepted", func(t *testing.T) { + t.Parallel() + + reg, err := registry.NewFromConfig(t.Context(), nil, registry.WithFactories(nil)) + require.NoError(t, err) + require.NotNil(t, reg) + }) + + t.Run("invalid JSON errors", func(t *testing.T) { + t.Parallel() + + reg, err := registry.NewFromConfig(t.Context(), []byte(`{not json`)) + require.Error(t, err) + require.Nil(t, reg) + }) +} + +func TestNewFromConfigUnknownProviderType(t *testing.T) { + t.Parallel() + + t.Run("required store with unknown type fails initialization", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "mystery": {"type": "does_not_exist", "required": true, "config": {}} + } + } + }`) + + reg, err := registry.NewFromConfig(t.Context(), doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Inline: dataFactory(nil), + })) + require.Error(t, err) + require.Contains(t, err.Error(), "does_not_exist") + require.Nil(t, reg) + }) + + t.Run("optional store with unknown type is skipped, others initialize", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "mystery": {"type": "does_not_exist", "required": false, "config": {}}, + "my-values": {"type": "inline", "config": {"data": {"k": "v"}}} + } + } + }`) + + reg := newRegistry(t, doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Inline: dataFactory(nil), + })) + + _, err := reg.GetStore("mystery") + require.ErrorIs(t, err, kv.ErrStoreNotFound) + + _, err = reg.GetStore("my-values") + require.NoError(t, err) + }) +} + +func TestNewFromConfigCacheConfig(t *testing.T) { + t.Parallel() + + t.Run("kv.cache settings reach the store wrapper", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "cache": {"enabled": true, "ttl": "1m"}, + "stores": { + "remote": {"type": "hashicorp_vault", "config": {}} + } + } + }`) + + provider := &fakeProvider{} + reg := newRegistry(t, doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Vault: recordingFactory(nil, provider, nil), + })) + + store, err := reg.GetStore("remote") + require.NoError(t, err) + + for range 3 { + _, err = store.Get(t.Context(), "some/key") + require.NoError(t, err) + } + + require.Equal(t, int32(1), provider.calls.Load(), "enabled cache must serve repeat Gets") + }) + + t.Run("without cache config every Get reaches the provider", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "remote": {"type": "hashicorp_vault", "config": {}} + } + } + }`) + + provider := &fakeProvider{} + reg := newRegistry(t, doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Vault: recordingFactory(nil, provider, nil), + })) + + store, err := reg.GetStore("remote") + require.NoError(t, err) + + for range 3 { + _, err = store.Get(t.Context(), "some/key") + require.NoError(t, err) + } + + require.Equal(t, int32(3), provider.calls.Load()) + }) +} + +func TestNewFromConfigLifecycle(t *testing.T) { + t.Run("context cancellation propagates and partial stores are cleaned up", func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + doc := []byte(`{ + "kv": { + "stores": { + "blocked": {"type": "hashicorp_vault", "required": true, "config": {}}, + "healthy": {"type": "hashicorp_consul", "required": true, "config": {}} + } + } + }`) + + healthy := &fakeProvider{} + blocked := &fakeProvider{ + initFunc: func(ctx context.Context) error { + <-ctx.Done() + return ctx.Err() + }, + } + + ctx, cancel := context.WithCancel(t.Context()) + go func() { + time.Sleep(time.Second) + cancel() + }() + + reg, err := registry.NewFromConfig(ctx, doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Vault: recordingFactory(nil, blocked, nil), + kv.Consul: recordingFactory(nil, healthy, nil), + })) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, reg) + + synctest.Wait() + require.True(t, healthy.closed.Load(), "stores initialized before the failure must be closed") + }) + }) + + t.Run("Close shuts down every initialized store", func(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "a": {"type": "hashicorp_vault", "config": {}}, + "b": {"type": "hashicorp_consul", "config": {}} + } + } + }`) + + providerA := &fakeProvider{} + providerB := &fakeProvider{} + + reg, err := registry.NewFromConfig(t.Context(), doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Vault: recordingFactory(nil, providerA, nil), + kv.Consul: recordingFactory(nil, providerB, nil), + })) + require.NoError(t, err) + + require.NoError(t, reg.Close(t.Context())) + require.True(t, providerA.closed.Load()) + require.True(t, providerB.closed.Load()) + + _, err = reg.GetStore("a") + require.ErrorIs(t, err, kv.ErrStoreNotFound) + }) +} + +func TestNewFromConfigWithFactoriesOverridesDefault(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "file": {"type": "file", "config": {}} + } + } + }`) + + override := &fakeProvider{data: map[string]string{"my-key": "from-override"}} + + reg := newRegistry(t, doc, registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.File: recordingFactory(nil, override, nil), + })) + + store, err := reg.GetStore("file") + require.NoError(t, err) + + got, err := store.Get(t.Context(), "my-key") + require.NoError(t, err) + require.Equal(t, "from-override", got, + "the WithFactories kv.File factory must override the OSS default, not error on collision") +} + +func TestNewFromConfigRejectsInvalidFactories(t *testing.T) { + t.Parallel() + + t.Run("empty provider type", func(t *testing.T) { + t.Parallel() + + reg, err := registry.NewFromConfig(t.Context(), nil, registry.WithFactories( + map[kv.ProviderType]kv.ProviderFactory{ + "": recordingFactory(nil, &fakeProvider{}, nil), + }, + )) + require.Error(t, err) + require.Nil(t, reg) + require.ErrorContains(t, err, "provider type cannot be empty") + }) + + t.Run("nil factory", func(t *testing.T) { + t.Parallel() + + reg, err := registry.NewFromConfig(t.Context(), nil, registry.WithFactories( + map[kv.ProviderType]kv.ProviderFactory{ + kv.AWS: nil, + }, + )) + require.Error(t, err) + require.Nil(t, reg) + require.ErrorContains(t, err, "factory cannot be nil") + }) +} + +func TestNewFromConfig_WithLogger_ForwardsSkipWarning(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "mystery": {"type": "does_not_exist", "required": false, "config": {}} + } + } + }`) + + l := &mockLogger{} + + reg, err := registry.NewFromConfig(t.Context(), doc, + registry.WithInitLogger(l), + ) + require.NoError(t, err, "an optional store with an unknown type is skipped, not fatal") + require.NotNil(t, reg) + + require.Positive(t, l.warnCalls.Load()) +} + +func TestNewFromConfig_NoLogger_DoesNotPanic(t *testing.T) { + t.Parallel() + + doc := []byte(`{ + "kv": { + "stores": { + "mystery": {"type": "does_not_exist", "required": false, "config": {}} + } + } + }`) + + reg, err := registry.NewFromConfig(t.Context(), doc) + require.NoError(t, err) + require.NotNil(t, reg) +} diff --git a/kv/registry/integration_test.go b/kv/registry/integration_test.go new file mode 100644 index 00000000..4ee96e6f --- /dev/null +++ b/kv/registry/integration_test.go @@ -0,0 +1,447 @@ +package registry_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/TykTechnologies/storage/kv" + "github.com/TykTechnologies/storage/kv/providers/env" + "github.com/TykTechnologies/storage/kv/registry" + "github.com/TykTechnologies/storage/kv/resolver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func promotedDefaults(t *testing.T, basePath string, secrets map[string]string) map[string]kv.StoreConfig { + t.Helper() + + envCfg, err := json.Marshal(map[string]any{"prefix": "TYK_SECRET_", "uppercase": true}) + require.NoError(t, err) + + stores := map[string]kv.StoreConfig{ + "env": {Type: kv.Env, Config: envCfg}, + } + + if basePath != "" { + fileCfg, err := json.Marshal(map[string]any{"base_path": basePath}) + require.NoError(t, err) + + stores["file"] = kv.StoreConfig{Type: kv.File, Config: fileCfg} + } + + if secrets != nil { + inlineCfg, err := json.Marshal(map[string]any{"data": secrets}) + require.NoError(t, err) + + stores["secrets"] = kv.StoreConfig{Type: kv.Inline, Config: inlineCfg} + } + + return stores +} + +func clearVaultEnv(t *testing.T) { + t.Helper() + + for _, k := range []string{ + "VAULT_ADDR", "VAULT_AGENT_ADDR", "VAULT_TOKEN", + "VAULT_CACERT", "VAULT_CAPATH", "VAULT_CLIENT_CERT", + "VAULT_CLIENT_KEY", "VAULT_SKIP_VERIFY", "VAULT_TLS_SERVER_NAME", + } { + t.Setenv(k, "") + } +} + +func clearConsulEnv(t *testing.T) { + t.Helper() + + for _, k := range []string{ + "CONSUL_HTTP_ADDR", "CONSUL_HTTP_TOKEN", "CONSUL_HTTP_TOKEN_FILE", + "CONSUL_HTTP_AUTH", "CONSUL_HTTP_SSL", "CONSUL_HTTP_SSL_VERIFY", + "CONSUL_CACERT", "CONSUL_CAPATH", "CONSUL_CLIENT_CERT", + "CONSUL_CLIENT_KEY", "CONSUL_TLS_SERVER_NAME", "CONSUL_NAMESPACE", + } { + t.Setenv(k, "") + } +} + +func consulKVResponse(w http.ResponseWriter, key, value string) { + w.Header().Set("Content-Type", "application/json") + + //nolint:errcheck + _ = json.NewEncoder(w).Encode([]struct { + Key string `json:"Key"` + Value []byte `json:"Value"` + }{ + {Key: key, Value: []byte(value)}, + }) +} + +func consulAddr(url string) string { + return url[len("http://"):] +} + +// TestIntegrationResolveAllLocalProviders resolves a whole config document that +// mixes every local provider, both reference syntaxes, and a JSON-pointer +// fragment — proving the providers cooperate through the resolver exactly as the +// caller expects. +func TestIntegrationResolveAllLocalProviders(t *testing.T) { + t.Setenv("TYK_SECRET_DB_PASSWORD", "s3cret") + + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "tls.crt"), []byte("CERTDATA\n"), 0o600)) + + secrets := map[string]string{ + "api_key": "inline-key", + "blob": `{"user":"admin","pass":"pw"}`, + } + + reg := newRegistry(t, nil, registry.WithDefaultStores(promotedDefaults(t, dir, secrets))) + res := resolver.NewResolver(reg) + + doc := []byte(`{ + "db_password": "kv://env/db_password", + "api_key": "kv://secrets/api_key", + "cert": "kv://file/tls.crt", + "url": "https://$kv{secrets:blob#user}.example.com", + "admin_pass": "kv://secrets/blob#pass", + "missing_env": "kv://env/NOT_SET_ANYWHERE" + }`) + + out, err := res.ResolveAll(t.Context(), doc) + require.NoError(t, err) + + var got struct { + DBPassword string `json:"db_password"` + APIKey string `json:"api_key"` + Cert string `json:"cert"` + URL string `json:"url"` + AdminPass string `json:"admin_pass"` + MissingEnv string `json:"missing_env"` + } + require.NoError(t, json.Unmarshal(out, &got)) + + assert.Equal(t, "s3cret", got.DBPassword, "env: TYK_SECRET_ prefix + uppercase") + assert.Equal(t, "inline-key", got.APIKey, "inline: whole-value lookup") + assert.Equal(t, "CERTDATA", got.Cert, "file: read with trailing newline trimmed") + assert.Equal(t, "https://admin.example.com", got.URL, "inline + $kv{} inline token + #fragment") + assert.Equal(t, "pw", got.AdminPass, "inline: kv:// whole-value with #fragment") + assert.Equal(t, "", got.MissingEnv, "env: missing variable resolves to empty, no error") +} + +func TestIntegrationInlineMissingKeyIsFatal(t *testing.T) { + t.Parallel() + + reg := newRegistry(t, nil, registry.WithDefaultStores( + promotedDefaults(t, "", map[string]string{"present": "v"}), + )) + res := resolver.NewResolver(reg) + + _, err := res.Resolve(t.Context(), "kv://secrets/absent") + + var notFound *kv.KeyNotFoundError + require.ErrorAs(t, err, ¬Found) + assert.Equal(t, "absent", notFound.KeyPath) +} + +func TestIntegrationEnvMissingKeyIsNotFatal(t *testing.T) { + t.Parallel() + + reg := newRegistry(t, nil, registry.WithDefaultStores(promotedDefaults(t, "", nil))) + res := resolver.NewResolver(reg) + + got, err := res.Resolve(t.Context(), "kv://env/UNSET_OPTIONAL_SECRET") + require.NoError(t, err) + assert.Equal(t, "", got) +} + +func TestIntegrationPhase1ResolvesRemoteTokenFromRealEnv(t *testing.T) { + t.Setenv("TYK_SECRET_VAULT_TOKEN", "hvs.real") + + doc := []byte(`{ + "kv": { + "stores": { + "vault": {"type": "hashicorp_vault", "config": {"token": "kv://env/VAULT_TOKEN"}} + } + } + }`) + + rec := &configRecorder{} + reg := newRegistry(t, doc, + registry.WithDefaultStores(promotedDefaults(t, "", nil)), + registry.WithFactories(map[kv.ProviderType]kv.ProviderFactory{ + kv.Vault: recordingFactory(rec, &fakeProvider{}, nil), + }), + ) + + _, err := reg.GetStore("vault") + require.NoError(t, err) + assert.Equal(t, "hvs.real", tokenOf(t, rec.single(t)), + "real env provider must resolve the vault token in Phase 1") +} + +func TestIntegrationEnvEmptyPrefixRejectedThroughStack(t *testing.T) { + t.Parallel() + + openEnv, err := json.Marshal(map[string]any{"prefix": "", "uppercase": true}) + require.NoError(t, err) + + reg := newRegistry(t, nil, registry.WithDefaultStores(map[string]kv.StoreConfig{ + "openenv": {Type: kv.Env, Config: openEnv}, + })) + res := resolver.NewResolver(reg) + + _, err = res.Resolve(t.Context(), "kv://openenv/PATH") + require.ErrorIs(t, err, env.ErrPrefixRequired) +} + +func TestIntegrationVaultProviderResolvesThroughDefaultRegistry(t *testing.T) { + clearVaultEnv(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + // KV v2 envelope: {"data": {"data": , "metadata": {...}}}. + //nolint:errcheck + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "data": map[string]any{"password": "s3cr3t"}, + "metadata": map[string]any{"version": 1}, + }, + }) + })) + t.Cleanup(srv.Close) + + doc := []byte(fmt.Sprintf(`{ + "kv": { + "stores": { + "vault": { + "type": "hashicorp_vault", + "required": true, + "config": {"address": %q, "token": "root", "kv_version": 2} + } + } + } + }`, srv.URL)) + + reg := newRegistry(t, doc) + res := resolver.NewResolver(reg) + + got, err := res.Resolve(t.Context(), "kv://vault/secret/myapp#password") + require.NoError(t, err) + assert.Equal(t, "s3cr3t", got) +} + +func TestIntegrationVaultStoreServesRepeatedReadsFromCache(t *testing.T) { + clearVaultEnv(t) + + var hits atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + + w.Header().Set("Content-Type", "application/json") + //nolint:errcheck + _ = json.NewEncoder(w).Encode(map[string]any{ + "data": map[string]any{ + "data": map[string]any{"password": "s3cr3t"}, + "metadata": map[string]any{"version": 1}, + }, + }) + })) + t.Cleanup(srv.Close) + + doc := []byte(fmt.Sprintf(`{ + "kv": { + "cache": {"enabled": true, "ttl": "1m"}, + "stores": { + "vault": { + "type": "hashicorp_vault", + "required": true, + "config": {"address": %q, "token": "root", "kv_version": 2} + } + } + } + }`, srv.URL)) + + reg := newRegistry(t, doc) + res := resolver.NewResolver(reg) + + for range 3 { + got, err := res.Resolve(t.Context(), "kv://vault/secret/myapp#password") + require.NoError(t, err) + assert.Equal(t, "s3cr3t", got) + } + + assert.Equal(t, int32(1), hits.Load(), + "3 resolves of the same key must hit the vault backend only once (SecretStore cache)") +} + +func TestIntegrationConsulProviderResolvesThroughDefaultRegistry(t *testing.T) { + clearConsulEnv(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + consulKVResponse(w, "services/redis", `{"host":"cache01","port":"6379"}`) + })) + t.Cleanup(srv.Close) + + doc := []byte(fmt.Sprintf(`{ + "kv": { + "stores": { + "consul": { + "type": "hashicorp_consul", + "required": true, + "config": {"address": %q} + } + } + } + }`, consulAddr(srv.URL))) + + reg := newRegistry(t, doc) + res := resolver.NewResolver(reg) + + whole, err := res.Resolve(t.Context(), "kv://consul/services/redis") + require.NoError(t, err) + assert.Equal(t, `{"host":"cache01","port":"6379"}`, whole) + + got, err := res.Resolve(t.Context(), "kv://consul/services/redis#host") + require.NoError(t, err) + assert.Equal(t, "cache01", got) +} + +func TestIntegrationConsulStoreServesRepeatedReadsFromCache(t *testing.T) { + clearConsulEnv(t) + + var hits atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + consulKVResponse(w, "services/redis", "cache01") + })) + t.Cleanup(srv.Close) + + doc := []byte(fmt.Sprintf(`{ + "kv": { + "cache": {"enabled": true, "ttl": "1m"}, + "stores": { + "consul": { + "type": "hashicorp_consul", + "required": true, + "config": {"address": %q} + } + } + } + }`, consulAddr(srv.URL))) + + reg := newRegistry(t, doc) + res := resolver.NewResolver(reg) + + for range 3 { + got, err := res.Resolve(t.Context(), "kv://consul/services/redis") + require.NoError(t, err) + assert.Equal(t, "cache01", got) + } + + assert.Equal(t, int32(1), hits.Load(), + "3 resolves of the same key must hit the consul backend only once (SecretStore cache)") +} + +func TestIntegrationConsulStoreNegativeCachesNotFound(t *testing.T) { + clearConsulEnv(t) + + var hits atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + + doc := []byte(fmt.Sprintf(`{ + "kv": { + "cache": {"enabled": true, "ttl": "1m", "negative_ttl_not_found": "1m"}, + "stores": { + "consul": { + "type": "hashicorp_consul", + "required": true, + "config": {"address": %q} + } + } + } + }`, consulAddr(srv.URL))) + + reg := newRegistry(t, doc) + res := resolver.NewResolver(reg) + + for range 3 { + _, err := res.Resolve(t.Context(), "kv://consul/services/absent") + + var notFound *kv.KeyNotFoundError + require.ErrorAs(t, err, ¬Found) + } + + assert.Equal(t, int32(1), hits.Load(), + "a not-found must be negatively cached (negative_ttl_not_found bucket), not re-fetched") +} + +// BenchmarkVaultStoreGet measures a Get against a real vault provider talking +// to a co-located (in-process httptest) KVv2 backend, through the full +// production stack: NewFromConfig registry → SecretStore wrapper → vault API +// client → HTTP. The cache-off run is the true cost of one backend round trip +// (the "co-located provider" startup criterion); the cache-on run is the +// steady-state request path. +func BenchmarkVaultStoreGet(b *testing.B) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + // nolint:errcheck + _, _ = w.Write([]byte(`{"data":{"data":{"password":"hunter2"}}}`)) + })) + defer srv.Close() + + ctx := context.Background() + + for _, tc := range []struct { + name string + cacheEnabled bool + }{ + {name: "cache-off", cacheEnabled: false}, + {name: "cache-on", cacheEnabled: true}, + } { + rawConfig := fmt.Sprintf(`{ + "kv": { + "cache": {"enabled": %t, "ttl": "1h"}, + "stores": { + "vault": { + "type": "hashicorp_vault", + "config": {"address": %q, "token": "bench-token", "kv_version": 2} + } + } + } + }`, tc.cacheEnabled, srv.URL) + + reg, err := registry.NewFromConfig(ctx, []byte(rawConfig)) + if err != nil { + b.Fatal(err) + } + + store, err := reg.GetStore("vault") + if err != nil { + b.Fatal(err) + } + + b.Run(tc.name, func(b *testing.B) { + for b.Loop() { + if _, err := store.Get(ctx, "secret/bench"); err != nil { + b.Fatal(err) + } + } + }) + + _ = reg.Close(ctx) + } +} diff --git a/kv/registry/registry.go b/kv/registry/registry.go index d9814bc1..45383933 100644 --- a/kv/registry/registry.go +++ b/kv/registry/registry.go @@ -10,6 +10,11 @@ import ( "github.com/TykTechnologies/storage/kv" "github.com/TykTechnologies/storage/kv/internal/store" + "github.com/TykTechnologies/storage/kv/providers/consul" + "github.com/TykTechnologies/storage/kv/providers/env" + "github.com/TykTechnologies/storage/kv/providers/file" + "github.com/TykTechnologies/storage/kv/providers/inline" + "github.com/TykTechnologies/storage/kv/providers/vault" "golang.org/x/sync/errgroup" ) @@ -29,6 +34,7 @@ type Registry struct { type Option func(r *Registry) +// WithLogger sets the logger the registry emits warnings. func WithLogger(l kv.Logger) Option { return func(r *Registry) { if l != nil { @@ -56,6 +62,41 @@ func NewRegistry(opts ...Option) *Registry { func NewDefaultRegistry(opts ...Option) *Registry { r := NewRegistry(opts...) + err := r.Add(kv.File, file.NewFactory()) + if err != nil { + r.logger.Warn("Failed to add default file factory", map[string]any{ + "error": err, + }) + } + + err = r.Add(kv.Env, env.NewFactory()) + if err != nil { + r.logger.Warn("Failed to add default env factory", map[string]any{ + "error": err, + }) + } + + err = r.Add(kv.Inline, inline.NewFactory()) + if err != nil { + r.logger.Warn("Failed to add default inline factory", map[string]any{ + "error": err, + }) + } + + err = r.Add(kv.Vault, vault.NewFactory()) + if err != nil { + r.logger.Warn("Failed to add default vault factory", map[string]any{ + "error": err, + }) + } + + err = r.Add(kv.Consul, consul.NewFactory()) + if err != nil { + r.logger.Warn("Failed to add default consul factory", map[string]any{ + "error": err, + }) + } + return r } @@ -230,6 +271,24 @@ func (r *Registry) commitStores(tempStores map[string]kv.Provider) error { return nil } +// set registers a factory, replacing any existing one. Unline Add, it permits +// override - used by the composition layer so WithFactories wins over OSS defaults. +func (r *Registry) set(pt kv.ProviderType, factory kv.ProviderFactory) error { + if pt == "" { + return errors.New("provider type cannot be empty") + } + + if factory == nil { + return errors.New("factory cannot be nil") + } + + r.mu.Lock() + defer r.mu.Unlock() + r.factories[pt] = factory + + return nil +} + // closeStores closes any store that implements Closer, using a cancellation-free // context so cleanup runs even when the original context is already done. func closeStores(ctx context.Context, stores map[string]kv.Provider) { @@ -275,7 +334,7 @@ func (r *Registry) Close(ctx context.Context) error { if closer, ok := kv.AsCloser(store); ok { if err := closer.Close(ctx); err != nil { mu.Lock() - errs = append(errs, fmt.Errorf("failed to close store %q: %w", name, err)) + errs = append(errs, fmt.Errorf("close store %q: %w", name, err)) mu.Unlock() } } @@ -296,13 +355,13 @@ func buildSingleStore( ) (kv.Provider, error) { provider, err := factory(storeCfg.Config) if err != nil { - return nil, fmt.Errorf("failed to create provider %q (type: %s): %w", name, storeCfg.Type, err) + return nil, fmt.Errorf("create provider for %q store: %w", name, err) } if initializer, ok := kv.AsInitializer(provider); ok { err := initializer.Init(ctx) if err != nil { - return nil, fmt.Errorf("failed to initialize store %q (type: %s): %w", name, storeCfg.Type, err) + return nil, fmt.Errorf("initialize store %q: %w", name, err) } } @@ -322,7 +381,7 @@ func buildSingleStore( store.WithTimeout(timeout), ) if err != nil { - return nil, fmt.Errorf("failed to wrap store %q: %w", name, err) + return nil, err } return ss, nil diff --git a/kv/registry/registry_test.go b/kv/registry/registry_test.go index 4bf26289..db99c6cd 100644 --- a/kv/registry/registry_test.go +++ b/kv/registry/registry_test.go @@ -48,21 +48,21 @@ func (m *mockProvider) IsStandalone() bool { } type mockLogger struct { - debugCalls int - warnCalls int - errorCalls int + debugCalls atomic.Int32 + warnCalls atomic.Int32 + errorCalls atomic.Int32 } func (l *mockLogger) Debug(_ string, _ map[string]any) { - l.debugCalls++ + l.debugCalls.Add(1) } func (l *mockLogger) Warn(_ string, _ map[string]any) { - l.warnCalls++ + l.warnCalls.Add(1) } func (l *mockLogger) Error(_ string, _ map[string]any) { - l.errorCalls++ + l.errorCalls.Add(1) } func newFactory(initFunc, closeFunc func(ctx context.Context) error) kv.ProviderFactory { @@ -83,7 +83,16 @@ func TestNewRegistry(t *testing.T) { require.NotNil(t, registry.factories) } -func TestNewDefaultRegistry(t *testing.T) {} +func TestNewDefaultRegistry(t *testing.T) { + r := NewDefaultRegistry() + + require.NotNil(t, r) + require.NotEmpty(t, r.factories[kv.File]) + require.NotEmpty(t, r.factories[kv.Env]) + require.NotEmpty(t, r.factories[kv.Inline]) + require.NotEmpty(t, r.factories[kv.Vault]) + require.NotEmpty(t, r.factories[kv.Consul]) +} func TestAddFactory(t *testing.T) { t.Parallel() @@ -332,7 +341,7 @@ func TestInitStores_EdgeCases(t *testing.T) { }, }) require.Error(t, err) - require.Contains(t, err.Error(), "failed to initialize store") + require.Contains(t, err.Error(), "initialize store") if validInitialized { require.True( @@ -363,7 +372,7 @@ func TestInitStores_EdgeCases(t *testing.T) { }, }) require.Error(t, err) - require.Contains(t, err.Error(), "failed to wrap store") + require.Contains(t, err.Error(), `secret store "valid-1"`) require.False(t, r.isInitialized.Load()) }) @@ -384,7 +393,7 @@ func TestInitStores_EdgeCases(t *testing.T) { }, }) require.NoError(t, err) - require.Equal(t, 1, l.warnCalls) + require.Equal(t, int32(1), l.warnCalls.Load()) }) t.Run("should skip secret store wrapping if provider is standalone", func(t *testing.T) {