Skip to content

Commit 26b3ad6

Browse files
Copilotintel352
Copilot
and
intel352
authored
feat: Add step.db_query_cached — single-step DB query with in-process TTL caching (#228)
* Initial plan * feat: add step.db_query_cached pipeline step with in-process TTL caching Co-authored-by: intel352 <77607+intel352@users.noreply.github.com> * feat: Add step.db_query_cached — single-step DB query with in-process TTL caching Co-authored-by: intel352 <77607+intel352@users.noreply.github.com> * fix: address review comments - stampede protection, TTL validation, schema clarity, test coverage Co-authored-by: intel352 <77607+intel352@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: intel352 <77607+intel352@users.noreply.github.com>
1 parent 17abf60 commit 26b3ad6

9 files changed

Lines changed: 714 additions & 24 deletions

File tree

‎cmd/wfctl/type_registry.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -586,6 +586,11 @@ func KnownStepTypes() map[string]StepTypeInfo {
586586
Plugin: "pipelinesteps",
587587
ConfigKeys: []string{"database", "query", "params"},
588588
},
589+
"step.db_query_cached": {
590+
Type: "step.db_query_cached",
591+
Plugin: "pipelinesteps",
592+
ConfigKeys: []string{"database", "query", "params", "cache_key", "cache_ttl", "scan_fields"},
593+
},
589594
"step.json_response": {
590595
Type: "step.json_response",
591596
Plugin: "pipelinesteps",

‎example/go.mod‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ require (
2222
github.com/BurntSushi/toml v1.6.0 // indirect
2323
github.com/CrisisTextLine/modular/modules/auth v0.4.0 // indirect
2424
github.com/CrisisTextLine/modular/modules/cache v0.4.0 // indirect
25-
github.com/CrisisTextLine/modular/modules/eventbus/v2 v2.0.0 // indirect
25+
github.com/CrisisTextLine/modular/modules/eventbus/v2 v2.1.0 // indirect
2626
github.com/CrisisTextLine/modular/modules/jsonschema v1.4.0 // indirect
2727
github.com/CrisisTextLine/modular/modules/reverseproxy/v2 v2.2.0 // indirect
2828
github.com/CrisisTextLine/modular/modules/scheduler v0.4.0 // indirect
@@ -31,7 +31,7 @@ require (
3131
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
3232
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
3333
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
34-
github.com/IBM/sarama v1.46.3 // indirect
34+
github.com/IBM/sarama v1.47.0 // indirect
3535
github.com/Microsoft/go-winio v0.6.2 // indirect
3636
github.com/aws/aws-sdk-go-v2 v1.41.2 // indirect
3737
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
@@ -77,7 +77,6 @@ require (
7777
github.com/docker/go-units v0.5.0 // indirect
7878
github.com/dustin/go-humanize v1.0.1 // indirect
7979
github.com/eapache/go-resiliency v1.7.0 // indirect
80-
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
8180
github.com/eapache/queue v1.1.0 // indirect
8281
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
8382
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
@@ -95,7 +94,6 @@ require (
9594
github.com/gogo/protobuf v1.3.2 // indirect
9695
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
9796
github.com/golang/protobuf v1.5.4 // indirect
98-
github.com/golang/snappy v0.0.4 // indirect
9997
github.com/golobby/cast v1.3.3 // indirect
10098
github.com/google/gnostic-models v0.6.8 // indirect
10199
github.com/google/go-cmp v0.7.0 // indirect
@@ -133,7 +131,7 @@ require (
133131
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
134132
github.com/josharian/intern v1.0.0 // indirect
135133
github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12 // indirect
136-
github.com/klauspost/compress v1.18.3 // indirect
134+
github.com/klauspost/compress v1.18.4 // indirect
137135
github.com/mailru/easyjson v0.7.7 // indirect
138136
github.com/mattn/go-isatty v0.0.20 // indirect
139137
github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -149,7 +147,7 @@ require (
149147
github.com/ncruces/go-strftime v1.0.0 // indirect
150148
github.com/opencontainers/go-digest v1.0.0 // indirect
151149
github.com/opencontainers/image-spec v1.1.1 // indirect
152-
github.com/pierrec/lz4/v4 v4.1.22 // indirect
150+
github.com/pierrec/lz4/v4 v4.1.25 // indirect
153151
github.com/pkg/errors v0.9.1 // indirect
154152
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
155153
github.com/prometheus/client_golang v1.19.1 // indirect
@@ -184,7 +182,7 @@ require (
184182
go.uber.org/zap v1.27.0 // indirect
185183
golang.org/x/crypto v0.48.0 // indirect
186184
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
187-
golang.org/x/net v0.50.0 // indirect
185+
golang.org/x/net v0.51.0 // indirect
188186
golang.org/x/oauth2 v0.35.0 // indirect
189187
golang.org/x/sync v0.19.0 // indirect
190188
golang.org/x/sys v0.41.0 // indirect

‎example/go.sum‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ github.com/CrisisTextLine/modular/modules/cache v0.4.0 h1:vlPXAsucSM1M0RsPly9cWy
3232
github.com/CrisisTextLine/modular/modules/cache v0.4.0/go.mod h1:4irZOGXxUlgJqAnWlpMyPC3C1tM/f5145/wMThYnAsY=
3333
github.com/CrisisTextLine/modular/modules/eventbus v1.7.0 h1:SSeu7rjuECDgFa+iNyndn94YPQxffHxJgfR7U4psz6E=
3434
github.com/CrisisTextLine/modular/modules/eventbus v1.7.0/go.mod h1:I1tGf3DmadwyMP2NE2m6XHYl9ebXB9wBc/KZLywTR4c=
35-
github.com/CrisisTextLine/modular/modules/eventbus/v2 v2.0.0 h1:bDNWBparvVzXnhLxjFPJ9MDg7N4NUnNOjfn56G/CwGU=
36-
github.com/CrisisTextLine/modular/modules/eventbus/v2 v2.0.0/go.mod h1:5DmacIYrhhiN18i2OHyAVBiNKbN2jHuEv2UJoRToMg0=
35+
github.com/CrisisTextLine/modular/modules/eventbus/v2 v2.1.0 h1:jCG/5cuCITnGH4ztOrU5vY00+ykP9j+RL0zXy/CX1ak=
36+
github.com/CrisisTextLine/modular/modules/eventbus/v2 v2.1.0/go.mod h1:5DmacIYrhhiN18i2OHyAVBiNKbN2jHuEv2UJoRToMg0=
3737
github.com/CrisisTextLine/modular/modules/jsonschema v1.4.0 h1:NIhTrDgjhGwMi2D0ukGsd3n/M1W807u6Rhlqm89Sj8Q=
3838
github.com/CrisisTextLine/modular/modules/jsonschema v1.4.0/go.mod h1:TeM3mt/+1X5VmlWF4nZpgp4qCGPmAahQs5jAzuWLbOo=
3939
github.com/CrisisTextLine/modular/modules/reverseproxy/v2 v2.2.0 h1:SUJEPA61IbjdUwKdSembQTbX9rKz5v4vmyr/cbvb4tY=
@@ -52,8 +52,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0
5252
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA=
5353
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk=
5454
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
55-
github.com/IBM/sarama v1.46.3 h1:njRsX6jNlnR+ClJ8XmkO+CM4unbrNr/2vB5KK6UA+IE=
56-
github.com/IBM/sarama v1.46.3/go.mod h1:GTUYiF9DMOZVe3FwyGT+dtSPceGFIgA+sPc5u6CBwko=
55+
github.com/IBM/sarama v1.47.0 h1:GcQFEd12+KzfPYeLgN69Fh7vLCtYRhVIx0rO4TZO318=
56+
github.com/IBM/sarama v1.47.0/go.mod h1:7gLLIU97nznOmA6TX++Qds+DRxH89P2XICY2KAQUzAY=
5757
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
5858
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
5959
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
@@ -91,8 +91,8 @@ github.com/aws/aws-sdk-go-v2/service/ecs v1.72.0 h1:hggRKpv26DpYMOik3wWo1Ty5MkAN
9191
github.com/aws/aws-sdk-go-v2/service/ecs v1.72.0/go.mod h1:pMlGFDpHoLTJOIZHGdJOAWmi+xeIlQXuFTuQxs1epYE=
9292
github.com/aws/aws-sdk-go-v2/service/eks v1.80.0 h1:moQGV8cPbVTN7r2Xte1Mybku35QDePSJEd3onYVmBtY=
9393
github.com/aws/aws-sdk-go-v2/service/eks v1.80.0/go.mod h1:Qg678m+87sCuJhcsZojenz8mblYG+Tq86V4m3hjVz0s=
94-
github.com/aws/aws-sdk-go-v2/service/iam v1.53.2 h1:62G6btFUwAa5uR5iPlnlNVAM0zJSLbWgDfKOfUC7oW4=
95-
github.com/aws/aws-sdk-go-v2/service/iam v1.53.2/go.mod h1:av9clChrbZbJ5E21msSsiT2oghl2BJHfQGhCkXmhyu8=
94+
github.com/aws/aws-sdk-go-v2/service/iam v1.53.3 h1:boKZv8dNdHznhAA68hb/dqFz5pxoWmRAOJr9LtscVCI=
95+
github.com/aws/aws-sdk-go-v2/service/iam v1.53.3/go.mod h1:E0QHh3aEwxYb7xshjvxYDELiOda7KBYJ77e/TvGhpcM=
9696
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0=
9797
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc=
9898
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
@@ -168,8 +168,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
168168
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
169169
github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA=
170170
github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
171-
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws=
172-
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
173171
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
174172
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
175173
github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
@@ -221,8 +219,6 @@ github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArs
221219
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
222220
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
223221
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
224-
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
225-
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
226222
github.com/golobby/cast v1.3.3 h1:s2Lawb9RMz7YyYf8IrfMQY4IFmA1R/lgfmj97Vc6fig=
227223
github.com/golobby/cast v1.3.3/go.mod h1:0oDO5IT84HTXcbLDf1YXuk0xtg/cRDrxhbpWKxwtJCY=
228224
github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
@@ -323,8 +319,8 @@ github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12 h1:9Nu54bhS/H/
323319
github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12/go.mod h1:TBzl5BIHNXfS9+C35ZyJaklL7mLDbgUkcgXzSLa8Tk0=
324320
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
325321
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
326-
github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
327-
github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
322+
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
323+
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
328324
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
329325
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
330326
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -383,8 +379,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
383379
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
384380
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
385381
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
386-
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
387-
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
382+
github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0=
383+
github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
388384
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
389385
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
390386
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -514,8 +510,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
514510
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
515511
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
516512
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
517-
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
518-
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
513+
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
514+
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
519515
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
520516
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
521517
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

0 commit comments

Comments
 (0)