Skip to content

Commit a692591

Browse files
Copilotintel352
Copilot
and
intel352
authored
step.request_parse: support application/x-www-form-urlencoded body parsing (#258)
* Initial plan * step.request_parse: support application/x-www-form-urlencoded body parsing Co-authored-by: intel352 <77607+intel352@users.noreply.github.com> * step.request_parse: use _raw_body cache, always cache body bytes Co-authored-by: intel352 <77607+intel352@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: intel352 <77607+intel352@users.noreply.github.com> Co-authored-by: Jonathan Langevin <codingsloth@pm.me>
1 parent 39e1b5a commit a692591

2 files changed

Lines changed: 234 additions & 6 deletions

File tree

‎module/pipeline_step_request_parse.go‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"encoding/json"
66
"io"
77
"net/http"
8+
"net/url"
89
"strings"
910

1011
"github.com/CrisisTextLine/modular"
@@ -155,12 +156,41 @@ func (s *RequestParseStep) Execute(_ context.Context, pc *PipelineContext) (*Ste
155156
output["body"] = body
156157
} else {
157158
req, _ := pc.Metadata["_http_request"].(*http.Request)
158-
if req != nil && req.Body != nil {
159-
bodyBytes, err := io.ReadAll(req.Body)
160-
if err == nil && len(bodyBytes) > 0 {
161-
var bodyData map[string]any
162-
if json.Unmarshal(bodyBytes, &bodyData) == nil {
163-
output["body"] = bodyData
159+
if req != nil {
160+
// Prefer cached raw body (set by a prior step, e.g. step.webhook_verify)
161+
// to avoid consuming req.Body a second time.
162+
var bodyBytes []byte
163+
if cached, ok := pc.Metadata["_raw_body"].([]byte); ok && len(cached) > 0 {
164+
bodyBytes = cached
165+
} else if req.Body != nil {
166+
b, err := io.ReadAll(req.Body)
167+
if err == nil && len(b) > 0 {
168+
bodyBytes = b
169+
pc.Metadata["_raw_body"] = bodyBytes
170+
}
171+
}
172+
if len(bodyBytes) > 0 {
173+
ct := req.Header.Get("Content-Type")
174+
if idx := strings.Index(ct, ";"); idx != -1 {
175+
ct = strings.TrimSpace(ct[:idx])
176+
}
177+
if strings.EqualFold(ct, "application/x-www-form-urlencoded") {
178+
if formValues, parseErr := url.ParseQuery(string(bodyBytes)); parseErr == nil {
179+
bodyData := make(map[string]any)
180+
for k, v := range formValues {
181+
if len(v) == 1 {
182+
bodyData[k] = v[0]
183+
} else {
184+
bodyData[k] = v
185+
}
186+
}
187+
output["body"] = bodyData
188+
}
189+
} else {
190+
var bodyData map[string]any
191+
if json.Unmarshal(bodyBytes, &bodyData) == nil {
192+
output["body"] = bodyData
193+
}
164194
}
165195
}
166196
}

‎module/pipeline_step_request_parse_test.go‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,204 @@ func TestRequestParseStep_WildcardPathParam_SingleSegment(t *testing.T) {
188188
}
189189
}
190190

191+
func TestRequestParseStep_ParseBody_FormURLEncoded(t *testing.T) {
192+
factory := NewRequestParseStepFactory()
193+
step, err := factory("parse-form", map[string]any{
194+
"parse_body": true,
195+
}, nil)
196+
if err != nil {
197+
t.Fatalf("factory error: %v", err)
198+
}
199+
200+
body := bytes.NewBufferString(`Body=Hello&From=%2B15551234567&To=%2B15559876543&MessageSid=SM1234`)
201+
req, _ := http.NewRequest("POST", "/webhook", body)
202+
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
203+
204+
pc := NewPipelineContext(nil, map[string]any{
205+
"_http_request": req,
206+
})
207+
208+
result, err := step.Execute(context.Background(), pc)
209+
if err != nil {
210+
t.Fatalf("execute error: %v", err)
211+
}
212+
213+
bodyData, ok := result.Output["body"].(map[string]any)
214+
if !ok {
215+
t.Fatal("expected body in output")
216+
}
217+
if bodyData["Body"] != "Hello" {
218+
t.Errorf("expected Body='Hello', got %v", bodyData["Body"])
219+
}
220+
if bodyData["From"] != "+15551234567" {
221+
t.Errorf("expected From='+15551234567', got %v", bodyData["From"])
222+
}
223+
if bodyData["To"] != "+15559876543" {
224+
t.Errorf("expected To='+15559876543', got %v", bodyData["To"])
225+
}
226+
if bodyData["MessageSid"] != "SM1234" {
227+
t.Errorf("expected MessageSid='SM1234', got %v", bodyData["MessageSid"])
228+
}
229+
230+
// Raw body should be cached in metadata
231+
rawBody, ok := pc.Metadata["_raw_body"].([]byte)
232+
if !ok {
233+
t.Fatal("expected _raw_body in metadata")
234+
}
235+
if string(rawBody) != `Body=Hello&From=%2B15551234567&To=%2B15559876543&MessageSid=SM1234` {
236+
t.Errorf("unexpected _raw_body: %s", rawBody)
237+
}
238+
}
239+
240+
func TestRequestParseStep_ParseBody_FormURLEncoded_MultiValue(t *testing.T) {
241+
factory := NewRequestParseStepFactory()
242+
step, err := factory("parse-form-multi", map[string]any{
243+
"parse_body": true,
244+
}, nil)
245+
if err != nil {
246+
t.Fatalf("factory error: %v", err)
247+
}
248+
249+
body := bytes.NewBufferString(`tag=foo&tag=bar&name=test`)
250+
req, _ := http.NewRequest("POST", "/webhook", body)
251+
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
252+
253+
pc := NewPipelineContext(nil, map[string]any{
254+
"_http_request": req,
255+
})
256+
257+
result, err := step.Execute(context.Background(), pc)
258+
if err != nil {
259+
t.Fatalf("execute error: %v", err)
260+
}
261+
262+
bodyData, ok := result.Output["body"].(map[string]any)
263+
if !ok {
264+
t.Fatal("expected body in output")
265+
}
266+
// Single value should be a string
267+
if bodyData["name"] != "test" {
268+
t.Errorf("expected name='test', got %v", bodyData["name"])
269+
}
270+
// Multiple values should be []string
271+
tags, ok := bodyData["tag"].([]string)
272+
if !ok {
273+
t.Fatalf("expected tag to be []string, got %T", bodyData["tag"])
274+
}
275+
if len(tags) != 2 || tags[0] != "foo" || tags[1] != "bar" {
276+
t.Errorf("expected tag=['foo','bar'], got %v", tags)
277+
}
278+
}
279+
280+
func TestRequestParseStep_ParseBody_FormURLEncoded_ContentTypeWithCharset(t *testing.T) {
281+
factory := NewRequestParseStepFactory()
282+
step, err := factory("parse-form-charset", map[string]any{
283+
"parse_body": true,
284+
}, nil)
285+
if err != nil {
286+
t.Fatalf("factory error: %v", err)
287+
}
288+
289+
body := bytes.NewBufferString(`key=value`)
290+
req, _ := http.NewRequest("POST", "/webhook", body)
291+
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
292+
293+
pc := NewPipelineContext(nil, map[string]any{
294+
"_http_request": req,
295+
})
296+
297+
result, err := step.Execute(context.Background(), pc)
298+
if err != nil {
299+
t.Fatalf("execute error: %v", err)
300+
}
301+
302+
bodyData, ok := result.Output["body"].(map[string]any)
303+
if !ok {
304+
t.Fatal("expected body in output")
305+
}
306+
if bodyData["key"] != "value" {
307+
t.Errorf("expected key='value', got %v", bodyData["key"])
308+
}
309+
}
310+
311+
func TestRequestParseStep_ParseBody_FormURLEncoded_CachedRawBody(t *testing.T) {
312+
// Simulate scenario where req.Body has already been consumed by a prior step
313+
// (e.g. step.webhook_verify) and the raw bytes are cached in _raw_body.
314+
factory := NewRequestParseStepFactory()
315+
step, err := factory("parse-form-cached", map[string]any{
316+
"parse_body": true,
317+
}, nil)
318+
if err != nil {
319+
t.Fatalf("factory error: %v", err)
320+
}
321+
322+
rawBody := `Body=Hello&From=%2B15551234567`
323+
// req.Body is empty/consumed (simulate body already read)
324+
req, _ := http.NewRequest("POST", "/webhook", bytes.NewBufferString(""))
325+
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
326+
327+
pc := NewPipelineContext(nil, map[string]any{
328+
"_http_request": req,
329+
"_raw_body": []byte(rawBody),
330+
})
331+
332+
result, err := step.Execute(context.Background(), pc)
333+
if err != nil {
334+
t.Fatalf("execute error: %v", err)
335+
}
336+
337+
bodyData, ok := result.Output["body"].(map[string]any)
338+
if !ok {
339+
t.Fatal("expected body in output when reading from _raw_body cache")
340+
}
341+
if bodyData["Body"] != "Hello" {
342+
t.Errorf("expected Body='Hello', got %v", bodyData["Body"])
343+
}
344+
if bodyData["From"] != "+15551234567" {
345+
t.Errorf("expected From='+15551234567', got %v", bodyData["From"])
346+
}
347+
}
348+
349+
func TestRequestParseStep_ParseBody_JSON_CachesRawBody(t *testing.T) {
350+
// Verify that reading a JSON body also caches the raw bytes in _raw_body.
351+
factory := NewRequestParseStepFactory()
352+
step, err := factory("parse-json-cache", map[string]any{
353+
"parse_body": true,
354+
}, nil)
355+
if err != nil {
356+
t.Fatalf("factory error: %v", err)
357+
}
358+
359+
bodyStr := `{"name":"test"}`
360+
req, _ := http.NewRequest("POST", "/api/resource", bytes.NewBufferString(bodyStr))
361+
req.Header.Set("Content-Type", "application/json")
362+
363+
pc := NewPipelineContext(nil, map[string]any{
364+
"_http_request": req,
365+
})
366+
367+
result, err := step.Execute(context.Background(), pc)
368+
if err != nil {
369+
t.Fatalf("execute error: %v", err)
370+
}
371+
372+
bodyData, ok := result.Output["body"].(map[string]any)
373+
if !ok {
374+
t.Fatal("expected body in output")
375+
}
376+
if bodyData["name"] != "test" {
377+
t.Errorf("expected name='test', got %v", bodyData["name"])
378+
}
379+
380+
rawBody, ok := pc.Metadata["_raw_body"].([]byte)
381+
if !ok {
382+
t.Fatal("expected _raw_body cached in metadata for JSON body")
383+
}
384+
if string(rawBody) != bodyStr {
385+
t.Errorf("unexpected _raw_body: %s", rawBody)
386+
}
387+
}
388+
191389
func TestRequestParseStep_EmptyConfig(t *testing.T) {
192390
factory := NewRequestParseStepFactory()
193391
step, err := factory("parse-empty", map[string]any{}, nil)

0 commit comments

Comments
 (0)