From 2e508c5c4bd096663a9c6cb6cfc66d0b1b3a8ddc Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Sat, 21 Mar 2026 11:37:52 +0100 Subject: [PATCH 1/9] fix(observability): add debug logging and span attributes to webhook path Webhook traces carried no identifying attributes, making it impossible to correlate which specific webhooks flowed through or were lost. This enriches existing spans with delivery ID, event type, job ID, repo, and action, and adds debug-level logging with full payload bodies across the gateway and planner consume path. --- internal/planner/consumer.go | 16 ++++++++++++++++ internal/webhook/server.go | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/internal/planner/consumer.go b/internal/planner/consumer.go index 23885078..444bb804 100644 --- a/internal/planner/consumer.go +++ b/internal/planner/consumer.go @@ -16,6 +16,7 @@ import ( "github.com/google/go-github/v82/github" amqp "github.com/rabbitmq/amqp091-go" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" oteltrace "go.opentelemetry.io/otel/trace" ) @@ -273,6 +274,7 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { job, err := getWorkflowJob(ctx, msg.Headers, msg.Body) if err != nil { logger.ErrorContext(ctx, "cannot parse webhook payload, discarding to DLQ", "error", err) + logger.DebugContext(ctx, "discarded webhook body", "body", string(msg.Body)) span.RecordError(err) c.metrics.ObserveWebhookError(ctx, platform) c.discardMessage(ctx, &msg) @@ -280,11 +282,25 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { } if job == nil { + logger.DebugContext(ctx, "ignored webhook body", "body", string(msg.Body)) c.metrics.ObserveDiscardedWebhook(ctx, platform) c.ignoreMessage(ctx, &msg) return nil } + span.SetAttributes( + attribute.String("github.job_id", job.id), + attribute.String("github.repo", job.repo), + attribute.String("github.action", job.action), + ) + logger.DebugContext(ctx, "consuming webhook", + "job_id", job.id, + "repo", job.repo, + "action", job.action, + "labels", strings.Join(job.labels, ","), + "body", string(msg.Body), + ) + err = c.handleMessage(ctx, job) if err == nil { c.consumedMessage(ctx, &msg) diff --git a/internal/webhook/server.go b/internal/webhook/server.go index e9970505..a03afe32 100644 --- a/internal/webhook/server.go +++ b/internal/webhook/server.go @@ -19,6 +19,7 @@ import ( "github.com/canonical/github-runner-operators/internal/queue" "github.com/canonical/github-runner-operators/internal/server" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" ) @@ -75,6 +76,9 @@ func (h *Handler) receiveWebhook(ctx context.Context, r *http.Request) ([]byte, logger.DebugContext(ctx, "invalid signature", "signature", signature) return nil, &httpError{code: http.StatusForbidden, message: "webhook contains invalid signature"} } + deliveryID := r.Header.Get(WebhookDeliveryHeader) + eventType := r.Header.Get(WebhookEventHeader) + logger.DebugContext(ctx, "received webhook", "delivery_id", deliveryID, "event", eventType) return body, nil } @@ -93,6 +97,11 @@ func (h *Handler) sendWebhook(ctx context.Context, githubHeaders map[string]stri if err != nil { return fmt.Errorf("failed to send webhook: %v", err) } + logger.DebugContext(ctx, "sent webhook to queue", + "delivery_id", githubHeaders[WebhookDeliveryHeader], + "event", githubHeaders[WebhookEventHeader], + "body", string(body), + ) return nil } @@ -103,6 +112,12 @@ func (h *Handler) serveHTTP(ctx context.Context, r *http.Request) error { } }() ctx, span := trace.Start(ctx, "serve webhook") + deliveryID := r.Header.Get(WebhookDeliveryHeader) + eventType := r.Header.Get(WebhookEventHeader) + span.SetAttributes( + attribute.String("github.delivery_id", deliveryID), + attribute.String("github.event", eventType), + ) webhook, err := h.receiveWebhook(ctx, r) if err != nil { inboundWebhookErrors.Add(ctx, 1) @@ -124,6 +139,10 @@ func (h *Handler) serveHTTP(ctx context.Context, r *http.Request) error { } ctx, span = trace.Start(ctx, "send webhook") + span.SetAttributes( + attribute.String("github.delivery_id", deliveryID), + attribute.String("github.event", eventType), + ) err = h.sendWebhook(ctx, githubHeaders, webhook) if err != nil { outboundWebhookErrors.Add(ctx, 1) From 6ee930d4e3e1d1a03ac037be42c0cd10e51a91fe Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Mon, 23 Mar 2026 08:01:26 +0100 Subject: [PATCH 2/9] feat(webhooks): add delivery id to consumer span --- internal/planner/consumer.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/planner/consumer.go b/internal/planner/consumer.go index 444bb804..1a6921b1 100644 --- a/internal/planner/consumer.go +++ b/internal/planner/consumer.go @@ -28,9 +28,10 @@ type JobDatabase interface { } const ( - platform = "github" - githubEventHeaderKey = "X-GitHub-Event" - maxBackoff = 5 * time.Minute + platform = "github" + githubEventHeaderKey = "X-GitHub-Event" + githubDeliveryHeaderKey = "X-GitHub-Delivery" + maxBackoff = 5 * time.Minute ) var supportedActions = []string{"queued", "in_progress", "completed"} @@ -271,10 +272,13 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { ctx, span := trace.Start(ctx, "consume webhook") defer span.End() + deliveryID, _ := msg.Headers[githubDeliveryHeaderKey].(string) + span.SetAttributes(attribute.String("github.delivery_id", deliveryID)) + job, err := getWorkflowJob(ctx, msg.Headers, msg.Body) if err != nil { logger.ErrorContext(ctx, "cannot parse webhook payload, discarding to DLQ", "error", err) - logger.DebugContext(ctx, "discarded webhook body", "body", string(msg.Body)) + logger.DebugContext(ctx, "discarded webhook body", "delivery_id", deliveryID, "body", string(msg.Body)) span.RecordError(err) c.metrics.ObserveWebhookError(ctx, platform) c.discardMessage(ctx, &msg) @@ -282,7 +286,7 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { } if job == nil { - logger.DebugContext(ctx, "ignored webhook body", "body", string(msg.Body)) + logger.DebugContext(ctx, "ignored webhook body", "delivery_id", deliveryID, "body", string(msg.Body)) c.metrics.ObserveDiscardedWebhook(ctx, platform) c.ignoreMessage(ctx, &msg) return nil @@ -294,6 +298,7 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { attribute.String("github.action", job.action), ) logger.DebugContext(ctx, "consuming webhook", + "delivery_id", deliveryID, "job_id", job.id, "repo", job.repo, "action", job.action, From 65bd5a18343a21ac3cefa494b81c0854c09924c2 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Mon, 23 Mar 2026 08:12:05 +0100 Subject: [PATCH 3/9] chore(webhooks): extract constants to new internal package --- internal/github/headers.go | 17 +++++++++++++ internal/planner/consumer.go | 11 ++++----- internal/webhook/server.go | 42 +++++++++++++-------------------- internal/webhook/server_test.go | 11 +++++---- 4 files changed, 45 insertions(+), 36 deletions(-) create mode 100644 internal/github/headers.go diff --git a/internal/github/headers.go b/internal/github/headers.go new file mode 100644 index 00000000..d83c7a2f --- /dev/null +++ b/internal/github/headers.go @@ -0,0 +1,17 @@ +/* + * Copyright 2025 Canonical Ltd. + * See LICENSE file for licensing details. + */ + +// Package github provides shared constants for GitHub webhook integration. +package github + +const ( + SignatureHeader = "X-Hub-Signature-256" + EventHeader = "X-GitHub-Event" + HookIDHeader = "X-GitHub-Hook-ID" + DeliveryHeader = "X-GitHub-Delivery" + HookInstallationTargetTypeHeader = "X-GitHub-Hook-Installation-Target-Type" + HookInstallationTargetIDHeader = "X-GitHub-Hook-Installation-Target-ID" + SignaturePrefix = "sha256=" +) diff --git a/internal/planner/consumer.go b/internal/planner/consumer.go index 1a6921b1..01378cb4 100644 --- a/internal/planner/consumer.go +++ b/internal/planner/consumer.go @@ -12,6 +12,7 @@ import ( "time" "github.com/canonical/github-runner-operators/internal/database" + gh "github.com/canonical/github-runner-operators/internal/github" "github.com/canonical/github-runner-operators/internal/queue" "github.com/google/go-github/v82/github" amqp "github.com/rabbitmq/amqp091-go" @@ -28,10 +29,8 @@ type JobDatabase interface { } const ( - platform = "github" - githubEventHeaderKey = "X-GitHub-Event" - githubDeliveryHeaderKey = "X-GitHub-Delivery" - maxBackoff = 5 * time.Minute + platform = "github" + maxBackoff = 5 * time.Minute ) var supportedActions = []string{"queued", "in_progress", "completed"} @@ -96,7 +95,7 @@ func getWorkflowJob(ctx context.Context, headers map[string]interface{}, body [] // Returns (nil, nil) for non-workflow_job events (should be ignored). // Returns error only for malformed webhooks. func parseWorkflowJobEvent(ctx context.Context, headers map[string]interface{}, body []byte) (*github.WorkflowJobEvent, error) { - eventTypeHeader, ok := headers[githubEventHeaderKey] + eventTypeHeader, ok := headers[gh.EventHeader] if !ok { return nil, fmt.Errorf("missing X-GitHub-Event header") } @@ -272,7 +271,7 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { ctx, span := trace.Start(ctx, "consume webhook") defer span.End() - deliveryID, _ := msg.Headers[githubDeliveryHeaderKey].(string) + deliveryID, _ := msg.Headers[gh.DeliveryHeader].(string) span.SetAttributes(attribute.String("github.delivery_id", deliveryID)) job, err := getWorkflowJob(ctx, msg.Headers, msg.Body) diff --git a/internal/webhook/server.go b/internal/webhook/server.go index a03afe32..25e31a9d 100644 --- a/internal/webhook/server.go +++ b/internal/webhook/server.go @@ -16,6 +16,7 @@ import ( "net/http" "time" + gh "github.com/canonical/github-runner-operators/internal/github" "github.com/canonical/github-runner-operators/internal/queue" "github.com/canonical/github-runner-operators/internal/server" "go.opentelemetry.io/otel" @@ -23,16 +24,7 @@ import ( "go.opentelemetry.io/otel/propagation" ) -const ( - WebhookSignatureHeader = "X-Hub-Signature-256" - WebhookEventHeader = "X-GitHub-Event" - WebhookHookIDHeader = "X-GitHub-Hook-ID" - WebhookDeliveryHeader = "X-GitHub-Delivery" - WebhookHookInstallationTargetTypeHeader = "X-GitHub-Hook-Installation-Target-Type" - WebhookHookInstallationTargetIDHeader = "X-GitHub-Hook-Installation-Target-ID" - bodyLimit = 1048576 - WebhookSignaturePrefix = "sha256=" -) +const bodyLimit = 1048576 type httpError struct { code int @@ -58,7 +50,7 @@ type Handler struct { func (h *Handler) receiveWebhook(ctx context.Context, r *http.Request) ([]byte, error) { reader := io.LimitReader(r.Body, bodyLimit+1) - signature := r.Header.Get(WebhookSignatureHeader) + signature := r.Header.Get(gh.SignatureHeader) if signature == "" { logger.DebugContext(ctx, "missing signature header", "header", r.Header) return nil, &httpError{code: http.StatusForbidden, message: "missing signature header"} @@ -76,8 +68,8 @@ func (h *Handler) receiveWebhook(ctx context.Context, r *http.Request) ([]byte, logger.DebugContext(ctx, "invalid signature", "signature", signature) return nil, &httpError{code: http.StatusForbidden, message: "webhook contains invalid signature"} } - deliveryID := r.Header.Get(WebhookDeliveryHeader) - eventType := r.Header.Get(WebhookEventHeader) + deliveryID := r.Header.Get(gh.DeliveryHeader) + eventType := r.Header.Get(gh.EventHeader) logger.DebugContext(ctx, "received webhook", "delivery_id", deliveryID, "event", eventType) return body, nil } @@ -98,8 +90,8 @@ func (h *Handler) sendWebhook(ctx context.Context, githubHeaders map[string]stri return fmt.Errorf("failed to send webhook: %v", err) } logger.DebugContext(ctx, "sent webhook to queue", - "delivery_id", githubHeaders[WebhookDeliveryHeader], - "event", githubHeaders[WebhookEventHeader], + "delivery_id", githubHeaders[gh.DeliveryHeader], + "event", githubHeaders[gh.EventHeader], "body", string(body), ) return nil @@ -112,8 +104,8 @@ func (h *Handler) serveHTTP(ctx context.Context, r *http.Request) error { } }() ctx, span := trace.Start(ctx, "serve webhook") - deliveryID := r.Header.Get(WebhookDeliveryHeader) - eventType := r.Header.Get(WebhookEventHeader) + deliveryID := r.Header.Get(gh.DeliveryHeader) + eventType := r.Header.Get(gh.EventHeader) span.SetAttributes( attribute.String("github.delivery_id", deliveryID), attribute.String("github.event", eventType), @@ -131,11 +123,11 @@ func (h *Handler) serveHTTP(ctx context.Context, r *http.Request) error { // Extract GitHub headers from the request githubHeaders := map[string]string{ - WebhookEventHeader: r.Header.Get(WebhookEventHeader), - WebhookHookIDHeader: r.Header.Get(WebhookHookIDHeader), - WebhookDeliveryHeader: r.Header.Get(WebhookDeliveryHeader), - WebhookHookInstallationTargetTypeHeader: r.Header.Get(WebhookHookInstallationTargetTypeHeader), - WebhookHookInstallationTargetIDHeader: r.Header.Get(WebhookHookInstallationTargetIDHeader), + gh.EventHeader: r.Header.Get(gh.EventHeader), + gh.HookIDHeader: r.Header.Get(gh.HookIDHeader), + gh.DeliveryHeader: r.Header.Get(gh.DeliveryHeader), + gh.HookInstallationTargetTypeHeader: r.Header.Get(gh.HookInstallationTargetTypeHeader), + gh.HookInstallationTargetIDHeader: r.Header.Get(gh.HookInstallationTargetIDHeader), } ctx, span = trace.Start(ctx, "send webhook") @@ -181,13 +173,13 @@ func (h *Handler) Webhook(w http.ResponseWriter, r *http.Request) { } func validateSignature(message []byte, secret string, signature string) bool { - if len(signature) < len(WebhookSignaturePrefix) { + if len(signature) < len(gh.SignaturePrefix) { return false } - if signature[:len(WebhookSignaturePrefix)] != WebhookSignaturePrefix { + if signature[:len(gh.SignaturePrefix)] != gh.SignaturePrefix { return false } - signatureWithoutPrefix := signature[len(WebhookSignaturePrefix):] + signatureWithoutPrefix := signature[len(gh.SignaturePrefix):] h := hmac.New(sha256.New, []byte(secret)) h.Write(message) sig, err := hex.DecodeString(signatureWithoutPrefix) diff --git a/internal/webhook/server_test.go b/internal/webhook/server_test.go index e7f448e4..2c9056ef 100644 --- a/internal/webhook/server_test.go +++ b/internal/webhook/server_test.go @@ -14,6 +14,7 @@ import ( "strings" "testing" + gh "github.com/canonical/github-runner-operators/internal/github" "github.com/canonical/github-runner-operators/internal/telemetry" "github.com/stretchr/testify/assert" ) @@ -75,8 +76,8 @@ func TestWebhookForwarded(t *testing.T) { func setupRequest() *http.Request { req := httptest.NewRequest(http.MethodPost, webhookPath, strings.NewReader(payload)) req.Header.Set("Content-Type", "application/json") - req.Header.Set(WebhookSignatureHeader, valid_signature_header) - req.Header.Set(WebhookEventHeader, "workflow_job") + req.Header.Set(gh.SignatureHeader, valid_signature_header) + req.Header.Set(gh.EventHeader, "workflow_job") return req } @@ -127,7 +128,7 @@ func TestWebhookInvalidSignature(t *testing.T) { }, { name: "Wrong Prefix", - signature: "mac256=" + valid_signature_header[len(WebhookSignaturePrefix):], + signature: "mac256=" + valid_signature_header[len(gh.SignaturePrefix):], expectedResponseMessage: "invalid signature", }, { @@ -157,9 +158,9 @@ func TestWebhookInvalidSignature(t *testing.T) { defer telemetry.ReleaseTestMetricReader(t) req := setupRequest() if tt.name == "Missing Signature Header" { - req.Header.Del(WebhookSignatureHeader) + req.Header.Del(gh.SignatureHeader) } else { - req.Header.Set(WebhookSignatureHeader, tt.signature) + req.Header.Set(gh.SignatureHeader, tt.signature) } w := httptest.NewRecorder() From 22c8c352538f39f2fed6ee4c2b42e655e2e4b2fe Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Mon, 23 Mar 2026 09:01:59 +0100 Subject: [PATCH 4/9] perf(webhooks): add debug log guards --- internal/planner/consumer.go | 33 ++++++++++++++++++++++----------- internal/webhook/server.go | 13 ++++++++----- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/internal/planner/consumer.go b/internal/planner/consumer.go index 01378cb4..d7d1102e 100644 --- a/internal/planner/consumer.go +++ b/internal/planner/consumer.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "slices" "strconv" "strings" @@ -272,12 +273,18 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { defer span.End() deliveryID, _ := msg.Headers[gh.DeliveryHeader].(string) - span.SetAttributes(attribute.String("github.delivery_id", deliveryID)) + eventType, _ := msg.Headers[gh.EventHeader].(string) + span.SetAttributes( + attribute.String("github.delivery_id", deliveryID), + attribute.String("github.event", eventType), + ) job, err := getWorkflowJob(ctx, msg.Headers, msg.Body) if err != nil { logger.ErrorContext(ctx, "cannot parse webhook payload, discarding to DLQ", "error", err) - logger.DebugContext(ctx, "discarded webhook body", "delivery_id", deliveryID, "body", string(msg.Body)) + if logger.Enabled(ctx, slog.LevelDebug) { + logger.DebugContext(ctx, "discarded webhook body", "delivery_id", deliveryID, "body", string(msg.Body)) + } span.RecordError(err) c.metrics.ObserveWebhookError(ctx, platform) c.discardMessage(ctx, &msg) @@ -285,7 +292,9 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { } if job == nil { - logger.DebugContext(ctx, "ignored webhook body", "delivery_id", deliveryID, "body", string(msg.Body)) + if logger.Enabled(ctx, slog.LevelDebug) { + logger.DebugContext(ctx, "ignored webhook body", "delivery_id", deliveryID, "body", string(msg.Body)) + } c.metrics.ObserveDiscardedWebhook(ctx, platform) c.ignoreMessage(ctx, &msg) return nil @@ -296,14 +305,16 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { attribute.String("github.repo", job.repo), attribute.String("github.action", job.action), ) - logger.DebugContext(ctx, "consuming webhook", - "delivery_id", deliveryID, - "job_id", job.id, - "repo", job.repo, - "action", job.action, - "labels", strings.Join(job.labels, ","), - "body", string(msg.Body), - ) + if logger.Enabled(ctx, slog.LevelDebug) { + logger.DebugContext(ctx, "consuming webhook", + "delivery_id", deliveryID, + "job_id", job.id, + "repo", job.repo, + "action", job.action, + "labels", strings.Join(job.labels, ","), + "body", string(msg.Body), + ) + } err = c.handleMessage(ctx, job) if err == nil { diff --git a/internal/webhook/server.go b/internal/webhook/server.go index 25e31a9d..20f90b84 100644 --- a/internal/webhook/server.go +++ b/internal/webhook/server.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "io" + "log/slog" "net/http" "time" @@ -89,11 +90,13 @@ func (h *Handler) sendWebhook(ctx context.Context, githubHeaders map[string]stri if err != nil { return fmt.Errorf("failed to send webhook: %v", err) } - logger.DebugContext(ctx, "sent webhook to queue", - "delivery_id", githubHeaders[gh.DeliveryHeader], - "event", githubHeaders[gh.EventHeader], - "body", string(body), - ) + if logger.Enabled(ctx, slog.LevelDebug) { + logger.DebugContext(ctx, "sent webhook to queue", + "delivery_id", githubHeaders[gh.DeliveryHeader], + "event", githubHeaders[gh.EventHeader], + "body", string(body), + ) + } return nil } From d8a88ac7451d0fd1a8c7c139a4caa9aedc264d79 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Mon, 23 Mar 2026 09:21:01 +0100 Subject: [PATCH 5/9] fix(webhooks): avoid logging body to reduce log volume --- internal/planner/consumer.go | 26 +++++++++----------------- internal/webhook/server.go | 29 ++++++++++++----------------- 2 files changed, 21 insertions(+), 34 deletions(-) diff --git a/internal/planner/consumer.go b/internal/planner/consumer.go index d7d1102e..f6afc87c 100644 --- a/internal/planner/consumer.go +++ b/internal/planner/consumer.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "log/slog" "slices" "strconv" "strings" @@ -282,9 +281,7 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { job, err := getWorkflowJob(ctx, msg.Headers, msg.Body) if err != nil { logger.ErrorContext(ctx, "cannot parse webhook payload, discarding to DLQ", "error", err) - if logger.Enabled(ctx, slog.LevelDebug) { - logger.DebugContext(ctx, "discarded webhook body", "delivery_id", deliveryID, "body", string(msg.Body)) - } + logger.DebugContext(ctx, "discarded webhook", "delivery_id", deliveryID) span.RecordError(err) c.metrics.ObserveWebhookError(ctx, platform) c.discardMessage(ctx, &msg) @@ -292,9 +289,7 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { } if job == nil { - if logger.Enabled(ctx, slog.LevelDebug) { - logger.DebugContext(ctx, "ignored webhook body", "delivery_id", deliveryID, "body", string(msg.Body)) - } + logger.DebugContext(ctx, "ignored webhook", "delivery_id", deliveryID) c.metrics.ObserveDiscardedWebhook(ctx, platform) c.ignoreMessage(ctx, &msg) return nil @@ -305,16 +300,13 @@ func (c *JobConsumer) pullMessage(ctx context.Context) error { attribute.String("github.repo", job.repo), attribute.String("github.action", job.action), ) - if logger.Enabled(ctx, slog.LevelDebug) { - logger.DebugContext(ctx, "consuming webhook", - "delivery_id", deliveryID, - "job_id", job.id, - "repo", job.repo, - "action", job.action, - "labels", strings.Join(job.labels, ","), - "body", string(msg.Body), - ) - } + logger.DebugContext(ctx, "consuming webhook", + "delivery_id", deliveryID, + "job_id", job.id, + "repo", job.repo, + "action", job.action, + "labels", strings.Join(job.labels, ","), + ) err = c.handleMessage(ctx, job) if err == nil { diff --git a/internal/webhook/server.go b/internal/webhook/server.go index 20f90b84..35978894 100644 --- a/internal/webhook/server.go +++ b/internal/webhook/server.go @@ -13,7 +13,6 @@ import ( "errors" "fmt" "io" - "log/slog" "net/http" "time" @@ -90,13 +89,10 @@ func (h *Handler) sendWebhook(ctx context.Context, githubHeaders map[string]stri if err != nil { return fmt.Errorf("failed to send webhook: %v", err) } - if logger.Enabled(ctx, slog.LevelDebug) { - logger.DebugContext(ctx, "sent webhook to queue", - "delivery_id", githubHeaders[gh.DeliveryHeader], - "event", githubHeaders[gh.EventHeader], - "body", string(body), - ) - } + logger.DebugContext(ctx, "sent webhook to queue", + "delivery_id", githubHeaders[gh.DeliveryHeader], + "event", githubHeaders[gh.EventHeader], + ) return nil } @@ -107,22 +103,21 @@ func (h *Handler) serveHTTP(ctx context.Context, r *http.Request) error { } }() ctx, span := trace.Start(ctx, "serve webhook") - deliveryID := r.Header.Get(gh.DeliveryHeader) - eventType := r.Header.Get(gh.EventHeader) - span.SetAttributes( - attribute.String("github.delivery_id", deliveryID), - attribute.String("github.event", eventType), - ) webhook, err := h.receiveWebhook(ctx, r) if err != nil { inboundWebhookErrors.Add(ctx, 1) span.RecordError(err) span.End() return err - } else { - inboundWebhook.Add(ctx, 1) - span.End() } + deliveryID := r.Header.Get(gh.DeliveryHeader) + eventType := r.Header.Get(gh.EventHeader) + span.SetAttributes( + attribute.String("github.delivery_id", deliveryID), + attribute.String("github.event", eventType), + ) + inboundWebhook.Add(ctx, 1) + span.End() // Extract GitHub headers from the request githubHeaders := map[string]string{ From 31b5f7013dc97c079f18943acc7ff582031834dc Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Mon, 23 Mar 2026 09:41:47 +0100 Subject: [PATCH 6/9] chore(webhooks): use constant --- internal/planner/consumer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/planner/consumer.go b/internal/planner/consumer.go index f6afc87c..ad48aa54 100644 --- a/internal/planner/consumer.go +++ b/internal/planner/consumer.go @@ -97,12 +97,12 @@ func getWorkflowJob(ctx context.Context, headers map[string]interface{}, body [] func parseWorkflowJobEvent(ctx context.Context, headers map[string]interface{}, body []byte) (*github.WorkflowJobEvent, error) { eventTypeHeader, ok := headers[gh.EventHeader] if !ok { - return nil, fmt.Errorf("missing X-GitHub-Event header") + return nil, fmt.Errorf("missing %s header", gh.EventHeader) } eventType, ok := eventTypeHeader.(string) if !ok { - return nil, fmt.Errorf("X-GitHub-Event must be string") + return nil, fmt.Errorf("%s must be string", gh.EventHeader) } event, err := github.ParseWebHook(eventType, body) From 259f86a2f3bfbc58acc0d9e6dec1a62b2c3c59a8 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Mon, 23 Mar 2026 09:53:39 +0100 Subject: [PATCH 7/9] Update internal/webhook/server.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- internal/webhook/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/webhook/server.go b/internal/webhook/server.go index 35978894..c0730014 100644 --- a/internal/webhook/server.go +++ b/internal/webhook/server.go @@ -87,7 +87,7 @@ func (h *Handler) sendWebhook(ctx context.Context, githubHeaders map[string]stri } err := h.Producer.Push(ctx, rabbitHeaders, body) if err != nil { - return fmt.Errorf("failed to send webhook: %v", err) + return fmt.Errorf("failed to send webhook: %w", err) } logger.DebugContext(ctx, "sent webhook to queue", "delivery_id", githubHeaders[gh.DeliveryHeader], From 4d535fa0ddd9cc6ba57ae3e9e9a1fba4f79eeaa0 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Mon, 23 Mar 2026 09:54:21 +0100 Subject: [PATCH 8/9] chore(webhooks): use constant for X-GitHub-Event --- internal/planner/consumer.go | 2 +- internal/webhook/server_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/planner/consumer.go b/internal/planner/consumer.go index ad48aa54..38c594d4 100644 --- a/internal/planner/consumer.go +++ b/internal/planner/consumer.go @@ -113,7 +113,7 @@ func parseWorkflowJobEvent(ctx context.Context, headers map[string]interface{}, jobEvent, ok := event.(*github.WorkflowJobEvent) if !ok { if eventType == "workflow_job" { - logger.WarnContext(ctx, "received workflow_job in \"X-GitHub-Event\" header but payload did not parse to expected type; possible GitHub API change or library issue") + logger.WarnContext(ctx, fmt.Sprintf("received workflow_job in %q header but payload did not parse to expected type; possible GitHub API change or library issue", gh.EventHeader)) } else { logger.DebugContext(ctx, "ignoring non-workflow_job event", "event_type", eventType) } diff --git a/internal/webhook/server_test.go b/internal/webhook/server_test.go index 2c9056ef..a39dd568 100644 --- a/internal/webhook/server_test.go +++ b/internal/webhook/server_test.go @@ -65,7 +65,7 @@ func TestWebhookForwarded(t *testing.T) { assert.Equal(t, 1, len(fakeProducer.Messages), "expected 1 message in queue") assert.Equal(t, payload, string(fakeProducer.Messages[0]), "expected message body to match") assert.Equal(t, 1, len(fakeProducer.Headers), "expected 1 header set") - assert.Equal(t, "workflow_job", fakeProducer.Headers[0]["X-GitHub-Event"], "expected X-GitHub-Event header to match") + assert.Equal(t, "workflow_job", fakeProducer.Headers[0][gh.EventHeader], "expected X-GitHub-Event header to match") m := mr.Collect(t) assert.Equal(t, 1.0, m.Counter(t, "github-runner.webhook.gateway.inbound")) assert.Equal(t, 0.0, m.Counter(t, "github-runner.webhook.gateway.inbound.errors")) From 5d05511bd34f6918b2a5cbf4b49bdd2f32639a35 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Mon, 23 Mar 2026 10:02:25 +0100 Subject: [PATCH 9/9] Update internal/planner/consumer.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- internal/planner/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/planner/consumer.go b/internal/planner/consumer.go index 38c594d4..905f8ae2 100644 --- a/internal/planner/consumer.go +++ b/internal/planner/consumer.go @@ -113,7 +113,7 @@ func parseWorkflowJobEvent(ctx context.Context, headers map[string]interface{}, jobEvent, ok := event.(*github.WorkflowJobEvent) if !ok { if eventType == "workflow_job" { - logger.WarnContext(ctx, fmt.Sprintf("received workflow_job in %q header but payload did not parse to expected type; possible GitHub API change or library issue", gh.EventHeader)) + logger.WarnContext(ctx, "received workflow_job event but payload did not parse to expected type; possible GitHub API change or library issue", "header_name", gh.EventHeader) } else { logger.DebugContext(ctx, "ignoring non-workflow_job event", "event_type", eventType) }