Skip to content

Commit 5dffeb3

Browse files
committed
Add externalTrafficPolicy to CaddyConfig for client IP preservation
When set to 'Local', the L4 LoadBalancer skips SNAT and preserves the real client source IP. Without this, X-Forwarded-For only contains internal GKE node IPs regardless of trustedProxies configuration. Required for correct client IP detection in applications behind Caddy on GKE Autopilot with L4 LoadBalancer (which is the default service type). Usage in server.yaml: caddy: enable: true externalTrafficPolicy: Local trustedProxies: - 10.0.0.0/8 - 173.245.48.0/20
1 parent a336188 commit 5dffeb3

File tree

5 files changed

+115
-9
lines changed

5 files changed

+115
-9
lines changed

pkg/clouds/k8s/types.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ type CaddyConfig struct {
4747
UseSSL *bool `json:"useSSL,omitempty" yaml:"useSSL,omitempty"` // whether to use ssl by default (default: true)
4848
// Deployment name override for existing Caddy deployments (used when adopting clusters)
4949
DeploymentName *string `json:"deploymentName,omitempty" yaml:"deploymentName,omitempty"` // override deployment name when adopting existing Caddy
50+
// ExternalTrafficPolicy for LoadBalancer service. "Local" preserves client source IP
51+
// (required for correct X-Forwarded-For when behind L4 LB). Default: "Cluster".
52+
ExternalTrafficPolicy *string `json:"externalTrafficPolicy,omitempty" yaml:"externalTrafficPolicy,omitempty"`
5053
}
5154

5255
type DisruptionBudget struct {

pkg/clouds/pulumi/kubernetes/caddy.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,8 @@ func DeployCaddyService(ctx *sdk.Context, caddy CaddyDeployment, input api.Resou
250250
SecretVolumes: caddy.SecretVolumes, // Cloud credentials volumes (e.g., GCP service account)
251251
SecretVolumeOutputs: caddy.SecretVolumeOutputs, // Pulumi outputs for secret volumes
252252
SecretEnvs: secretEnvs, // Secret environment variables
253-
VPA: caddy.VPA, // Vertical Pod Autoscaler configuration for Caddy
253+
VPA: caddy.VPA, // Vertical Pod Autoscaler configuration for Caddy
254+
ExternalTrafficPolicy: lo.FromPtr(caddy.CaddyConfig).ExternalTrafficPolicy,
254255
Images: []*ContainerImage{
255256
{
256257
Container: caddyContainer,

pkg/clouds/pulumi/kubernetes/deployment.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ type Args struct {
4545
VPA *k8s.VPAConfig // Vertical Pod Autoscaler configuration
4646
ReadinessProbe *k8s.CloudRunProbe // Global readiness probe configuration
4747
LivenessProbe *k8s.CloudRunProbe // Global liveness probe configuration
48+
ExternalTrafficPolicy *string // "Local" preserves client IP on LoadBalancer services
4849
}
4950

5051
func DeploySimpleContainer(ctx *sdk.Context, args Args, opts ...sdk.ResourceOption) (*SimpleContainer, error) {
@@ -236,8 +237,9 @@ func DeploySimpleContainer(ctx *sdk.Context, args Args, opts ...sdk.ResourceOpti
236237
NodeSelector: args.NodeSelector,
237238
Affinity: args.Affinity,
238239
Sidecars: args.Sidecars,
239-
VPA: args.VPA, // Pass VPA configuration to SimpleContainer
240-
Scale: args.Deployment.Scale, // Pass Scale configuration to SimpleContainer
240+
VPA: args.VPA, // Pass VPA configuration to SimpleContainer
241+
Scale: args.Deployment.Scale, // Pass Scale configuration to SimpleContainer
242+
ExternalTrafficPolicy: args.ExternalTrafficPolicy,
241243
PodDisruption: lo.If(args.Deployment.DisruptionBudget != nil, args.Deployment.DisruptionBudget).Else(&k8s.DisruptionBudget{
242244
MinAvailable: lo.ToPtr(1),
243245
}),

pkg/clouds/pulumi/kubernetes/simple_container.go

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,8 @@ type SimpleContainerArgs struct {
112112
NodeSelector map[string]string `json:"nodeSelector" yaml:"nodeSelector"`
113113
Affinity *k8s.AffinityRules `json:"affinity" yaml:"affinity"`
114114
IngressContainer *k8s.CloudRunContainer `json:"ingressContainer" yaml:"ingressContainer"`
115-
ServiceType *string `json:"serviceType" yaml:"serviceType"`
115+
ServiceType *string `json:"serviceType" yaml:"serviceType"`
116+
ExternalTrafficPolicy *string `json:"externalTrafficPolicy" yaml:"externalTrafficPolicy"`
116117
ProvisionIngress bool `json:"provisionIngress" yaml:"provisionIngress"`
117118
Headers *k8s.Headers `json:"headers" yaml:"headers"`
118119
Volumes []k8s.SimpleTextVolume `json:"volumes" yaml:"volumes"`
@@ -589,11 +590,7 @@ ${proto}://${domain} {
589590
Labels: sdk.ToStringMap(appLabels),
590591
Annotations: sdk.ToStringMap(serviceAnnotations),
591592
},
592-
Spec: &corev1.ServiceSpecArgs{
593-
Selector: sdk.ToStringMap(appLabels),
594-
Ports: servicePorts,
595-
Type: serviceType,
596-
},
593+
Spec: serviceSpec(appLabels, servicePorts, serviceType, args.ExternalTrafficPolicy),
597594
}, opts...)
598595
if err != nil {
599596
return nil, err
@@ -850,6 +847,21 @@ func createVPA(ctx *sdk.Context, args *SimpleContainerArgs, deploymentName, name
850847
return nil
851848
}
852849

850+
// serviceSpec builds a ServiceSpecArgs, optionally setting ExternalTrafficPolicy
851+
// when the service type is LoadBalancer. "Local" preserves the client source IP
852+
// by skipping SNAT — required for correct X-Forwarded-For behind an L4 LB.
853+
func serviceSpec(appLabels map[string]string, ports corev1.ServicePortArray, serviceType sdk.StringInput, externalTrafficPolicy *string) *corev1.ServiceSpecArgs {
854+
spec := &corev1.ServiceSpecArgs{
855+
Selector: sdk.ToStringMap(appLabels),
856+
Ports: ports,
857+
Type: serviceType,
858+
}
859+
if externalTrafficPolicy != nil {
860+
spec.ExternalTrafficPolicy = sdk.StringPtr(*externalTrafficPolicy)
861+
}
862+
return spec
863+
}
864+
853865
func ToImagePullSecretName(deploymentName string) string {
854866
return fmt.Sprintf("%s-docker-config", deploymentName)
855867
}

pkg/clouds/pulumi/kubernetes/simple_container_advanced_test.go

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -660,3 +660,91 @@ func TestSimpleContainer_ServiceTypeVariations(t *testing.T) {
660660
})
661661
}
662662
}
663+
664+
func TestServiceSpec_ExternalTrafficPolicy(t *testing.T) {
665+
RegisterTestingT(t)
666+
667+
t.Run("nil policy uses default", func(t *testing.T) {
668+
spec := serviceSpec(
669+
map[string]string{"app": "test"},
670+
corev1.ServicePortArray{},
671+
sdk.String("LoadBalancer"),
672+
nil,
673+
)
674+
Expect(spec).ToNot(BeNil())
675+
Expect(spec.ExternalTrafficPolicy).To(BeNil())
676+
})
677+
678+
t.Run("Local policy is set", func(t *testing.T) {
679+
policy := "Local"
680+
spec := serviceSpec(
681+
map[string]string{"app": "test"},
682+
corev1.ServicePortArray{},
683+
sdk.String("LoadBalancer"),
684+
&policy,
685+
)
686+
Expect(spec).ToNot(BeNil())
687+
Expect(spec.ExternalTrafficPolicy).ToNot(BeNil())
688+
})
689+
690+
t.Run("Cluster policy is set", func(t *testing.T) {
691+
policy := "Cluster"
692+
spec := serviceSpec(
693+
map[string]string{"app": "test"},
694+
corev1.ServicePortArray{},
695+
sdk.String("ClusterIP"),
696+
&policy,
697+
)
698+
Expect(spec).ToNot(BeNil())
699+
Expect(spec.ExternalTrafficPolicy).ToNot(BeNil())
700+
})
701+
}
702+
703+
func TestSimpleContainer_ExternalTrafficPolicy(t *testing.T) {
704+
RegisterTestingT(t)
705+
706+
t.Run("LoadBalancer with Local traffic policy", func(t *testing.T) {
707+
mocks := NewSimpleContainerMocks()
708+
709+
err := pulumi.RunErr(func(ctx *pulumi.Context) error {
710+
policy := "Local"
711+
args := &SimpleContainerArgs{
712+
Namespace: "traffic-test",
713+
Service: "traffic-test",
714+
ScEnv: "test",
715+
Deployment: "traffic-deployment",
716+
Replicas: 1,
717+
Log: logger.New(),
718+
719+
IngressContainer: &k8s.CloudRunContainer{
720+
Name: "traffic-container",
721+
Ports: []int{8080},
722+
MainPort: lo.ToPtr(8080),
723+
},
724+
ServiceType: lo.ToPtr("LoadBalancer"),
725+
ExternalTrafficPolicy: &policy,
726+
727+
Containers: []corev1.ContainerArgs{
728+
{
729+
Name: sdk.String("traffic-container"),
730+
Image: sdk.String("nginx:latest"),
731+
Ports: corev1.ContainerPortArray{
732+
&corev1.ContainerPortArgs{
733+
ContainerPort: sdk.Int(8080),
734+
},
735+
},
736+
},
737+
},
738+
}
739+
740+
sc, err := NewSimpleContainer(ctx, args)
741+
Expect(err).ToNot(HaveOccurred())
742+
Expect(sc).ToNot(BeNil())
743+
Expect(sc.Service).ToNot(BeNil())
744+
745+
return nil
746+
}, pulumi.WithMocks("project", "stack", mocks))
747+
748+
Expect(err).ToNot(HaveOccurred())
749+
})
750+
}

0 commit comments

Comments
 (0)