From ff488583d99072496fb6d392a12d208b1c4c88a2 Mon Sep 17 00:00:00 2001 From: Dmitrii Creed Date: Wed, 1 Apr 2026 16:05:02 +0400 Subject: [PATCH 1/4] feat: add trustedProxies support for Caddy reverse proxy Adds `trustedProxies` field to CaddyConfig. When set, injects Caddy `servers { trusted_proxies static }` into the global options block alongside GCS storage config. This preserves X-Forwarded-For headers from trusted upstream proxies (e.g., Cloudflare, GKE LB) instead of overwriting them with the immediate client IP. Usage in server.yaml: ```yaml caddy: enable: true trustedProxies: - "10.0.0.0/8" # GKE internal - "172.16.0.0/12" # GKE internal - "173.245.48.0/20" # Cloudflare - "103.21.244.0/22" # Cloudflare ``` --- pkg/clouds/k8s/types.go | 1 + pkg/clouds/pulumi/gcp/gke_autopilot.go | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pkg/clouds/k8s/types.go b/pkg/clouds/k8s/types.go index 2728c3b0..0009e492 100644 --- a/pkg/clouds/k8s/types.go +++ b/pkg/clouds/k8s/types.go @@ -43,6 +43,7 @@ type CaddyConfig struct { Replicas *int `json:"replicas,omitempty" yaml:"replicas,omitempty"` Resources *Resources `json:"resources,omitempty" yaml:"resources,omitempty"` // CPU and memory limits/requests for Caddy container VPA *VPAConfig `json:"vpa,omitempty" yaml:"vpa,omitempty"` // Vertical Pod Autoscaler configuration for Caddy + TrustedProxies []string `json:"trustedProxies,omitempty" yaml:"trustedProxies,omitempty"` // CIDR ranges trusted as reverse proxies (preserves X-Forwarded-For from these sources) UsePrefixes bool `json:"usePrefixes,omitempty" yaml:"usePrefixes,omitempty"` // whether to use prefixes instead of domains (default: false) ServiceType *string `json:"serviceType,omitempty" yaml:"serviceType,omitempty"` // whether to use custom service type instead of LoadBalancer (default: LoadBalancer) ProvisionIngress bool `json:"provisionIngress,omitempty" yaml:"provisionIngress,omitempty"` // whether to provision ingress for caddy (default: false) diff --git a/pkg/clouds/pulumi/gcp/gke_autopilot.go b/pkg/clouds/pulumi/gcp/gke_autopilot.go index 95eca0c6..50af78a2 100644 --- a/pkg/clouds/pulumi/gcp/gke_autopilot.go +++ b/pkg/clouds/pulumi/gcp/gke_autopilot.go @@ -217,21 +217,26 @@ func GkeAutopilot(ctx *sdk.Context, stack api.Stack, input api.ResourceInput, pa } // Build Caddyfile prefix with GCS storage configuration - // Merge with user-provided prefix if it exists + // Merge with user-provided prefix and trusted proxies if configured caddyfilePrefix := bucket.Name.ApplyT(func(bucketName string) string { - gcsStorageConfig := fmt.Sprintf(`{ - storage gcs { - bucket-name %s - } -}`, bucketName) + var globalOpts []string - // If user provided custom prefix, merge it + globalOpts = append(globalOpts, fmt.Sprintf(" storage gcs {\n bucket-name %s\n }", bucketName)) + + // Add trusted_proxies if configured (preserves X-Forwarded-For from these CIDRs) + if len(gkeInput.Caddy.TrustedProxies) > 0 { + cidrs := strings.Join(gkeInput.Caddy.TrustedProxies, " ") + globalOpts = append(globalOpts, fmt.Sprintf(" servers {\n trusted_proxies static %s\n }", cidrs)) + } + + result := fmt.Sprintf("{\n%s\n}", strings.Join(globalOpts, "\n")) + + // If user provided custom prefix, append after global block if gkeInput.Caddy.CaddyfilePrefix != nil && *gkeInput.Caddy.CaddyfilePrefix != "" { - // User prefix first, then GCS storage config - return fmt.Sprintf("%s\n\n%s", gcsStorageConfig, *gkeInput.Caddy.CaddyfilePrefix) + return fmt.Sprintf("%s\n\n%s", result, *gkeInput.Caddy.CaddyfilePrefix) } - return gcsStorageConfig + return result }).(sdk.StringOutput) // Prepare GCP credentials as a secret volume output (Pulumi output) From d4a0a0cc644c79f908f2879a5b89bbb54879b945 Mon Sep 17 00:00:00 2001 From: Dmitrii Creed Date: Wed, 1 Apr 2026 16:20:22 +0400 Subject: [PATCH 2/4] feat: add trustedProxies support for Caddy reverse proxy Adds `trustedProxies` field to CaddyConfig. When set, injects Caddy `servers { trusted_proxies static }` into the global options block alongside storage config. This preserves X-Forwarded-For headers from trusted upstream proxies (e.g., Cloudflare, GKE LB) instead of overwriting them with the immediate connecting IP. Changes: - pkg/clouds/k8s/types.go: add TrustedProxies []string to CaddyConfig - pkg/clouds/pulumi/kubernetes/caddy_global_opts.go: extracted testable BuildTrustedProxiesBlock (with CIDR validation) and BuildCaddyfileGlobalOptions helpers - pkg/clouds/pulumi/kubernetes/caddy.go: support trustedProxies in non-GKE (standalone K8s) code path - pkg/clouds/pulumi/gcp/gke_autopilot.go: use extracted helpers, validate CIDRs before Pulumi apply - pkg/clouds/pulumi/kubernetes/caddy_global_opts_test.go: 9 test cases covering empty, valid, invalid, and combination scenarios --- pkg/clouds/pulumi/gcp/gke_autopilot.go | 30 +++---- pkg/clouds/pulumi/kubernetes/caddy.go | 18 +++-- .../pulumi/kubernetes/caddy_global_opts.go | 58 +++++++++++++ .../kubernetes/caddy_global_opts_test.go | 81 +++++++++++++++++++ 4 files changed, 160 insertions(+), 27 deletions(-) create mode 100644 pkg/clouds/pulumi/kubernetes/caddy_global_opts.go create mode 100644 pkg/clouds/pulumi/kubernetes/caddy_global_opts_test.go diff --git a/pkg/clouds/pulumi/gcp/gke_autopilot.go b/pkg/clouds/pulumi/gcp/gke_autopilot.go index 50af78a2..1dcc70da 100644 --- a/pkg/clouds/pulumi/gcp/gke_autopilot.go +++ b/pkg/clouds/pulumi/gcp/gke_autopilot.go @@ -216,27 +216,17 @@ func GkeAutopilot(ctx *sdk.Context, stack api.Stack, input api.ResourceInput, pa return nil, errors.Wrapf(err, "failed to provision ACME storage for Caddy in cluster %q", clusterName) } - // Build Caddyfile prefix with GCS storage configuration - // Merge with user-provided prefix and trusted proxies if configured - caddyfilePrefix := bucket.Name.ApplyT(func(bucketName string) string { - var globalOpts []string - - globalOpts = append(globalOpts, fmt.Sprintf(" storage gcs {\n bucket-name %s\n }", bucketName)) - - // Add trusted_proxies if configured (preserves X-Forwarded-For from these CIDRs) - if len(gkeInput.Caddy.TrustedProxies) > 0 { - cidrs := strings.Join(gkeInput.Caddy.TrustedProxies, " ") - globalOpts = append(globalOpts, fmt.Sprintf(" servers {\n trusted_proxies static %s\n }", cidrs)) - } - - result := fmt.Sprintf("{\n%s\n}", strings.Join(globalOpts, "\n")) - - // If user provided custom prefix, append after global block - if gkeInput.Caddy.CaddyfilePrefix != nil && *gkeInput.Caddy.CaddyfilePrefix != "" { - return fmt.Sprintf("%s\n\n%s", result, *gkeInput.Caddy.CaddyfilePrefix) - } + // Validate trusted proxies early (before Pulumi apply) + trustedProxiesBlock, err := pulumiKubernetes.BuildTrustedProxiesBlock(lo.FromPtrOr(gkeInput.Caddy, k8s.CaddyConfig{})) + if err != nil { + return nil, errors.Wrapf(err, "invalid caddy trusted proxies for cluster %q", clusterName) + } - return result + // Build Caddyfile prefix with GCS storage configuration and trusted proxies + caddyfilePrefix := bucket.Name.ApplyT(func(bucketName string) string { + storageBlock := fmt.Sprintf(" storage gcs {\n bucket-name %s\n }", bucketName) + userPrefix := lo.FromPtrOr(gkeInput.Caddy.CaddyfilePrefix, "") + return pulumiKubernetes.BuildCaddyfileGlobalOptions(storageBlock, trustedProxiesBlock, userPrefix) }).(sdk.StringOutput) // Prepare GCP credentials as a secret volume output (Pulumi output) diff --git a/pkg/clouds/pulumi/kubernetes/caddy.go b/pkg/clouds/pulumi/kubernetes/caddy.go index 940f4486..60cab6b6 100644 --- a/pkg/clouds/pulumi/kubernetes/caddy.go +++ b/pkg/clouds/pulumi/kubernetes/caddy.go @@ -156,17 +156,21 @@ func DeployCaddyService(ctx *sdk.Context, caddy CaddyDeployment, input api.Resou // Add Caddyfile prefix - prefer dynamic output over static config if isPulumiOutputSet(caddy.CaddyfilePrefixOut) { - // Use dynamic Caddyfile prefix from cloud provider (e.g., GCS storage config) + // Use dynamic Caddyfile prefix from cloud provider (e.g., GCS storage config with trusted proxies baked in) envVars = append(envVars, corev1.EnvVarArgs{ Name: sdk.String("CADDYFILE_PREFIX"), Value: caddy.CaddyfilePrefixOut.ToStringOutput(), }) - } else if caddy.CaddyfilePrefix != nil { - // Use static Caddyfile prefix from config - envVars = append(envVars, corev1.EnvVarArgs{ - Name: sdk.String("CADDYFILE_PREFIX"), - Value: sdk.String(lo.FromPtr(caddy.CaddyfilePrefix)), - }) + } else { + // Build static Caddyfile prefix from config (non-GKE path) + trustedBlock, _ := BuildTrustedProxiesBlock(lo.FromPtrOr(caddy.CaddyConfig, k8s.CaddyConfig{})) + userPrefix := lo.FromPtrOr(caddy.CaddyfilePrefix, "") + if prefix := BuildCaddyfileGlobalOptions("", trustedBlock, userPrefix); prefix != "" { + envVars = append(envVars, corev1.EnvVarArgs{ + Name: sdk.String("CADDYFILE_PREFIX"), + Value: sdk.String(prefix), + }) + } } return envVars diff --git a/pkg/clouds/pulumi/kubernetes/caddy_global_opts.go b/pkg/clouds/pulumi/kubernetes/caddy_global_opts.go new file mode 100644 index 00000000..ad7576ad --- /dev/null +++ b/pkg/clouds/pulumi/kubernetes/caddy_global_opts.go @@ -0,0 +1,58 @@ +package kubernetes + +import ( + "fmt" + "net" + "strings" + + "github.com/pkg/errors" + + "github.com/simple-container-com/api/pkg/clouds/k8s" +) + +// BuildTrustedProxiesBlock builds the Caddy servers { trusted_proxies ... } block +// from the CaddyConfig. Returns empty string if no trusted proxies are configured. +func BuildTrustedProxiesBlock(cfg k8s.CaddyConfig) (string, error) { + if len(cfg.TrustedProxies) == 0 { + return "", nil + } + + for _, cidr := range cfg.TrustedProxies { + if _, _, err := net.ParseCIDR(cidr); err != nil { + if net.ParseIP(cidr) == nil { + return "", errors.Errorf("invalid trusted proxy entry %q: must be a valid CIDR or IP address", cidr) + } + } + } + + cidrs := strings.Join(cfg.TrustedProxies, " ") + return fmt.Sprintf(" servers {\n trusted_proxies static %s\n }", cidrs), nil +} + +// BuildCaddyfileGlobalOptions builds the Caddyfile global options block. +// storageBlock is optional (empty string if not needed, e.g., non-GKE deployments). +// userPrefix is appended after the global block if provided. +func BuildCaddyfileGlobalOptions(storageBlock string, trustedProxiesBlock string, userPrefix string) string { + var globalOpts []string + + if storageBlock != "" { + globalOpts = append(globalOpts, storageBlock) + } + if trustedProxiesBlock != "" { + globalOpts = append(globalOpts, trustedProxiesBlock) + } + + var result string + if len(globalOpts) > 0 { + result = fmt.Sprintf("{\n%s\n}", strings.Join(globalOpts, "\n")) + } + + if userPrefix != "" { + if result != "" { + return fmt.Sprintf("%s\n\n%s", result, userPrefix) + } + return userPrefix + } + + return result +} diff --git a/pkg/clouds/pulumi/kubernetes/caddy_global_opts_test.go b/pkg/clouds/pulumi/kubernetes/caddy_global_opts_test.go new file mode 100644 index 00000000..39e8ef91 --- /dev/null +++ b/pkg/clouds/pulumi/kubernetes/caddy_global_opts_test.go @@ -0,0 +1,81 @@ +package kubernetes + +import ( + "testing" + + . "github.com/onsi/gomega" + + "github.com/simple-container-com/api/pkg/clouds/k8s" +) + +func TestBuildTrustedProxiesBlock(t *testing.T) { + g := NewGomegaWithT(t) + + t.Run("empty proxies returns empty string", func(t *testing.T) { + block, err := BuildTrustedProxiesBlock(k8s.CaddyConfig{}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(block).To(BeEmpty()) + }) + + t.Run("valid CIDRs produce servers block", func(t *testing.T) { + block, err := BuildTrustedProxiesBlock(k8s.CaddyConfig{ + TrustedProxies: []string{"10.0.0.0/8", "172.16.0.0/12"}, + }) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(block).To(ContainSubstring("trusted_proxies static 10.0.0.0/8 172.16.0.0/12")) + g.Expect(block).To(ContainSubstring("servers {")) + }) + + t.Run("single IP is valid", func(t *testing.T) { + block, err := BuildTrustedProxiesBlock(k8s.CaddyConfig{ + TrustedProxies: []string{"10.0.0.1"}, + }) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(block).To(ContainSubstring("10.0.0.1")) + }) + + t.Run("invalid CIDR returns error", func(t *testing.T) { + _, err := BuildTrustedProxiesBlock(k8s.CaddyConfig{ + TrustedProxies: []string{"not-a-cidr"}, + }) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("invalid trusted proxy entry")) + }) +} + +func TestBuildCaddyfileGlobalOptions(t *testing.T) { + g := NewGomegaWithT(t) + + storageBlock := " storage gcs {\n bucket-name test-bucket\n }" + trustedBlock := " servers {\n trusted_proxies static 10.0.0.0/8\n }" + + t.Run("storage only matches previous format", func(t *testing.T) { + result := BuildCaddyfileGlobalOptions(storageBlock, "", "") + g.Expect(result).To(Equal("{\n storage gcs {\n bucket-name test-bucket\n }\n}")) + }) + + t.Run("storage plus trusted proxies", func(t *testing.T) { + result := BuildCaddyfileGlobalOptions(storageBlock, trustedBlock, "") + g.Expect(result).To(ContainSubstring("storage gcs")) + g.Expect(result).To(ContainSubstring("trusted_proxies static")) + g.Expect(result).To(HavePrefix("{")) + g.Expect(result).To(HaveSuffix("}")) + }) + + t.Run("with user prefix appended after", func(t *testing.T) { + result := BuildCaddyfileGlobalOptions(storageBlock, "", "import custom") + g.Expect(result).To(ContainSubstring("storage gcs")) + g.Expect(result).To(ContainSubstring("import custom")) + }) + + t.Run("empty everything returns empty", func(t *testing.T) { + result := BuildCaddyfileGlobalOptions("", "", "") + g.Expect(result).To(BeEmpty()) + }) + + t.Run("only trusted proxies no storage", func(t *testing.T) { + result := BuildCaddyfileGlobalOptions("", trustedBlock, "") + g.Expect(result).To(ContainSubstring("trusted_proxies")) + g.Expect(result).To(HavePrefix("{")) + }) +} From c6ae1a950bff2dff47361cd30f3c02160d3faada Mon Sep 17 00:00:00 2001 From: Dmitrii Creed Date: Wed, 1 Apr 2026 19:11:39 +0400 Subject: [PATCH 3/4] fix: normalize empty TrustedProxies slice in CaddyReadConfig and test fixtures ConvertDescriptor (yaml round-trip) produces []string{} instead of nil for unset slice fields. Normalize in CaddyReadConfig and update test expected values to match deserialization behavior. --- pkg/api/tests/refapp_kubernetes.go | 14 ++++++++------ pkg/clouds/k8s/caddy.go | 12 +++++++++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/pkg/api/tests/refapp_kubernetes.go b/pkg/api/tests/refapp_kubernetes.go index a9d9da9d..bc6b71a7 100644 --- a/pkg/api/tests/refapp_kubernetes.go +++ b/pkg/api/tests/refapp_kubernetes.go @@ -16,9 +16,10 @@ var RefappKubernetesServerResources = map[string]api.ResourceDescriptor{ Kubeconfig: "${auth:kubernetes}", }, CaddyConfig: &k8s.CaddyConfig{ - Enable: lo.ToPtr(true), - Namespace: lo.ToPtr("caddy"), - Replicas: lo.ToPtr(2), + Enable: lo.ToPtr(true), + Namespace: lo.ToPtr("caddy"), + Replicas: lo.ToPtr(2), + TrustedProxies: []string{}, }, }}, }, @@ -75,9 +76,10 @@ var ResolvedRefappKubernetesServerResources = map[string]api.ResourceDescriptor{ Kubeconfig: "", }, CaddyConfig: &k8s.CaddyConfig{ - Enable: lo.ToPtr(true), - Namespace: lo.ToPtr("caddy"), - Replicas: lo.ToPtr(2), + Enable: lo.ToPtr(true), + Namespace: lo.ToPtr("caddy"), + Replicas: lo.ToPtr(2), + TrustedProxies: []string{}, }, }}, }, diff --git a/pkg/clouds/k8s/caddy.go b/pkg/clouds/k8s/caddy.go index cb9e3653..01827351 100644 --- a/pkg/clouds/k8s/caddy.go +++ b/pkg/clouds/k8s/caddy.go @@ -10,5 +10,15 @@ type CaddyResource struct { } func CaddyReadConfig(config *api.Config) (api.Config, error) { - return api.ConvertConfig(config, &CaddyResource{}) + cfg, err := api.ConvertConfig(config, &CaddyResource{}) + if err != nil { + return cfg, err + } + // Normalize empty slices to nil for consistent serialization + if res, ok := cfg.Config.(*CaddyResource); ok && res.CaddyConfig != nil { + if len(res.CaddyConfig.TrustedProxies) == 0 { + res.CaddyConfig.TrustedProxies = nil + } + } + return cfg, nil } From 95e91d1d68f77e13aff3e89f6ed25c13f4208eb7 Mon Sep 17 00:00:00 2001 From: Dmitrii Creed Date: Wed, 1 Apr 2026 20:08:25 +0400 Subject: [PATCH 4/4] fix: handle nil vs empty TrustedProxies in both test paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Unresolved fixture (RefappKubernetesServerResources): nil — matches CaddyReadConfig normalization after ReadServerDescriptor - Resolved fixture (ResolvedRefappKubernetesServerResources): []string{} — matches placeholder resolution which converts nil slices to empty via reflect deep-copy - CaddyReadConfig normalizes []string{} → nil after ConvertConfig --- pkg/api/tests/refapp_kubernetes.go | 9 ++++----- pkg/clouds/k8s/caddy.go | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/api/tests/refapp_kubernetes.go b/pkg/api/tests/refapp_kubernetes.go index bc6b71a7..53755e4e 100644 --- a/pkg/api/tests/refapp_kubernetes.go +++ b/pkg/api/tests/refapp_kubernetes.go @@ -16,10 +16,9 @@ var RefappKubernetesServerResources = map[string]api.ResourceDescriptor{ Kubeconfig: "${auth:kubernetes}", }, CaddyConfig: &k8s.CaddyConfig{ - Enable: lo.ToPtr(true), - Namespace: lo.ToPtr("caddy"), - Replicas: lo.ToPtr(2), - TrustedProxies: []string{}, + Enable: lo.ToPtr(true), + Namespace: lo.ToPtr("caddy"), + Replicas: lo.ToPtr(2), }, }}, }, @@ -79,7 +78,7 @@ var ResolvedRefappKubernetesServerResources = map[string]api.ResourceDescriptor{ Enable: lo.ToPtr(true), Namespace: lo.ToPtr("caddy"), Replicas: lo.ToPtr(2), - TrustedProxies: []string{}, + TrustedProxies: []string{}, // placeholder resolution converts nil → empty slice }, }}, }, diff --git a/pkg/clouds/k8s/caddy.go b/pkg/clouds/k8s/caddy.go index 01827351..2b98781a 100644 --- a/pkg/clouds/k8s/caddy.go +++ b/pkg/clouds/k8s/caddy.go @@ -14,7 +14,8 @@ func CaddyReadConfig(config *api.Config) (api.Config, error) { if err != nil { return cfg, err } - // Normalize empty slices to nil for consistent serialization + // Normalize empty slices to nil — yaml.Unmarshal into inline pointer + // structs can produce []string{} instead of nil for absent fields. if res, ok := cfg.Config.(*CaddyResource); ok && res.CaddyConfig != nil { if len(res.CaddyConfig.TrustedProxies) == 0 { res.CaddyConfig.TrustedProxies = nil