Add Talos as an additional cluster provider

Makefile.vars.mk

@@ -1,20 +1,33 @@
 ## These are some common variables for Make
-crossplane_sentinel = $(kind_dir)/crossplane_sentinel
-certmanager_sentinel = $(kind_dir)/certmanager_sentinel
-k8up_sentinel = $(kind_dir)/k8up_sentinel
-prometheus_sentinel = $(kind_dir)/prometheus_sentinel
-local_pv_sentinel = $(kind_dir)/local_pv_sentinel
-csi_sentinel = $(kind_dir)/csi_provider_sentinel
-metallb_sentinel = $(kind_dir)/metallb_sentinel
-komoplane_sentinel = $(kind_dir)/komoplane_sentinel
-netpols_sentinel = $(kind_dir)/netpols_sentinel
-espejo_sentinel = $(kind_dir)/espejo_sentinel
-forgejo_sentinel = $(kind_dir)/forgejo_sentinel
-argocd_sentinel = $(kind_dir)/argocd_sentinel
-secret_generator_sentinel = $(kind_dir)/secret_generator_sentinel
-mariadb_operator_sentinel = $(kind_dir)/mariadb-operator_sentinel
-minio_sentinel = $(kind_dir)/minio_sentinel
-registry_sentinel = $(kind_dir)/registry
+
+## Cluster provider: kind (default) or talos
+CLUSTER_PROVIDER ?= kind
+
+## Sentinel directory (shared across providers)
+cluster_dir ?= $(PWD)/.kind
+
+crossplane_sentinel = $(cluster_dir)/crossplane_sentinel
+certmanager_sentinel = $(cluster_dir)/certmanager_sentinel
+k8up_sentinel = $(cluster_dir)/k8up_sentinel
+prometheus_sentinel = $(cluster_dir)/prometheus_sentinel
+local_pv_sentinel = $(cluster_dir)/local_pv_sentinel
+csi_sentinel = $(cluster_dir)/csi_provider_sentinel
+metallb_sentinel = $(cluster_dir)/metallb_sentinel
+komoplane_sentinel = $(cluster_dir)/komoplane_sentinel
+netpols_sentinel = $(cluster_dir)/netpols_sentinel
+espejo_sentinel = $(cluster_dir)/espejo_sentinel
+forgejo_sentinel = $(cluster_dir)/forgejo_sentinel
+argocd_sentinel = $(cluster_dir)/argocd_sentinel
+secret_generator_sentinel = $(cluster_dir)/secret_generator_sentinel
+mariadb_operator_sentinel = $(cluster_dir)/mariadb-operator_sentinel
+minio_sentinel = $(cluster_dir)/minio_sentinel
+kgateway_sentinel = $(cluster_dir)/kgateway_sentinel
+registry_sentinel = $(cluster_dir)/registry
+
+KGATEWAY_VERSION ?= 2.2.1
+GATEWAY_API_VERSION ?= 1.4.0
+KGATEWAY_PORT_START ?= 10000
+KGATEWAY_PORT_END ?= 10019
 
 enable_xfn = true
 
@@ -31,9 +44,27 @@ DOCKER_CMD ?= docker
 KIND_NODE_VERSION ?= v1.33.4
 KIND_IMAGE ?= docker.io/kindest/node:$(KIND_NODE_VERSION)
 KIND_CMD ?= go run sigs.k8s.io/kind
-KIND_KUBECONFIG ?= $(kind_dir)/kind-kubeconfig-$(KIND_NODE_VERSION)
+KIND_KUBECONFIG ?= $(cluster_dir)/kind-kubeconfig-$(KIND_NODE_VERSION)
 KIND_CLUSTER ?= $(PROJECT_NAME)
 
+## TALOS:setup
+TALOS_CLUSTER_NAME ?= kindev-talos
+TALOS_IMAGE ?= ghcr.io/siderolabs/talos:v1.12.4
+TALOS_K8S_VERSION ?= 1.35.1
+TALOS_SUBNET ?= 10.5.0.0/24
+TALOS_KUBECONFIG ?= $(cluster_dir)/talos-kubeconfig
+
+## Provider-specific settings
+ifeq ($(CLUSTER_PROVIDER),talos)
+  CLUSTER_KUBECONFIG ?= $(TALOS_KUBECONFIG)
+  DOCKER_CONTAINER ?= $(TALOS_CLUSTER_NAME)-controlplane-1
+  DOCKER_NETWORK ?= $(TALOS_CLUSTER_NAME)
+else
+  CLUSTER_KUBECONFIG ?= $(KIND_KUBECONFIG)
+  DOCKER_CONTAINER ?= kindev-control-plane
+  DOCKER_NETWORK ?= kind
+endif
+
 ## PROMETHEUS
 PROM_VALUES=prometheus/values.yaml
 

kgateway/gateway.yaml

@@ -0,0 +1,35 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: ssh-gateway
+  namespace: kgateway-system
+spec:
+  gatewayClassName: kgateway
+  allowedListeners:
+    namespaces:
+      from: Same
+  listeners:
+    - name: dummy
+      port: 9999
+      protocol: TCP
+      allowedRoutes:
+        namespaces:
+          from: All
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: ssh-gateway-2
+  namespace: kgateway-system
+spec:
+  gatewayClassName: kgateway
+  allowedListeners:
+    namespaces:
+      from: Same
+  listeners:
+    - name: dummy
+      port: 9999
+      protocol: TCP
+      allowedRoutes:
+        namespaces:
+          from: All

kgateway/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kgateway-system
+  labels:
+    pod-security.kubernetes.io/enforce: privileged

kgateway/values.yaml

@@ -0,0 +1,2 @@
+env:
+  KGW_ENABLE_GATEWAY_API_EXPERIMENTAL_FEATURES: true

kind/kind.mk

@@ -1,8 +1,6 @@
 go_bin ?= $(PWD)/.work/bin
 $(go_bin):
 	@mkdir -p $@
-
-kind_dir ?= $(PWD)/.kind
 kind_bin = $(go_bin)/kind
 
 # Prepare kind binary
@@ -25,6 +23,7 @@ kind-setup: $(KIND_KUBECONFIG) ## Creates the kind cluster
 kind-setup-ingress: export KUBECONFIG = $(KIND_KUBECONFIG)
 kind-setup-ingress: kind-setup ## Install NGINX as ingress controller onto kind cluster (localhost:8088)
 	kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
+	kubectl -n ingress-nginx patch deployment ingress-nginx-controller --type=json -p '[{"op":"add","path":"/spec/template/spec/nodeSelector/ingress-ready","value":"true"}]'
 	kubectl -n ingress-nginx wait --for condition=Ready pods -l app.kubernetes.io/component=controller --timeout 180s
 	# We need to restart nginx, because it can't properly find the endpoints otherwise...
 	kubectl -n ingress-nginx rollout restart deployment ingress-nginx-controller
@@ -42,15 +41,15 @@ kind-clean: export KUBECONFIG = $(KIND_KUBECONFIG)
 kind-clean: $(kind_bin)
 kind-clean: ## Removes the kind Cluster
 	@$(kind_bin) delete cluster --name $(KIND_CLUSTER) || true
-	rm -rf $(kind_dir) $(kind_bin)
+	rm -rf $(cluster_dir) $(kind_bin)
 
 $(KIND_KUBECONFIG): export KUBECONFIG = $(KIND_KUBECONFIG)
 $(KIND_KUBECONFIG): $(kind_bin)
 	$(kind_bin) create cluster \
 		--name $(KIND_CLUSTER) \
 		--image $(KIND_IMAGE) \
 		--config kind/config.yaml
-	cp $(KIND_KUBECONFIG) $(kind_dir)/kind-config
+	cp $(KIND_KUBECONFIG) $(cluster_dir)/kind-config
 	kubectl taint nodes --all node-role.kubernetes.io/control-plane- node-role.kubernetes.io/master- || true
 	@kubectl version
 	@kubectl cluster-info

registry/registry.yaml

@@ -13,7 +13,8 @@ spec:
       labels:
         app: registry
     spec:
-      nodeName: kindev-control-plane
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
       tolerations:
         - effect: NoSchedule
           key: node-role.kubernetes.io/master

talos/config-patch-controlplane.yaml

@@ -0,0 +1,37 @@
+machine:
+  nodeLabels:
+    ingress-ready: "true"
+  features:
+    kubePrism:
+      enabled: true
+      port: 7445
+    hostDNS:
+      enabled: true
+      forwardKubeDNSToHost: true
+cluster:
+  extraManifests:
+    - https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
+  controllerManager:
+    extraArgs:
+      bind-address: 0.0.0.0
+  scheduler:
+    extraArgs:
+      bind-address: 0.0.0.0
+  network:
+    cni:
+      name: none
+  proxy:
+    disabled: true
+  apiServer:
+    admissionControl:
+      - name: PodSecurity
+        configuration:
+          apiVersion: pod-security.admission.config.k8s.io/v1
+          kind: PodSecurityConfiguration
+          defaults:
+            enforce: privileged
+            enforce-version: latest
+            audit: restricted
+            audit-version: latest
+            warn: restricted
+            warn-version: latest

talos/config-patch-worker.yaml

@@ -0,0 +1,8 @@
+machine:
+  features:
+    kubePrism:
+      enabled: true
+      port: 7445
+    hostDNS:
+      enabled: true
+      forwardKubeDNSToHost: true

talos/talos.mk

@@ -0,0 +1,75 @@
+.PHONY: talos-setup
+talos-setup: $(TALOS_KUBECONFIG)
+
+$(TALOS_KUBECONFIG):
+	@mkdir -p $(cluster_dir)
+	@set -e; \
+	talosctl cluster create docker \
+		--name $(TALOS_CLUSTER_NAME) \
+		--image $(TALOS_IMAGE) \
+		--kubernetes-version $(TALOS_K8S_VERSION) \
+		--host-ip 0.0.0.0 \
+		--workers 1 \
+		--memory-controlplanes 8GiB \
+		--memory-workers 8GiB \
+		-p 8088:80/tcp,8443:443/tcp,5000:5000/tcp,$$(seq $(KGATEWAY_PORT_START) $(KGATEWAY_PORT_END) | sed 's/.*/&:&\/tcp/' | paste -sd,) \
+		--config-patch-controlplanes @talos/config-patch-controlplane.yaml \
+		--config-patch-workers @talos/config-patch-worker.yaml & \
+	talos_pid=$$!; \
+	echo "Waiting for Talos API..."; \
+	until talosctl kubeconfig --force --nodes 10.5.0.2 $(TALOS_KUBECONFIG) 2>/dev/null; do sleep 2; done; \
+	echo "Waiting for Kubernetes API..."; \
+	until kubectl --kubeconfig=$(TALOS_KUBECONFIG) get nodes >/dev/null 2>&1; do sleep 2; done; \
+	echo "Installing Cilium CNI..."; \
+	helm repo add cilium https://helm.cilium.io/ 2>/dev/null || true; \
+	helm upgrade --install cilium cilium/cilium \
+		--namespace kube-system \
+		--kubeconfig $(TALOS_KUBECONFIG) \
+		--set ipam.mode=kubernetes \
+		--set kubeProxyReplacement=true \
+		--set k8sServiceHost=localhost \
+		--set k8sServicePort=7445 \
+		--set securityContext.capabilities.ciliumAgent="{CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}" \
+		--set securityContext.capabilities.cleanCiliumState="{NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}" \
+		--set cgroup.autoMount.enabled=false \
+		--set cgroup.hostRoot=/sys/fs/cgroup \
+		--wait; \
+	echo "Waiting for cluster readiness..."; \
+	wait $$talos_pid
+	@echo =======
+	@echo "Setup finished. To interact with the local dev cluster, set the KUBECONFIG environment variable as follows:"
+	@echo "export KUBECONFIG=$$(realpath "$(TALOS_KUBECONFIG)")"
+	@echo =======
+
+.PHONY: talos-cilium-setup
+talos-cilium-setup:
+	helm repo add cilium https://helm.cilium.io/ 2>/dev/null || true
+	helm upgrade --install cilium cilium/cilium \
+		--namespace kube-system \
+		--kubeconfig $(TALOS_KUBECONFIG) \
+		--set ipam.mode=kubernetes \
+		--set kubeProxyReplacement=true \
+		--set k8sServiceHost=localhost \
+		--set k8sServicePort=7445 \
+		--set securityContext.capabilities.ciliumAgent="{CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}" \
+		--set securityContext.capabilities.cleanCiliumState="{NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}" \
+		--set cgroup.autoMount.enabled=false \
+		--set cgroup.hostRoot=/sys/fs/cgroup \
+		--wait
+
+.PHONY: talos-setup-ingress
+talos-setup-ingress: export KUBECONFIG = $(CLUSTER_KUBECONFIG)
+talos-setup-ingress: talos-setup
+	kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
+	kubectl -n ingress-nginx patch deployment ingress-nginx-controller --type=json -p '[{"op":"add","path":"/spec/template/spec/nodeSelector/ingress-ready","value":"true"}]'
+	kubectl -n ingress-nginx wait --for=condition=Available deployment/ingress-nginx-controller --timeout=180s
+
+.PHONY: talos-clean
+talos-clean:
+	talosctl cluster destroy --name $(TALOS_CLUSTER_NAME) --force || true
+	rm -rf $(cluster_dir)
+
+.PHONY: talos-load-image
+talos-load-image: talos-setup build-docker
+	docker tag $(CONTAINER_IMG) localhost:5000/$(CONTAINER_IMG)
+	docker push localhost:5000/$(CONTAINER_IMG)