test(e2e): add Cilium-CNI e2e suite

.github/workflows/ci.yml

@@ -107,6 +107,41 @@ jobs:
           duration: 15m
           authorized-users: squat, leonnicolas, kvaps
 
+  e2e-cilium:
+    runs-on:
+      - nscloud-ubuntu-22.04-amd64-8x16-with-features
+      - namespace-features:kernel.release-channel=bleeding-edge
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+      - name: Build
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          platforms: linux/amd64
+          tags: squat/kilo:test
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          load: "true"
+          build-args: |
+            VERSION=${{ github.sha }}
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: v3.16.0
+      - uses: DeterminateSystems/determinate-nix-action@v3.17.2
+      - uses: DeterminateSystems/magic-nix-cache-action@v13
+      - env:
+          E2E_SKIP_TEARDOWN_ON_FAILURE: "true"
+        run: nix develop . --command make e2e-cilium
+      - name: Breakpoint if tests failed
+        if: failure()
+        uses: namespacelabs/breakpoint-action@v0
+        with:
+          duration: 15m
+          authorized-users: squat, leonnicolas, kvaps
+
   lint:
     runs-on: ubuntu-latest
     steps:

Makefile

@@ -87,7 +87,14 @@ unit:
 test: lint unit e2e
 
 e2e:
-	KILO_IMAGE=squat/kilo:test bash_unit $(BASH_UNIT_FLAGS) ./e2e/setup.sh ./e2e/full-mesh.sh ./e2e/location-mesh.sh ./e2e/multi-cluster.sh ./e2e/handlers.sh ./e2e/kgctl.sh ./e2e/teardown.sh
+	KILO_IMAGE=squat/kilo:test bash_unit $(BASH_UNIT_FLAGS) ./e2e/setup.sh ./e2e/full-mesh.sh ./e2e/location-mesh.sh ./e2e/cross-mesh.sh ./e2e/multi-cluster.sh ./e2e/handlers.sh ./e2e/kgctl.sh ./e2e/teardown.sh
+
+# e2e-cilium runs the Kilo --compatibility=cilium e2e suite against a
+# kind cluster where Cilium provides the CNI. It is a separate target
+# from `e2e` because the Cilium cluster is incompatible with the Kilo
+# bridge CNI used by the default suite.
+e2e-cilium:
+	KILO_IMAGE=squat/kilo:test bash_unit $(BASH_UNIT_FLAGS) ./e2e/cilium-setup.sh ./e2e/cilium-cross-mesh.sh ./e2e/cilium-teardown.sh
 
 docs/kg.md:
 	go run ./cmd/kg/... --help | head -n -2 > help.txt

cmd/kg/main.go

@@ -77,6 +77,7 @@ var (
 	availableGranularities = strings.Join([]string{
 		string(mesh.LogicalGranularity),
 		string(mesh.FullGranularity),
+		string(mesh.CrossGranularity),
 	}, ", ")
 	availableLogLevels = strings.Join([]string{
 		logLevelAll,
@@ -237,6 +238,7 @@ func runRoot(_ *cobra.Command, _ []string) error {
 	switch gr {
 	case mesh.LogicalGranularity:
 	case mesh.FullGranularity:
+	case mesh.CrossGranularity:
 	default:
 		return fmt.Errorf("mesh granularity %v unknown; possible values are: %s", granularity, availableGranularities)
 	}

cmd/kgctl/main.go

@@ -49,6 +49,7 @@ var (
 	availableGranularities = []string{
 		string(mesh.LogicalGranularity),
 		string(mesh.FullGranularity),
+		string(mesh.CrossGranularity),
 		string(mesh.AutoGranularity),
 	}
 	availableLogLevels = []string{
@@ -91,6 +92,7 @@ func runRoot(c *cobra.Command, _ []string) error {
 	switch opts.granularity {
 	case mesh.LogicalGranularity:
 	case mesh.FullGranularity:
+	case mesh.CrossGranularity:
 	case mesh.AutoGranularity:
 	default:
 		return fmt.Errorf("mesh granularity %s unknown; posible values are: %s", granularity, availableGranularities)
@@ -164,8 +166,9 @@ func determineGranularity(gr mesh.Granularity, ns []*mesh.Node) (mesh.Granularit
 		switch ret {
 		case mesh.LogicalGranularity:
 		case mesh.FullGranularity:
+		case mesh.CrossGranularity:
 		default:
-			return ret, fmt.Errorf("mesh granularity %s is not supported", opts.granularity)
+			return ret, fmt.Errorf("mesh granularity %s is not supported", ret)
 		}
 		return ret, nil
 	}

docs/kg.md

@@ -50,7 +50,7 @@ Flags:
       --local                          Should Kilo manage routes within a location? (default true)
       --log-level string               Log level to use. Possible values: all, debug, info, warn, error, none (default "info")
       --master string                  The address of the Kubernetes API server (overrides any value in kubeconfig).
-      --mesh-granularity string        The granularity of the network mesh to create. Possible values: location, full (default "location")
+      --mesh-granularity string        The granularity of the network mesh to create. Possible values: location, full, cross (default "location")
       --mtu string                     The MTU of the WireGuard interface created by Kilo. Set to 'auto' to detect from the underlay interface. (default "auto")
       --port int                       The port over which WireGuard peers should communicate. (default 51820)
       --prioritise-private-addresses   Prefer to assign a private IP address to the node's endpoint.

docs/topology.md

@@ -47,6 +47,11 @@ kgctl graph | circo -Tsvg > cluster.svg
 
 <img src="./graphs/full-mesh.svg" />
 
+# Cross Mesh
+
+In this topology all nodes within the same location are not encrypted. Traffic to any other node outside of current location is encrypted
+with direct node-to-node encryption. To use this mesh specify `--mesh-granularity=cross`.
+
 ## Mixed 
 
 The `kilo.squat.ai/location` annotation can be used to create cluster mixing some fully meshed nodes and some nodes grouped by logical location.

e2e/cilium-cross-mesh.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+# Cilium-CNI counterpart of e2e/cross-mesh.sh. The Kilo DaemonSet is the
+# one applied by create_cilium_cluster (kilo-kind-cilium.yaml), which
+# already runs Kilo with --cni=false --compatibility=cilium. This suite
+# only annotates locations and switches granularity to "cross".
+setup_suite() {
+	_kubectl annotate node "$KIND_CLUSTER-control-plane" kilo.squat.ai/location=loc-a --overwrite
+	_kubectl annotate node "$KIND_CLUSTER-worker"        kilo.squat.ai/location=loc-a --overwrite
+	_kubectl annotate node "$KIND_CLUSTER-worker2"       kilo.squat.ai/location=loc-b --overwrite
+	# shellcheck disable=SC2016
+	_kubectl patch ds -n kube-system kilo -p '{"spec":{"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--cni=false","--compatibility=cilium","--mesh-granularity=cross","--kubeconfig=/etc/kubernetes/kubeconfig","--internal-cidr=$(NODE_IP)/32"]}]}}}}'
+	block_until_ready_by_name kube-system kilo-userspace
+}
+
+test_cilium_cross_mesh_connectivity() {
+	assert "retry 30 5 '' check_ping" "should be able to ping all Pods over Cilium VXLAN + Kilo cross mesh"
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings"
+	echo "sleep for 30s (one reconciliation period) and try again..."
+	sleep 30
+	assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings after reconciling"
+}
+
+test_cilium_cross_peer_topology() {
+	local CP_PEERS WORKER_PEERS WORKER2_PEERS
+	CP_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-control-plane" | grep -c '^\[Peer\]')
+	WORKER_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-worker"        | grep -c '^\[Peer\]')
+	WORKER2_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-worker2"      | grep -c '^\[Peer\]')
+	assert_equals "1" "$CP_PEERS"      "control-plane (loc-a) should have 1 peer (the loc-b node)"
+	assert_equals "1" "$WORKER_PEERS"  "worker (loc-a) should have 1 peer (the loc-b node)"
+	assert_equals "2" "$WORKER2_PEERS" "worker2 (loc-b) should have 2 peers (both loc-a nodes)"
+}

e2e/cilium-setup.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+# Bring up a kind cluster with Cilium as the CNI for the Cilium-mode e2e
+# suite. Counterpart of e2e/setup.sh, which provisions a cluster that
+# uses the Kilo bridge CNI.
+setup_suite() {
+	create_cilium_cluster "$(build_kind_config 2)"
+}

e2e/cilium-teardown.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+teardown_suite () {
+	if [ -n "$E2E_SKIP_TEARDOWN_ON_FAILURE" ]; then
+		return
+	fi
+	delete_cluster
+}

e2e/cross-mesh.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091
+. lib.sh
+
+# This suite exercises --mesh-granularity=cross on the bridge-CNI test
+# cluster. Cross drops the WireGuard tunnel between nodes that share a
+# location and expects the underlying CNI to handle intra-location
+# traffic over its own overlay (e.g. Cilium VXLAN). The Kilo bridge CNI
+# used by this kind cluster has no such overlay, so cross-location peer
+# topology can be validated here but pod-to-pod connectivity cannot —
+# that lives in the Cilium-CNI suite (e2e/cilium-cross-mesh.sh).
+
+setup_suite() {
+	# Place control-plane and the first worker into one location, and the
+	# second worker into another, so that "cross" produces tunnels only
+	# between the two locations and not within a single location.
+	_kubectl annotate node "$KIND_CLUSTER-control-plane" kilo.squat.ai/location=loc-a --overwrite
+	_kubectl annotate node "$KIND_CLUSTER-worker"        kilo.squat.ai/location=loc-a --overwrite
+	_kubectl annotate node "$KIND_CLUSTER-worker2"       kilo.squat.ai/location=loc-b --overwrite
+	# shellcheck disable=SC2016
+	_kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=cross"]}]}}}}'
+	block_until_ready_by_name kube-system kilo-userspace
+}
+
+# Restore the cluster to a clean state for the suites that follow
+# (multi-cluster.sh, handlers.sh, kgctl.sh): remove the location
+# annotations this suite added and roll the DaemonSet back to
+# --mesh-granularity=location, matching the state location-mesh.sh
+# leaves behind.
+teardown_suite() {
+	_kubectl annotate node "$KIND_CLUSTER-control-plane" kilo.squat.ai/location- 2>/dev/null || true
+	_kubectl annotate node "$KIND_CLUSTER-worker"        kilo.squat.ai/location- 2>/dev/null || true
+	_kubectl annotate node "$KIND_CLUSTER-worker2"       kilo.squat.ai/location- 2>/dev/null || true
+	# shellcheck disable=SC2016
+	_kubectl patch ds -n kube-system kilo -p '{"spec": {"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--kubeconfig=/etc/kubernetes/kubeconfig","--mesh-granularity=location"]}]}}}}'
+	block_until_ready_by_name kube-system kilo-userspace
+}
+
+test_cross_mesh_peer() {
+	check_peer wg99 e2e 10.5.0.1/32 cross
+}
+
+test_mesh_granularity_auto_detect() {
+	assert_equals "$(_kgctl graph)" "$(_kgctl graph --mesh-granularity cross)"
+}
+
+# In "cross" granularity, every node in another location must appear as a
+# WireGuard peer (direct tunnels across locations), while nodes in the same
+# location must NOT appear as peers (intra-location traffic stays on the CNI).
+# In "location" the same-location worker would not have any [Peer] entry at
+# all (it is a non-leader); in "full" both same- and cross-location nodes
+# would appear as peers. This sanity-checks that "cross" sits in between.
+test_cross_peer_topology() {
+	local CP_PEERS WORKER_PEERS WORKER2_PEERS
+	CP_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-control-plane" | grep -c '^\[Peer\]')
+	WORKER_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-worker"        | grep -c '^\[Peer\]')
+	WORKER2_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-worker2"      | grep -c '^\[Peer\]')
+	# Each loc-a node should peer only with the single loc-b node.
+	assert_equals "1" "$CP_PEERS"      "control-plane (loc-a) should have 1 peer (the loc-b node)"
+	assert_equals "1" "$WORKER_PEERS"  "worker (loc-a) should have 1 peer (the loc-b node)"
+	# The loc-b node should peer with both loc-a nodes.
+	assert_equals "2" "$WORKER2_PEERS" "worker2 (loc-b) should have 2 peers (both loc-a nodes)"
+}

e2e/kilo-kind-cilium.yaml

@@ -0,0 +1,146 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+- kind: ServiceAccount
+  name: kilo
+  namespace: kube-system
+---
+# Kilo DaemonSet for the Cilium e2e suite. The CNI is provided by Cilium
+# (no kilo CNI ConfigMap and no install-cni init container), so Kilo runs
+# in --cni=false / --compatibility=cilium mode and only manages the WG mesh
+# on top of Cilium's overlay.
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo-userspace
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo-userspace
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo-userspace
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo:test
+        imagePullPolicy: Never
+        args:
+        - --hostname=$(NODE_NAME)
+        - --create-interface=false
+        - --cni=false
+        - --compatibility=cilium
+        - --mesh-granularity=full
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --internal-cidr=$(NODE_IP)/32
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: NODE_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
+      - name: wireguard
+        image: ghcr.io/masipcat/wireguard-go-docker:0.0.20230223
+        args:
+        - wireguard-go
+        - --foreground
+        - kilo0
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: wireguard
+          mountPath: /var/run/wireguard
+          readOnly: false
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: wireguard
+        hostPath:
+          path: /var/run/wireguard
+      - name: kubeconfig
+        secret:
+          secretName: kubeconfig