diff --git a/Makefile b/Makefile index 055a79e4..8d10666d 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,9 @@ MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_ else ifeq ($(MINI_LAB_FLAVOR),capms) LAB_TOPOLOGY=mini-lab.capms.yaml MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION) +else ifeq ($(MINI_LAB_FLAVOR),kamaji) +LAB_TOPOLOGY=mini-lab.kamaji.yaml +KAMAJI_ENABLED=true else ifeq ($(MINI_LAB_FLAVOR),gardener) GARDENER_ENABLED=true # usually gardener restricts the maximum version for k8s: @@ -163,6 +166,7 @@ cleanup-partition: sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.dell_sonic.yaml sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.capms.yaml + sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.kamaji.yaml docker network rm --force mini_lab_ext .PHONY: _privatenet diff --git a/README.md b/README.md index 057191fd..bdbf2a4d 100644 --- a/README.md +++ b/README.md @@ -190,11 +190,12 @@ make power-- ## Flavors -There are four flavors of the mini-lab environment: +All available mini-lab flavors are listed below: - `sonic`: runs two Community SONiC switches - `dell_sonic`: runs two Enterprise SONiC switches with a [locally built vrnetlab image](https://github.com/srl-labs/vrnetlab/tree/master/dell/dell_sonic) - `capms`: runs the `dell_sonic` flavor but with four instead of two machines (this is used for [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack) in order to have dedicated hosts for control plane / worker / firewall) +- `kamaji`: runs a variation of the `sonic` flavor. The working example is available at the [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack)'s `capi-lab`. - `gardener`: runs the `sonic` flavor and installs the [Gardener](https://gardener.cloud) in the mini-lab In order to start specific flavor, you can define the flavor as follows: diff --git a/compose.yaml b/compose.yaml index 8c2aa709..eb2967ef 100644 --- a/compose.yaml +++ b/compose.yaml @@ -21,6 +21,7 @@ services: - DOCKER_HUB_USER=${DOCKER_HUB_USER} - DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN} - GARDENER_ENABLED=${GARDENER_ENABLED:-} + - KAMAJI_ENABLED=${KAMAJI_ENABLED:-} - MONITORING_ENABLED=${MONITORING_ENABLED:-} network_mode: host working_dir: /mini-lab diff --git a/deploy_control_plane.yaml b/deploy_control_plane.yaml index 6fe5a2b4..9fa47a36 100644 --- a/deploy_control_plane.yaml +++ b/deploy_control_plane.yaml @@ -41,3 +41,7 @@ - name: deploy gardener import_playbook: deploy_gardener.yaml when: gardener_enabled + +- name: deploy kamaji + import_playbook: deploy_kamaji.yaml + when: kamaji_enabled diff --git a/deploy_kamaji.yaml b/deploy_kamaji.yaml new file mode 100644 index 00000000..7d1f0488 --- /dev/null +++ b/deploy_kamaji.yaml @@ -0,0 +1,13 @@ +--- +- name: deploy kamaji + hosts: control-plane + connection: local + gather_facts: false + + roles: + - name: ansible-common + tags: always + - name: kamaji + tags: kamaji + # vars: + # metal_control_plane_host_provider: metal diff --git a/docs/overview-kamaji.drawio.svg b/docs/overview-kamaji.drawio.svg new file mode 100644 index 00000000..f6e083c1 --- /dev/null +++ b/docs/overview-kamaji.drawio.svg @@ -0,0 +1,751 @@ + + + + + + + + + + + + + + + + + +
+
+
+ + linux-desktop + +
+
+
+ + + linux-desktop + + + + + + + + + + + + + + + + + +
+
+
+ + + containerlab + + +
+
+
+ + + containerlab + + + + + + + + + + + + + +
+
+
+ + + kind-cluster + + +
+
+
+ + + kind-cluster + + + + + + + + + + + + + + +
+
+
+ metal-stack +
+
+
+ + + metal-stack + + + + + + + + + + + + + + + + + + + + +
+
+
+ partition +
+
+
+ + + partition + + + + + + + + + + + + + + +
+
+
+ + machine02/ + +
+ + firewall + +
+
+
+
+ + + machine02/... + + + + + + + + + + + +
+
+
+ tenantnetwork +
+
+
+ + + tenantnetwork + + + + + + + + + + + + + + + + + + +
+
+
+ leaf02 +
+
+
+ + + leaf02 + + + + + + + + + + + +
+
+
+ metal-core +
+
+
+ + + metal-core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ masterdata-api +
+
+
+ + + masterdata-api + + + + + + + + + + + auditing + + + + + + + + + + + + + +
+
+
+ metal-db +
+
+
+ + + metal-db + + + + + + + + + + + + + + +
+
+
+ ipam-db +
+
+
+ + + ipam-db + + + + + + + + + + + +
+
+
+ ipam +
+
+
+ + + ipam + + + + + + + + + + + +
+
+
+ nsqd +
+
+
+ + + nsqd + + + + + + + + + + + +
+
+
+ metal-api +
+
+
+ + + metal-api + + + + + + + + + + + + + + + + + +
+
+
+ masterdata-db +
+
+
+ + + masterdata-... + + + + + + + + + + + + +
+
+
+ capi-lab +
+
+
+ + + capi-lab + + + + + + + + + + + +
+
+
+ docker network 172.17.0.1/16 +
+
+
+ + + docker network 172.17.0.1/16 + + + + + + + + + + + +
+
+
+ Ingress-Controller (NGINX) +
+
+
+ + + Ingress-Controller (... + + + + + + + + + + + + + + +
+
+
+ machine01 +
+
+
+ + + machine01 + + + + + + + + + + + +
+
+
+ exit +
+
+
+ + + exit + + + + + + + + + + + + + + + +
+
+
+ leaf01 +
+
+
+ + + leaf01 + + + + + + + + + + + + + + +
+
+
+ metal-core +
+
+
+ + + metal-core + + + + + + + + + + + + + + +
+
+
+ veth link +
+
+
+ + + veth link + + + + + + + + + + + + + + +
+
+
+ Kamaji Management Cluster +
+
+
+ + + Kamaji Management Cluster + + + + + + + + + + + + + + + + + +
+
+
+ tenant control plane A +
+
+
+ + + tenant control plane A + + + + + + + + + + + + + + Multi-Tenant + + + Datastore + + + + + + + + + + + + + + + + + +
+
+
+ tenant control plane B +
+
+
+ + + tenant control plane B + + + + + + + + + + + +
+
+
+ CAPMS (ClusterAPI Provider) +
+
+
+ + + CAPMS (ClusterAPI Provider) + + + + + + + + + + + + + Text is not SVG - cannot display + + + + \ No newline at end of file diff --git a/inventories/group_vars/control-plane/kamaji.yaml b/inventories/group_vars/control-plane/kamaji.yaml new file mode 100644 index 00000000..0e4c6f4d --- /dev/null +++ b/inventories/group_vars/control-plane/kamaji.yaml @@ -0,0 +1,2 @@ +--- +kamaji_enabled: "{{ lookup('env', 'KAMAJI_ENABLED') | default('', false) }}" diff --git a/mini-lab.kamaji.yaml b/mini-lab.kamaji.yaml new file mode 100644 index 00000000..49b50ac9 --- /dev/null +++ b/mini-lab.kamaji.yaml @@ -0,0 +1,70 @@ +# flavor for mini-lab kamaji with cluster-api-provider-metal-stack +name: mini-lab +prefix: "" + +mgmt: + network: mini_lab_ext + +topology: + defaults: + kind: linux + nodes: + # make metal-control-plane accessible via mini_lab_ext + metal-control-plane-control-plane: + kind: ext-container + exec: + - ip route add 203.0.113.128/25 via 203.0.113.128 dev eth0 + exit: + image: quay.io/frrouting/frr:10.3.0 + network-mode: none + binds: + - files/exit/daemons:/etc/frr/daemons + - files/exit/frr.conf:/etc/frr/frr.conf + - files/exit/vtysh.conf:/etc/frr/vtysh.conf + - files/exit/network.sh:/root/network.sh + exec: + - sh /root/network.sh + external_service: + image: docker.io/library/nginx:alpine-slim + network-mode: none + binds: + - files/external_service/network.sh:/root/network.sh + exec: + - sh /root/network.sh + mini_lab_ext: + kind: bridge + leaf01: + group: leaves + image: ${MINI_LAB_SONIC_IMAGE} + binds: + - files/ssh/id_rsa.pub:/authorized_keys + env: + QEMU_MEMORY: 4096 + leaf02: + group: leaves + image: ${MINI_LAB_SONIC_IMAGE} + binds: + - files/ssh/id_rsa.pub:/authorized_keys + env: + QEMU_MEMORY: 4096 + machine01: + group: machines + image: ${MINI_LAB_VM_IMAGE} + env: + UUID: 00000000-0000-0000-0000-000000000001 + machine02: + group: machines + image: ${MINI_LAB_VM_IMAGE} + env: + UUID: 00000000-0000-0000-0000-000000000002 + links: + - endpoints: ["exit:mini_lab_ext", "mini_lab_ext:exit"] + mtu: 9000 + - endpoints: ["external_service:mini_lab_ext", "mini_lab_ext:external_service"] + mtu: 9000 + - endpoints: ["leaf01:Ethernet0", "machine01:lan0"] + - endpoints: ["leaf02:Ethernet0", "machine01:lan1"] + - endpoints: ["leaf01:Ethernet1", "machine02:lan0"] + - endpoints: ["leaf02:Ethernet1", "machine02:lan1"] + - endpoints: ["leaf01:Ethernet120", "exit:eth1"] + - endpoints: ["leaf02:Ethernet120", "exit:eth2"] diff --git a/roles/kamaji/tasks/main.yaml b/roles/kamaji/tasks/main.yaml new file mode 100644 index 00000000..568c3da5 --- /dev/null +++ b/roles/kamaji/tasks/main.yaml @@ -0,0 +1,78 @@ +--- +- name: Add jetstack repo + kubernetes.core.helm_repository: + name: jetstack + repo_url: https://charts.jetstack.io + +- name: Add metallb repo + kubernetes.core.helm_repository: + name: metallb + repo_url: https://metallb.github.io/metallb + +- name: Add clastix repo + kubernetes.core.helm_repository: + name: clastix + repo_url: https://clastix.github.io/charts + +- name: Deploy cert manager + kubernetes.core.helm: + chart_ref: jetstack/cert-manager + name: cert-manager + release_namespace: cert-manager + create_namespace: true + wait: true + set_values: + - value: installCRDs=true + +# +# Claude Opus 4.6 was used to generate the following Ansible tasks based on the provided code snippets and context of +# the Getting Started guide for Kamaji with kind: https://kamaji.clastix.io/getting-started/kamaji-kind/ +# They are intended to install metallb with an address pool that matches the kind network. +- name: Deploy metallb + kubernetes.core.helm: + chart_ref: metallb/metallb + name: metallb + release_namespace: metallb-system + create_namespace: true + wait: true + +- name: Read cluster nodes + kubernetes.core.k8s_info: + api_version: v1 + kind: Node + register: cluster_nodes + +- name: Find node InternalIP + ansible.builtin.set_fact: + kind_node_ip: '{{ item.address }}' + loop: '{{ cluster_nodes.resources[0].status.addresses }}' + when: item.type == 'InternalIP' + +- name: Derive MetalLB network prefix + ansible.builtin.set_fact: + metallb_kind_net_prefix: "{{ kind_node_ip | regex_replace('^([0-9]+\\.[0-9]+)\\..*$', '\\1') }}" + +- name: Apply MetalLB IP address pool + kubernetes.core.k8s: + state: present + template: metallb-kind-ip-pool.yaml +# + +- name: Deploy kamaji + kubernetes.core.helm: + chart_ref: clastix/kamaji + chart_version: 0.0.0+latest + name: kamaji + release_namespace: kamaji-system + create_namespace: true + wait: true + set_values: + - value: image.tag=latest + - value: resources=null + +- name: Create kamaji tenant cluster namespace + kubernetes.core.k8s: + name: kamaji-tenants + api_version: v1 + kind: Namespace + state: present diff --git a/roles/kamaji/templates/metallb-kind-ip-pool.yaml b/roles/kamaji/templates/metallb-kind-ip-pool.yaml new file mode 100644 index 00000000..ad64657e --- /dev/null +++ b/roles/kamaji/templates/metallb-kind-ip-pool.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: kind-ip-pool + namespace: metallb-system +spec: + addresses: + - {{ metallb_kind_net_prefix }}.255.200-{{ metallb_kind_net_prefix }}.255.250 +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: empty + namespace: metallb-system \ No newline at end of file