Docs request: cluster with multiple apps & namespaces example

[SophiaH67]

Hi,

I've been trying to convert my current setup to kubenix. I think some parts like the helm integration work really well and are easy to understand.
However, I'm now trying to move some of my own yaml's I wrote to Nix, but I cannot figure out how this project wants me to do namespaces.

I'd really appreciate if there was more documentation or an example project on using namespaces.

Thanks!

[adrian-gierakowski]

This might be helpful #90 (comment)

[SophiaH67]

Thanks for the swift reply! I had a look through the issue, the linked documentation, and the reporter's profile to see if I can find anything, but I still really do not understand how this submodule system is supposed to work.

Do you(or anyone else) maybe know of any examples or repositories which use this feature that I can study instead?

The existing documentation is a bit hard to understand for me 😓

[sheepforce]

and the reporter's profile to see if I can find anything

Ah sorry for that, could have told a little more how it actually looks like in the end.

My setup looks like this:

.
├── flake.lock
├── flake.nix
├── kubenix
│   ├── all.nix
│   ├── lib
│   │   └── namespaced.nix
│   ├── cephfs-csi
│   │   ├── default.nix
│   │   └── secrets.yaml
│   ├── rbd-hdd-csi
│   │   ├── default.nix
│   │   └── secrets.yaml
......

---

kubenix/lib/namespaced.nix

{ config
, kubenix
, lib
, name
, args
, ...
}: {
  imports = with kubenix.modules; [
    submodule
    k8s
    helm
  ];

  options.submodule.args = {
    kubernetes = lib.mkOption {
      description = "Kubernetes config to be applied to a specific namespace.";
      type = lib.types.attrs;
      default = { };
    };
  };

  config = {
    submodule = {
      name = "namespaced";
      passthru.kubernetes.objects = config.kubernetes.objects;
    };

    kubernetes = lib.mkMerge [
      { namespace = name; }
      { resources.namespaces.${name} = { }; }
      args.kubernetes
    ];
  };
}

---

kubenix/cephfs-csi/default.nix

{ config
, lib
, pkgs
, kubenix
, ...
}:
{
  imports = with kubenix.modules; [
    k8s
    helm
    submodules
  ];

  submodules.imports = [
    ../lib/namespaced.nix
  ];

  submodules.instances.cephfs-csi-nix = {
    submodule = "namespaced";

    args.kubernetes = {
      helm.releases = {
        cephfs-csi = {
          chart = kubenix.lib.helm.fetch {
            repo = "https://ceph.github.io/csi-charts";
            chart = "ceph-csi-cephfs";
            version = "3.15.0";
            sha256 = "sha256-DllLrjFIGdLK/Uh35LNG4QRspigcOR4XMIHMmcxZA6Q=";
          };

          namespace = "cephfs-csi-nix";

          values = {
            csiConfig = [{
              clusterID = "REDACTED";
              monitors = [
                "10.238.2.64:6789"
                "10.238.2.65:6789"
                "10.238.2.66:6789"
                "10.238.2.67:6789"
                "10.238.2.80:6789"
              ];
              cephFS.subvolumeGroup = "k8s-cluster1";
            }];
            secret = {
              create = true;
              userID = "k8s-cluster1";
              userKey = "ref+sops://kubenix/cephfs-csi/secrets.yaml#stringData/userKey";
            };
            selinuxMount = false;
            storageClass = {
              create = true;
              name = "cephfs-csi";
              clusterID = "REDACTED";
              fsName = "cephfs";
              reclaimPolicy = "Delete";
              allowVolumeExpansion = true;
              mounter = "kernel";
            };
          };
        };
      };

      resources = {
        persistentVolumeClaims.ceph-test-pvc.spec = {
          accessModes = [ "ReadWriteMany" ];
          storageClassName = "cephfs-csi";
          resources.requests.storage = "1Gi";
        };

        pods.ceph-test-pod.spec = {
          restartPolicy = "Always";
          volumes = [{
            name = "ceph-test-pvc";
            persistentVolumeClaim.claimName = "ceph-test-pvc";
          }];
          containers = [{
            name = "ceph-test-container";
            image = "registry.k8s.io/e2e-test-images/agnhost:2.39";

            volumeMounts = [{
              name = "ceph-test-pvc";
              mountPath = "/mnt/cephfs";
            }];
          }];
        };
      };
    };
  };
}

---

And kubenix/rbd-hdd-csi/default.nix

{ config
, lib
, pkgs
, kubenix
, ...
}:
{
  imports = with kubenix.modules; [
    k8s
    helm
    submodules
  ];

  submodules.imports = [
    ../lib/namespaced.nix
  ];

  submodules.instances.rbd-hdd-csi-nix = {
    submodule = "namespaced";

    args.kubernetes = {
      helm.releases = {
        rbd-csi = {
          chart = kubenix.lib.helm.fetch {
            repo = "https://ceph.github.io/csi-charts";
            chart = "ceph-csi-rbd";
            version = "3.15.0";
            sha256 = "sha256-jsYqhYKoWjaGzPott9aCrBGPNprllPOSu5Dmlnsg7Kk=";
          };

          namespace = "rbd-hdd-csi-nix";

          values = {
            csiConfig = [{
              clusterID = "REDACTED";
              monitors = [
                "10.238.2.64:6789"
                "10.238.2.65:6789"
                "10.238.2.66:6789"
                "10.238.2.67:6789"
                "10.238.2.80:6789"
              ];
              # rdb = { };
              # readAffinity = { };
            }];
            secret = {
              create = true;
              userID = "kubernetes";
              userKey = "ref+sops://kubenix/rbd-hdd-csi/secrets.yaml#stringData/userKey";
            };
            storageClass = {
              create = true;
              name = "rbd-hdd-csi";
              clusterID = "REDACTED";
              pool = "kubernetes";
              fsType = "ext4";
              reclaimPolicy = "Delete";
              allowVolumeExpansion = true;
            };
          };
        };
      };

      resources = {
        persistentVolumeClaims.rbd-hdd-test-pvc.spec = {
          accessModes = [ "ReadWriteMany" ];
          storageClassName = "rbd-hdd-csi";
          resources.requests.storage = "1Gi";
          volumeMode = "Block";
        };

        pods.ceph-test-pod.spec = {
          restartPolicy = "Always";
          volumes = [{
            name = "rbd-block-dev";
            persistentVolumeClaim.claimName = "rbd-hdd-test-pvc";
          }];
          containers = [{
            name = "ceph-test-container";
            image = "registry.k8s.io/e2e-test-images/agnhost:2.39";

            volumeDevices = [{
              devicePath = "/dev/rbdblock";
              name = "rbd-block-dev";
            }];
          }];
        };
      };
    };
  };
}

---

kubenix/all.nix

{ config
, lib
, pkgs
, kubenix
, ...
}:

{
  imports = with kubenix.modules; [
    k8s
    helm
    submodules
    
    ./cephfs-csi/default.nix
    ./rbd-hdd-csi/default.nix
    # And all others you would like to import
  ];

  submodules.imports = [
    ./lib/namespaced.nix
  ];
}

---

In flake.nix to make a kubenix for these deployments:

{
  description = "Kubenix deployments";
  inputs = {
     nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-25.05";
     kubenix.url = "github:sheepforce/kubenix/helm-proxy"; # Includes patch to fetch helm charts with a forward proxy
  };

  outputs =
    { self
    , nixpkgs
    , kubenix
    }:
    let
      system = "x86_64-linux";
      pkgs = import nixpkgs {inherit system; };
      kubenixPkg = kubenix.packages."${system}".default.override {
        module = import ./kubenix/all.nix;
      };
    in {
      packages."${system}".kubenix = kubenix = kubenixPkg;
      devShells."${system}".default = pkgs.mkShell {
        buildInputs = [ kubenixPkg ];
      };
    };
}

And then I'm just deploying via

export KUBECONFIG=$PATH_TO_MY_KUBECONFIG
nix develop --command kubenix apply --all

If you require deployments to another cluster, too, you will have to define kubenixPkg1, kubenixPkgs2 and so with different kubenix modules and set KUBECONFIG accordingly before running kubenix apply.

The yaml secrets are simply encrypted with SOPS like sops encrypt --in-place $YAML

[SophiaH67]

Thank you so much! This is so much clearer for me now