diff --git a/README.md b/README.md index 56c18e3..97f0925 100644 --- a/README.md +++ b/README.md @@ -60,15 +60,37 @@ InterlinkMachine The controller creates a Pod and ClusterIP Service named `-plugin`, then derives the address as `http://-plugin..svc.cluster.local:`. +The [interlink-apptainer-plugin](https://github.com/interlink-hq/interlink-apptainer-plugin) +reads its configuration from a YAML file. Create a ConfigMap with the file and +mount it into the plugin container via `volumes` / `volumeMounts`: + ```yaml +# ConfigMap holding the Apptainer plugin configuration. +apiVersion: v1 +kind: ConfigMap +metadata: + name: apptainer-plugin-config + namespace: default +data: + ApptainerConfig.yaml: | + SidecarPort: "4000" + DataRootFolder: "/tmp/.interlink/" + ApptainerPath: "/usr/bin/apptainer" + ApptainerDefaultOptions: + - "--no-eval" + - "--containall" + ImagePrefix: "docker://" + BashPath: /bin/bash + ExportPodData: true + VerboseLogging: false + ErrorsOnlyLogging: false + EnableProbes: false +--- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: InterlinkMachine metadata: name: my-virtual-node spec: - # Optional: override the derived interLinkAddress - # interLinkAddress: "http://custom-endpoint:3000" - nodeName: "my-virtual-node" resources: cpu: "8" @@ -82,8 +104,8 @@ spec: effect: NoSchedule pluginSpec: - # Container image running the interLink plugin binary. - image: "ghcr.io/interlink-hq/interlink/plugin-apptainer:latest" + # Apptainer plugin image from https://github.com/interlink-hq/interlink-apptainer-plugin. + image: "ghcr.io/interlink-hq/interlink-apptainer-plugin:latest" # TCP port the plugin listens on (default: 4000). port: 4000 @@ -99,10 +121,20 @@ spec: operator: Exists effect: NoSchedule - # Plugin-specific environment variables. + # Point the plugin at the mounted configuration file. env: - - name: INTERLINK_PORT - value: "4000" + - name: APPTAINERCONFIGPATH + value: "/etc/interlink/ApptainerConfig.yaml" + + # Mount the ConfigMap so the plugin can read its configuration. + volumes: + - name: plugin-config + configMap: + name: apptainer-plugin-config + volumeMounts: + - name: plugin-config + mountPath: /etc/interlink + readOnly: true # Compute resources for the plugin container. resources: diff --git a/api/v1alpha1/interlinkmachine_types.go b/api/v1alpha1/interlinkmachine_types.go index d2f01bf..0fd190c 100644 --- a/api/v1alpha1/interlinkmachine_types.go +++ b/api/v1alpha1/interlinkmachine_types.go @@ -85,6 +85,18 @@ type PluginPodSpec struct { // Resources specifies the compute resources required by the plugin container. // +optional Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // VolumeMounts describe volume mounts for the plugin container. + // Use this together with Volumes to inject files (e.g. a ConfigMap containing + // ApptainerConfig.yaml) into the container at a specific path. + // +optional + VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` + + // Volumes is a list of volumes that can be mounted by the plugin container. + // Use this to make ConfigMaps, Secrets, or host paths available inside + // the container (e.g. mounting an apptainer configuration file). + // +optional + Volumes []corev1.Volume `json:"volumes,omitempty"` } // VirtualNodeResources describes the capacity of a virtual node. diff --git a/controllers/controllers_test.go b/controllers/controllers_test.go index f22cad0..5214e21 100644 --- a/controllers/controllers_test.go +++ b/controllers/controllers_test.go @@ -582,3 +582,87 @@ func TestInterlinkMachineReconciler_PluginPodMode_DefaultPort(t *testing.T) { } func boolPtr(b bool) *bool { return &b } + +func TestInterlinkMachineReconciler_PluginPodMode_VolumesAndMounts(t *testing.T) { + g := NewWithT(t) + scheme := buildScheme(t) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cluster", Namespace: "default"}, + } + machine := &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "apptainer-machine", + Namespace: "default", + Labels: map[string]string{clusterv1.ClusterNameLabel: "test-cluster"}, + }, + Spec: clusterv1.MachineSpec{ClusterName: "test-cluster"}, + } + interlinkMachine := &infrav1.InterlinkMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "apptainer-machine", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + {APIVersion: clusterv1.GroupVersion.String(), Kind: "Machine", Name: machine.Name, UID: machine.UID, Controller: boolPtr(true)}, + }, + }, + Spec: infrav1.InterlinkMachineSpec{ + NodeName: "virtual-node-apptainer", + PluginSpec: &infrav1.PluginPodSpec{ + Image: "ghcr.io/interlink-hq/interlink-apptainer-plugin:latest", + Port: 4000, + Env: []corev1.EnvVar{ + {Name: "APPTAINERCONFIGPATH", Value: "/etc/interlink/ApptainerConfig.yaml"}, + }, + Volumes: []corev1.Volume{ + { + Name: "plugin-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "apptainer-plugin-config"}, + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "plugin-config", MountPath: "/etc/interlink", ReadOnly: true}, + }, + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(cluster, machine, interlinkMachine). + WithStatusSubresource(interlinkMachine). + Build() + + reconciler := &controllers.InterlinkMachineReconciler{ + Client: fakeClient, + Scheme: scheme, + Log: ctrl.Log.WithName("test"), + } + + result, err := reconciler.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "default", Name: "apptainer-machine"}, + }) + g.Expect(err).ToNot(HaveOccurred()) + // Plugin pod is not Running yet; should requeue. + g.Expect(result.RequeueAfter).ToNot(BeZero()) + + // Verify the plugin Pod has the expected volumes and volumeMounts. + pod := &corev1.Pod{} + g.Expect(fakeClient.Get(context.Background(), types.NamespacedName{Namespace: "default", Name: "apptainer-machine-plugin"}, pod)).To(Succeed()) + g.Expect(pod.Spec.Containers[0].Image).To(Equal("ghcr.io/interlink-hq/interlink-apptainer-plugin:latest")) + g.Expect(pod.Spec.Volumes).To(HaveLen(1)) + g.Expect(pod.Spec.Volumes[0].Name).To(Equal("plugin-config")) + g.Expect(pod.Spec.Volumes[0].ConfigMap).ToNot(BeNil()) + g.Expect(pod.Spec.Volumes[0].ConfigMap.Name).To(Equal("apptainer-plugin-config")) + g.Expect(pod.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) + g.Expect(pod.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("plugin-config")) + g.Expect(pod.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/interlink")) + g.Expect(pod.Spec.Containers[0].VolumeMounts[0].ReadOnly).To(BeTrue()) + g.Expect(pod.Spec.Containers[0].Env).To(ContainElement( + corev1.EnvVar{Name: "APPTAINERCONFIGPATH", Value: "/etc/interlink/ApptainerConfig.yaml"}, + )) +} diff --git a/controllers/interlinkmachine_controller.go b/controllers/interlinkmachine_controller.go index ea6fdc8..e19d411 100644 --- a/controllers/interlinkmachine_controller.go +++ b/controllers/interlinkmachine_controller.go @@ -451,6 +451,7 @@ func buildPluginPodSpec(spec *infrav1.PluginPodSpec, port int32) corev1.PodSpec NodeSelector: spec.NodeSelector, Tolerations: spec.Tolerations, RestartPolicy: corev1.RestartPolicyAlways, + Volumes: spec.Volumes, Containers: []corev1.Container{ { Name: "plugin", @@ -462,8 +463,9 @@ func buildPluginPodSpec(spec *infrav1.PluginPodSpec, port int32) corev1.PodSpec Protocol: corev1.ProtocolTCP, }, }, - Env: spec.Env, - Resources: spec.Resources, + Env: spec.Env, + Resources: spec.Resources, + VolumeMounts: spec.VolumeMounts, }, }, } diff --git a/examples/pilot.yaml b/examples/pilot.yaml index 66b8a60..b33c40d 100644 --- a/examples/pilot.yaml +++ b/examples/pilot.yaml @@ -1,8 +1,9 @@ --- -# Pilot-mode example: on-demand interLink plugin instantiation. +# Pilot-mode example: on-demand interLink plugin instantiation using the +# Apptainer plugin (https://github.com/interlink-hq/interlink-apptainer-plugin). # # In this mode the controller creates a Pod (and a ClusterIP Service) that runs -# the interLink plugin binary on an *existing* virtual node that has access to +# the interLink apptainer plugin on an *existing* virtual node that has access to # the remote resource provider (e.g. an HPC cluster running Apptainer). # The Service address is automatically used as the interLinkAddress for the # VirtualNode, so no external interLink deployment is required beforehand. @@ -18,6 +19,30 @@ # The same InterlinkMachineTemplate can be referenced by a MachineDeployment # and scaled by the Cluster Autoscaler. +# -- Apptainer plugin configuration ------------------------------------------ +# The apptainer plugin reads its configuration from a YAML file. +# Mount this ConfigMap into the plugin Pod as /etc/interlink/ApptainerConfig.yaml. +# Adjust the values below to match your environment. +apiVersion: v1 +kind: ConfigMap +metadata: + name: apptainer-plugin-config + namespace: default +data: + ApptainerConfig.yaml: | + SidecarPort: "4000" + DataRootFolder: "/tmp/.interlink/" + ApptainerPath: "/usr/bin/apptainer" + ApptainerDefaultOptions: + - "--no-eval" + - "--containall" + ImagePrefix: "docker://" + BashPath: /bin/bash + ExportPodData: true + VerboseLogging: false + ErrorsOnlyLogging: false + EnableProbes: false +--- # -- Cluster ----------------------------------------------------------------- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster @@ -105,8 +130,10 @@ spec: effect: NoSchedule # Pilot-mode configuration. pluginSpec: - # Image that runs the interLink plugin binary (e.g. Apptainer plugin). - image: "ghcr.io/interlink-hq/interlink/plugin-apptainer:latest" + # Apptainer plugin image from https://github.com/interlink-hq/interlink-apptainer-plugin. + # Build the image from source and push it to a registry accessible from the cluster, + # or use a pre-built image if one is available. + image: "ghcr.io/interlink-hq/interlink-apptainer-plugin:latest" # TCP port the plugin listens on inside the container (default: 4000). port: 4000 # Schedule the plugin Pod onto an existing virtual node backed by the @@ -118,12 +145,21 @@ spec: - key: virtual-node.interlink.eu operator: Exists effect: NoSchedule - # Plugin-specific configuration passed as environment variables. + # Environment variables for the apptainer plugin. + # APPTAINERCONFIGPATH tells the plugin where to find its config file. + # SHARED_FS=true enables shared-filesystem mode (set by the container entrypoint). env: - - name: INTERLINK_PORT - value: "4000" - - name: APPTAINER_TMPDIR - value: "/tmp/apptainer" + - name: APPTAINERCONFIGPATH + value: "/etc/interlink/ApptainerConfig.yaml" + # Mount the ConfigMap defined above so the plugin can read its configuration. + volumes: + - name: plugin-config + configMap: + name: apptainer-plugin-config + volumeMounts: + - name: plugin-config + mountPath: /etc/interlink + readOnly: true resources: requests: cpu: "500m"