Pods: Gang Scheduling & Topology awareness

Makefile

@@ -6,7 +6,7 @@ GOBIN=$(shell go env GOBIN)
 endif
 
 .PHONY: all
-all: crds deepcopy lint test
+all: crds generate lint test
 
 .PHONY: help
 help: ## Display this help.
@@ -28,9 +28,53 @@ test: ## Run all tests.
 crds: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) rbac:roleName=manager-role crd:allowDangerousTypes=true webhook paths="./..." output:crd:artifacts:config=helm/library/cortex/files/crds
 
-.PHONY: deepcopy
-deepcopy: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
-	$(CONTROLLER_GEN) crd:allowDangerousTypes=true object:headerFile="hack/boilerplate.go.txt" paths="./..." output:crd:artifacts:config=helm/library/cortex/files/crds
+.PHONY: generate
+generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations as well as client, informer and lister implementations.
+	$(CONTROLLER_GEN) crd:allowDangerousTypes=true object:headerFile="hack/boilerplate.go.txt" paths="./..."
+	hack/generate-code.sh
+
+.PHONY: dekustomize
+dekustomize:
+	@echo "Backing up stuff that shouldn't be overridden by kubebuilder-helm..."
+	@TEMP_DIR=$$(mktemp -d); \
+	if [ -d "dist/chart/templates/rbac" ]; then \
+		cp -r dist/chart/templates/rbac "$$TEMP_DIR/rbac"; \
+	fi; \
+	if [ -d "dist/chart/templates/prometheus" ]; then \
+		cp -r dist/chart/templates/prometheus "$$TEMP_DIR/prometheus"; \
+	fi; \
+	if [ -d "dist/chart/templates/metrics" ]; then \
+		cp -r dist/chart/templates/metrics "$$TEMP_DIR/metrics"; \
+	fi; \
+	if [ -d ".github" ]; then \
+		cp -r .github "$$TEMP_DIR/github"; \
+	fi; \
+	echo "Generating helm chart..."; \
+	kubebuilder edit --plugins=helm/v1-alpha; \
+	echo "Restoring stuff that shouldn't be overridden by kubebuilder-helm..."; \
+	if [ -d "$$TEMP_DIR/rbac" ]; then \
+		rm -rf dist/chart/templates/rbac; \
+		cp -r "$$TEMP_DIR/rbac" dist/chart/templates/rbac; \
+	fi; \
+	if [ -d "$$TEMP_DIR/prometheus" ]; then \
+		rm -rf dist/chart/templates/prometheus; \
+		cp -r "$$TEMP_DIR/prometheus" dist/chart/templates/prometheus; \
+	fi; \
+	if [ -d "$$TEMP_DIR/metrics" ]; then \
+		rm -rf dist/chart/templates/metrics; \
+		cp -r "$$TEMP_DIR/metrics" dist/chart/templates/metrics; \
+	fi; \
+	if [ -d "$$TEMP_DIR/github" ]; then \
+		rm -rf .github; \
+		cp -r "$$TEMP_DIR/github" .github; \
+	fi; \
+	rm -rf "$$TEMP_DIR"; \
+	echo "Directories restored successfully."
+
+##@ Build
+
+.PHONY: build
+build: manifests generate dekustomize
 
 LOCALBIN ?= $(shell pwd)/bin
 $(LOCALBIN):

Tiltfile

@@ -200,10 +200,6 @@ if 'pods' in ACTIVE_DEPLOYMENTS:
     print("Activating Cortex Pods bundle")
     k8s_yaml(helm('./helm/bundles/cortex-pods', name='cortex-pods', values=tilt_values),)
     k8s_resource('cortex-pods-controller-manager', labels=['Cortex-Pods'])
-    # Deploy example resources
-    k8s_yaml('samples/pods/node.yaml')
-    k8s_yaml('samples/pods/pod.yaml')
-    k8s_resource('test-pod', labels=['Cortex-Pods'])
 
 ########### Dev Dependencies
 local('sh helm/sync.sh helm/dev/cortex-prometheus-operator')

api/delegation/pods/messages.go

@@ -14,7 +14,7 @@ type PodPipelineRequest struct {
 	// The available nodes.
 	Nodes []corev1.Node `json:"nodes"`
 	// The pod to be scheduled.
-	Pod corev1.Pod `json:"pod"`
+	Pod *corev1.Pod `json:"pod"`
 }
 
 func (r PodPipelineRequest) GetSubjects() []string {

api/v1alpha1/groupversion_info.go

@@ -15,9 +15,17 @@ var (
 	// GroupVersion is group version used to register these objects.
 	GroupVersion = schema.GroupVersion{Group: "cortex.cloud", Version: "v1alpha1"}
 
+	// SchemeGroupVersion is group version used to register these objects.
+	SchemeGroupVersion = GroupVersion
+
 	// SchemeBuilder is used to add go types to the GroupVersionKind scheme.
 	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
 
 	// AddToScheme adds the types in this group-version to the given scheme.
 	AddToScheme = SchemeBuilder.AddToScheme
 )
+
+// Resource takes an unqualified resource and returns a Group qualified GroupResource
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}

api/v1alpha1/podgroupset_types.go

@@ -0,0 +1,86 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package v1alpha1
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type PodGroupSpec struct {
+	Replicas int32          `json:"replicas"`
+	PodSpec  corev1.PodSpec `json:"podSpec"`
+}
+
+type PodGroup struct {
+	Name string       `json:"name"`
+	Spec PodGroupSpec `json:"spec"`
+}
+
+type PodGroupSetSpec struct {
+	PodGroups []PodGroup `json:"podGroups"`
+}
+
+type PodGroupSetPhase string
+
+const (
+	PodGroupSetPhasePending PodGroupSetPhase = "Pending"
+	// PodGroupSetPhaseScheduled means placements have been calculated for all pods
+	PodGroupSetPhaseScheduled PodGroupSetPhase = "Scheduled"
+	// PodGroupSetPhaseRunning means all pods have been created and are running
+	PodGroupSetPhaseRunning PodGroupSetPhase = "Running"
+	PodGroupSetPhaseFailed  PodGroupSetPhase = "Failed"
+)
+
+type PodPlacement struct {
+	PodName  string `json:"podName"`
+	NodeName string `json:"nodeName"`
+}
+
+type PodGroupSetStatus struct {
+	// +kubebuilder:validation:Optional
+	Phase PodGroupSetPhase `json:"phase,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	Placements []PodPlacement `json:"placements,omitempty"`
+
+	// The current status conditions of the pod group set.
+	// +kubebuilder:validation:Optional
+	Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
+}
+
+// +genclient
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Namespaced
+
+// PodGroupSet is the Schema for the podgroupsets API
+type PodGroupSet struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   PodGroupSetSpec   `json:"spec,omitempty"`
+	Status PodGroupSetStatus `json:"status,omitempty"`
+}
+
+// PodGroupSet.PodName constructs a unique identifier for its pods that is used
+// in the mapping of potential placements.
+func (pgs PodGroupSet) PodName(podGroupName string, replicaIdx int) string {
+	return fmt.Sprintf("%s-%s-%d", pgs.Name, podGroupName, replicaIdx)
+}
+
+// +kubebuilder:object:root=true
+
+// PodGroupSetList contains a list of PodGroupSet
+type PodGroupSetList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []PodGroupSet `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&PodGroupSet{}, &PodGroupSetList{})
+}