From 311d7385e1e0bad8285a142989eab28062c2af9d Mon Sep 17 00:00:00 2001 From: CYJiang Date: Wed, 10 Jun 2026 19:35:24 +0800 Subject: [PATCH] api: tighten runtime CRD validation Signed-off-by: CYJiang --- go.mod | 2 +- ...me.agentcube.volcano.sh_agentruntimes.yaml | 15 ++ ...agentcube.volcano.sh_codeinterpreters.yaml | 16 ++ pkg/apis/runtime/v1alpha1/agent_type.go | 5 + .../runtime/v1alpha1/codeinterpreter_types.go | 9 + .../runtime/v1alpha1/crd_validation_test.go | 220 ++++++++++++++++++ 6 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 pkg/apis/runtime/v1alpha1/crd_validation_test.go diff --git a/go.mod b/go.mod index f0a23c80..c637c9f7 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ toolchain go1.24.9 require ( github.com/agiledragon/gomonkey/v2 v2.13.0 github.com/alicebob/miniredis/v2 v2.35.0 - github.com/gin-contrib/gzip v1.0.1 github.com/fsnotify/fsnotify v1.9.0 + github.com/gin-contrib/gzip v1.0.1 github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt/v5 v5.2.2 github.com/google/uuid v1.6.0 diff --git a/manifests/charts/base/crds/runtime.agentcube.volcano.sh_agentruntimes.yaml b/manifests/charts/base/crds/runtime.agentcube.volcano.sh_agentruntimes.yaml index a6508b34..43564206 100644 --- a/manifests/charts/base/crds/runtime.agentcube.volcano.sh_agentruntimes.yaml +++ b/manifests/charts/base/crds/runtime.agentcube.volcano.sh_agentruntimes.yaml @@ -48,7 +48,11 @@ spec: description: |- MaxSessionDuration describes the maximum duration for a session. After this duration, the session will be terminated no matter active or inactive. + format: duration type: string + x-kubernetes-validations: + - message: maxSessionDuration must be greater than 0 + rule: duration(self) > duration('0s') podTemplate: description: PodTemplate describes the template that will be used to create an agent sandbox. @@ -8472,7 +8476,11 @@ spec: default: 15m description: SessionTimeout describes the duration after which an inactive session will be terminated. + format: duration type: string + x-kubernetes-validations: + - message: sessionTimeout must be greater than 0 + rule: duration(self) > duration('0s') targetPort: description: Ports is a list of ports that the agent runtime will expose. @@ -8486,10 +8494,13 @@ spec: description: |- PathPrefix is the path prefix to route to this port. For example, if PathPrefix is "/api", requests to "/api/..." will be routed to this port. + pattern: ^/.* type: string port: description: Port is the port number. format: int32 + maximum: 65535 + minimum: 1 type: integer protocol: default: HTTP @@ -8509,6 +8520,10 @@ spec: - sessionTimeout - targetPort type: object + x-kubernetes-validations: + - message: sessionTimeout must be less than or equal to maxSessionDuration + rule: '!has(self.sessionTimeout) || !has(self.maxSessionDuration) || + duration(self.sessionTimeout) <= duration(self.maxSessionDuration)' status: description: Status represents the current state of the AgentRuntime. properties: diff --git a/manifests/charts/base/crds/runtime.agentcube.volcano.sh_codeinterpreters.yaml b/manifests/charts/base/crds/runtime.agentcube.volcano.sh_codeinterpreters.yaml index a8cb4e3a..b133f7e2 100644 --- a/manifests/charts/base/crds/runtime.agentcube.volcano.sh_codeinterpreters.yaml +++ b/manifests/charts/base/crds/runtime.agentcube.volcano.sh_codeinterpreters.yaml @@ -64,7 +64,11 @@ spec: MaxSessionDuration describes the maximum duration for a code-interpreter session. After this duration, the session will be terminated regardless of activity, to prevent long-lived sandboxes from accumulating unbounded state. + format: duration type: string + x-kubernetes-validations: + - message: maxSessionDuration must be greater than 0 + rule: duration(self) > duration('0s') ports: description: |- Ports is a list of ports that the code interpreter runtime will expose. @@ -81,10 +85,13 @@ spec: description: |- PathPrefix is the path prefix to route to this port. For example, if PathPrefix is "/api", requests to "/api/..." will be routed to this port. + pattern: ^/.* type: string port: description: Port is the port number. format: int32 + maximum: 65535 + minimum: 1 type: integer protocol: default: HTTP @@ -104,7 +111,11 @@ spec: SessionTimeout describes the duration after which an inactive code-interpreter session will be terminated. Any sandbox that has not received requests within this duration is eligible for cleanup. + format: duration type: string + x-kubernetes-validations: + - message: sessionTimeout must be greater than 0 + rule: duration(self) > duration('0s') template: description: |- Template describes the template that will be used to create a code interpreter sandbox. @@ -405,10 +416,15 @@ spec: for this code interpreter runtime. Pre-warmed sandboxes can reduce startup latency for new sessions at the cost of additional resource usage. format: int32 + minimum: 0 type: integer required: - template type: object + x-kubernetes-validations: + - message: sessionTimeout must be less than or equal to maxSessionDuration + rule: '!has(self.sessionTimeout) || !has(self.maxSessionDuration) || + duration(self.sessionTimeout) <= duration(self.maxSessionDuration)' status: description: Status represents the current state of the CodeInterpreter. properties: diff --git a/pkg/apis/runtime/v1alpha1/agent_type.go b/pkg/apis/runtime/v1alpha1/agent_type.go index fe5413fa..64eaf2e1 100644 --- a/pkg/apis/runtime/v1alpha1/agent_type.go +++ b/pkg/apis/runtime/v1alpha1/agent_type.go @@ -38,6 +38,7 @@ type AgentRuntime struct { } // AgentRuntimeSpec describes how to create and manage agent runtime sandboxes. +// +kubebuilder:validation:XValidation:rule="!has(self.sessionTimeout) || !has(self.maxSessionDuration) || duration(self.sessionTimeout) <= duration(self.maxSessionDuration)",message="sessionTimeout must be less than or equal to maxSessionDuration" type AgentRuntimeSpec struct { // Ports is a list of ports that the agent runtime will expose. Ports []TargetPort `json:"targetPort"` @@ -48,12 +49,16 @@ type AgentRuntimeSpec struct { // SessionTimeout describes the duration after which an inactive session will be terminated. // +kubebuilder:validation:Required + // +kubebuilder:validation:Format=duration + // +kubebuilder:validation:XValidation:rule="duration(self) > duration('0s')",message="sessionTimeout must be greater than 0" // +kubebuilder:default="15m" SessionTimeout *metav1.Duration `json:"sessionTimeout,omitempty" protobuf:"bytes,2,opt,name=sessionTimeout"` // MaxSessionDuration describes the maximum duration for a session. // After this duration, the session will be terminated no matter active or inactive. // +kubebuilder:validation:Required + // +kubebuilder:validation:Format=duration + // +kubebuilder:validation:XValidation:rule="duration(self) > duration('0s')",message="maxSessionDuration must be greater than 0" // +kubebuilder:default="8h" MaxSessionDuration *metav1.Duration `json:"maxSessionDuration,omitempty" protobuf:"bytes,3,opt,name=maxSessionDuration"` } diff --git a/pkg/apis/runtime/v1alpha1/codeinterpreter_types.go b/pkg/apis/runtime/v1alpha1/codeinterpreter_types.go index c12154f3..f50fc411 100644 --- a/pkg/apis/runtime/v1alpha1/codeinterpreter_types.go +++ b/pkg/apis/runtime/v1alpha1/codeinterpreter_types.go @@ -42,6 +42,7 @@ type CodeInterpreter struct { } // CodeInterpreterSpec describes how to create and manage code-interpreter sandboxes. +// +kubebuilder:validation:XValidation:rule="!has(self.sessionTimeout) || !has(self.maxSessionDuration) || duration(self.sessionTimeout) <= duration(self.maxSessionDuration)",message="sessionTimeout must be less than or equal to maxSessionDuration" type CodeInterpreterSpec struct { // Ports is a list of ports that the code interpreter runtime will expose. // These ports are typically used by the router / apiserver to proxy HTTP or gRPC @@ -59,12 +60,16 @@ type CodeInterpreterSpec struct { // SessionTimeout describes the duration after which an inactive code-interpreter // session will be terminated. Any sandbox that has not received requests within // this duration is eligible for cleanup. + // +kubebuilder:validation:Format=duration + // +kubebuilder:validation:XValidation:rule="duration(self) > duration('0s')",message="sessionTimeout must be greater than 0" // +kubebuilder:default="15m" SessionTimeout *metav1.Duration `json:"sessionTimeout,omitempty"` // MaxSessionDuration describes the maximum duration for a code-interpreter session. // After this duration, the session will be terminated regardless of activity, to // prevent long-lived sandboxes from accumulating unbounded state. + // +kubebuilder:validation:Format=duration + // +kubebuilder:validation:XValidation:rule="duration(self) > duration('0s')",message="maxSessionDuration must be greater than 0" // +kubebuilder:default="8h" MaxSessionDuration *metav1.Duration `json:"maxSessionDuration,omitempty"` @@ -72,6 +77,7 @@ type CodeInterpreterSpec struct { // for this code interpreter runtime. Pre-warmed sandboxes can reduce startup // latency for new sessions at the cost of additional resource usage. // +optional + // +kubebuilder:validation:Minimum=0 WarmPoolSize *int32 `json:"warmPoolSize,omitempty"` // AuthMode specifies the authentication mode for the sandbox runtime. @@ -158,11 +164,14 @@ type TargetPort struct { // PathPrefix is the path prefix to route to this port. // For example, if PathPrefix is "/api", requests to "/api/..." will be routed to this port. // +optional + // +kubebuilder:validation:Pattern=`^/.*` PathPrefix string `json:"pathPrefix,omitempty"` // Name is the name of the port. // +optional Name string `json:"name,omitempty"` // Port is the port number. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 Port uint32 `json:"port"` // Protocol is the protocol of the port. // +kubebuilder:default=HTTP diff --git a/pkg/apis/runtime/v1alpha1/crd_validation_test.go b/pkg/apis/runtime/v1alpha1/crd_validation_test.go new file mode 100644 index 00000000..c4d6fe5e --- /dev/null +++ b/pkg/apis/runtime/v1alpha1/crd_validation_test.go @@ -0,0 +1,220 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "encoding/json" + "os" + "path/filepath" + goruntime "runtime" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/util/yaml" +) + +func TestGeneratedCRDValidationSchema(t *testing.T) { + tests := []struct { + name string + crdFile string + ports string + rule string + }{ + { + name: "AgentRuntime", + crdFile: "runtime.agentcube.volcano.sh_agentruntimes.yaml", + ports: "targetPort", + rule: "!has(self.sessionTimeout) || !has(self.maxSessionDuration) || duration(self.sessionTimeout) <= duration(self.maxSessionDuration)", + }, + { + name: "CodeInterpreter", + crdFile: "runtime.agentcube.volcano.sh_codeinterpreters.yaml", + ports: "ports", + rule: "!has(self.sessionTimeout) || !has(self.maxSessionDuration) || duration(self.sessionTimeout) <= duration(self.maxSessionDuration)", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + specSchema := loadCRDSpecSchema(t, tt.crdFile) + + portSchema := nestedMap(t, specSchema, "properties", tt.ports, "items", "properties", "port") + assertNumber(t, portSchema, "minimum", 1) + assertNumber(t, portSchema, "maximum", 65535) + + pathPrefixSchema := nestedMap(t, specSchema, "properties", tt.ports, "items", "properties", "pathPrefix") + assertString(t, pathPrefixSchema, "pattern", "^/.*") + + sessionTimeoutSchema := nestedMap(t, specSchema, "properties", "sessionTimeout") + assertString(t, sessionTimeoutSchema, "format", "duration") + assertHasValidationRule(t, sessionTimeoutSchema, "duration(self) > duration('0s')") + + maxSessionDurationSchema := nestedMap(t, specSchema, "properties", "maxSessionDuration") + assertString(t, maxSessionDurationSchema, "format", "duration") + assertHasValidationRule(t, maxSessionDurationSchema, "duration(self) > duration('0s')") + + assertHasValidationRule(t, specSchema, tt.rule) + }) + } +} + +func TestGeneratedCodeInterpreterWarmPoolSizeValidation(t *testing.T) { + specSchema := loadCRDSpecSchema(t, "runtime.agentcube.volcano.sh_codeinterpreters.yaml") + warmPoolSchema := nestedMap(t, specSchema, "properties", "warmPoolSize") + assertNumber(t, warmPoolSchema, "minimum", 0) +} + +func loadCRDSpecSchema(t *testing.T, crdFile string) map[string]interface{} { + t.Helper() + + _, currentFile, _, ok := goruntime.Caller(0) + if !ok { + t.Fatal("resolve current test file path") + } + path := filepath.Join(filepath.Dir(currentFile), "..", "..", "..", "..", "manifests", "charts", "base", "crds", crdFile) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read CRD %s: %v", path, err) + } + + jsonData, err := yaml.ToJSON(data) + if err != nil { + t.Fatalf("convert CRD %s to JSON: %v", crdFile, err) + } + + var crd map[string]interface{} + if err := json.Unmarshal(jsonData, &crd); err != nil { + t.Fatalf("decode CRD %s: %v", crdFile, err) + } + + versionSchema := crdVersionSchema(t, crd, "v1alpha1") + return nestedMap(t, versionSchema, "openAPIV3Schema", "properties", "spec") +} + +func crdVersionSchema(t *testing.T, crd map[string]interface{}, wantName string) map[string]interface{} { + t.Helper() + + spec := nestedMap(t, crd, "spec") + versions, ok := spec["versions"].([]interface{}) + if !ok { + t.Fatalf("spec.versions = %T(%v), want array", spec["versions"], spec["versions"]) + } + + var storageSchema map[string]interface{} + for _, version := range versions { + versionMap, ok := version.(map[string]interface{}) + if !ok { + t.Fatalf("CRD version = %T(%v), want map", version, version) + } + schema := nestedMap(t, versionMap, "schema") + if name, _ := versionMap["name"].(string); name == wantName { + return schema + } + if storage, _ := versionMap["storage"].(bool); storage { + storageSchema = schema + } + } + if storageSchema != nil { + return storageSchema + } + t.Fatalf("CRD versions do not include %q or a storage version", wantName) + return nil +} + +func nestedMap(t *testing.T, obj map[string]interface{}, path ...string) map[string]interface{} { + t.Helper() + + current := interface{}(obj) + for _, key := range path { + switch typed := current.(type) { + case map[string]interface{}: + var ok bool + current, ok = typed[key] + if !ok { + t.Fatalf("missing schema path %v at %q", path, key) + } + case []interface{}: + if key != "0" { + t.Fatalf("unsupported array path key %q in %v", key, path) + } + if len(typed) == 0 { + t.Fatalf("empty array at path %v", path) + } + current = typed[0] + default: + t.Fatalf("schema path %v reached non-object %T at %q", path, current, key) + } + } + + result, ok := current.(map[string]interface{}) + if !ok { + t.Fatalf("schema path %v resolved to %T, want map", path, current) + } + return result +} + +func assertNumber(t *testing.T, schema map[string]interface{}, field string, want float64) { + t.Helper() + + got, ok := schema[field].(float64) + if !ok { + t.Fatalf("schema field %q = %T(%v), want number %v", field, schema[field], schema[field], want) + } + if got != want { + t.Fatalf("schema field %q = %v, want %v", field, got, want) + } +} + +func assertString(t *testing.T, schema map[string]interface{}, field, want string) { + t.Helper() + + got, ok := schema[field].(string) + if !ok { + t.Fatalf("schema field %q = %T(%v), want string %q", field, schema[field], schema[field], want) + } + if got != want { + t.Fatalf("schema field %q = %q, want %q", field, got, want) + } +} + +func assertHasValidationRule(t *testing.T, schema map[string]interface{}, want string) { + t.Helper() + + validations, ok := schema["x-kubernetes-validations"].([]interface{}) + if !ok { + t.Fatalf("schema missing x-kubernetes-validations, want rule %q", want) + } + for _, validation := range validations { + validationMap, ok := validation.(map[string]interface{}) + if !ok { + t.Fatalf("validation entry = %T(%v), want map", validation, validation) + } + rule, ok := validationMap["rule"].(string) + if !ok { + t.Fatalf("validation rule = %T(%v), want string", validationMap["rule"], validationMap["rule"]) + } + if normalizeRule(rule) == normalizeRule(want) { + return + } + } + t.Fatalf("schema validations %v do not include rule %q", validations, want) +} + +func normalizeRule(rule string) string { + return strings.Join(strings.Fields(rule), " ") +}