diff --git a/api/v1alpha1/inferenceservice_types.go b/api/v1alpha1/inferenceservice_types.go index d73755e8..66931666 100644 --- a/api/v1alpha1/inferenceservice_types.go +++ b/api/v1alpha1/inferenceservice_types.go @@ -112,7 +112,8 @@ type InferenceServiceSpec struct { // "personaplex": NVIDIA PersonaPlex (Moshi) speech-to-speech server. // "vllm": vLLM OpenAI-compatible server with PagedAttention. // "tgi": HuggingFace Text Generation Inference server. - // +kubebuilder:validation:Enum=llamacpp;personaplex;vllm;tgi;generic + // "sglang": SGLang OpenAI-compatible server with RadixAttention prefix caching. + // +kubebuilder:validation:Enum=llamacpp;personaplex;vllm;tgi;sglang;generic // +kubebuilder:default=llamacpp // +optional Runtime string `json:"runtime,omitempty"` @@ -447,6 +448,11 @@ type InferenceServiceSpec struct { // +optional TGIConfig *TGIConfig `json:"tgiConfig,omitempty"` + // SGLangConfig holds configuration for the SGLang runtime. + // Only used when Runtime is "sglang". + // +optional + SGLangConfig *SGLangConfig `json:"sglangConfig,omitempty"` + // ImagePullSecrets for pulling container images from private registries. // +optional ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` @@ -1050,6 +1056,174 @@ type InferenceServiceList struct { Items []InferenceService `json:"items"` } +// SGLangConfig holds configuration for the SGLang inference server. +type SGLangConfig struct { + // Sharding + // TensorParallelSize sets the number of GPUs for tensor parallelism. + // Maps to SGLang --tp flag. + // +kubebuilder:validation:Minimum=1 + // +optional + TensorParallelSize *int32 `json:"tensorParallelSize,omitempty"` + + // ExpertParallelSize sets the number of GPUs for expert parallelism (MoE models). + // Maps to SGLang --ep flag. Not auto-derived; set explicitly. + // +kubebuilder:validation:Minimum=1 + // +optional + ExpertParallelSize *int32 `json:"expertParallelSize,omitempty"` + + // DataParallelSize sets the number of data-parallel replicas (SGLang-side controller). + // Maps to SGLang --dp flag. Not auto-derived; set explicitly. + // +kubebuilder:validation:Minimum=1 + // +optional + DataParallelSize *int32 `json:"dataParallelSize,omitempty"` + + // Memory & context + // ContextLength sets the maximum model context length. + // Maps to SGLang --context-length flag. + // +kubebuilder:validation:Minimum=128 + // +optional + ContextLength *int32 `json:"contextLength,omitempty"` + + // MemFractionStatic sets the fraction of GPU memory used for static state + // (model weights + KV cache). Range 0.1-0.99. Requires GPU. + // Maps to SGLang --mem-fraction-static flag. + // +kubebuilder:validation:Minimum=0.1 + // +kubebuilder:validation:Maximum=0.99 + // +optional + MemFractionStatic *float64 `json:"memFractionStatic,omitempty"` + + // Batching + // ChunkedPrefillSize sets the chunk size for chunked prefill (tokens). + // Maps to SGLang --chunked-prefill-size flag. + // +kubebuilder:validation:Minimum=512 + // +optional + ChunkedPrefillSize *int32 `json:"chunkedPrefillSize,omitempty"` + + // MaxRunningRequests caps concurrent in-flight requests. Maps to SGLang + // --max-running-requests flag. Spec.parallelSlots on the llama.cpp runtime + // is the analog; SGLang uses its own name. + // +kubebuilder:validation:Minimum=1 + // +optional + MaxRunningRequests *int32 `json:"maxRunningRequests,omitempty"` + + // Quantization & KV cache + // Quantization sets the quantization method. SGLang accepts fp8/awq/gptq/modelopt. + // Maps to SGLang --quantization flag. + // +optional + Quantization string `json:"quantization,omitempty"` + + // KVCacheDtype selects the KV cache element type. auto follows dtype. + // fp8_e5m2 / fp8_e4m3 cut KV memory roughly in half. Custom values not + // in the enum (e.g., TurboQuant) go in KVCacheCustomDtype. + // Maps to SGLang --kv-cache-dtype flag. + // +kubebuilder:validation:Enum=auto;fp8_e5m2;fp8_e4m3 + // +kubebuilder:default=auto + // +optional + KVCacheDtype *string `json:"kvCacheDtype,omitempty"` + + // KVCacheCustomDtype sets a custom SGLang KV cache type not in the standard + // enum. Maps to SGLang --kv-cache-dtype flag. Takes precedence over + // KVCacheDtype when both are set. LLMKube does not validate the string. + // +optional + KVCacheCustomDtype string `json:"kvCacheCustomDtype,omitempty"` + + // Attention + // AttentionBackend selects the attention implementation. flashinfer is + // fastest on recent NVIDIA GPUs; flash_attn is portable; torch_native is + // the fallback. Maps to SGLang --attention-backend flag. + // +kubebuilder:validation:Enum=flashinfer;flash_attn;torch_native + // +optional + AttentionBackend string `json:"attentionBackend,omitempty"` + + // EnablePrefixCaching turns on RadixAttention automatic prefix caching. + // Headline feature for agentic workloads with shared system-prompt + + // tool-definition + repo-context prefixes. Maps to SGLang --enable-prefix-caching. + // +optional + EnablePrefixCaching *bool `json:"enablePrefixCaching,omitempty"` + + // Agentic glue + // ToolCallParser selects the tool-call extraction format. For foreman + // tool-loop workloads. Maps to SGLang --tool-call-parser flag. + // +kubebuilder:validation:Enum=llama3;qwen3;qwen25;hermes;functionary;mistral + // +optional + ToolCallParser string `json:"toolCallParser,omitempty"` + + // ReasoningParser selects the reasoning-content extraction format. For + // thinking models (qwen3, deepseek-r1). Maps to SGLang --reasoning-parser. + // +kubebuilder:validation:Enum=qwen3;deepseek-r1 + // +optional + ReasoningParser string `json:"reasoningParser,omitempty"` + + // ChatTemplate overrides the model's bundled chat template. Maps to + // SGLang --chat-template flag. + // +optional + ChatTemplate string `json:"chatTemplate,omitempty"` + + // Speculative configures speculative decoding (EAGLE / EAGLE3 / Medusa). + // +optional + Speculative *SGLangSpeculativeConfig `json:"speculative,omitempty"` + + // LoRA (basic) + // LoraModules is a JSON array of LoRA specs (model_id -> path). Maps to + // SGLang --lora-modules flag. + // +optional + LoraModules []string `json:"loraModules,omitempty"` + + // MaxLoraRank sets the maximum LoRA rank accepted at load time. Maps to + // SGLang --max-lora-rank flag. + // +kubebuilder:validation:Minimum=1 + // +optional + MaxLoraRank *int32 `json:"maxLoraRank,omitempty"` + + // LoraTargetModules lists the modules LoRA adapters may target (e.g., + // "q_proj", "k_proj"). Maps to SGLang --lora-target-modules flag. + // +optional + LoraTargetModules []string `json:"loraTargetModules,omitempty"` + + // HFTokenSecretRef references a Secret containing the HuggingFace token. + // Injected as HF_TOKEN env var. + // +optional + HFTokenSecretRef *corev1.SecretKeySelector `json:"hfTokenSecretRef,omitempty"` +} + +// SGLangSpeculativeConfig configures speculative decoding for SGLang. +type SGLangSpeculativeConfig struct { + // Enabled toggles speculative decoding on. When false or nil, no flags + // are emitted regardless of other fields. + // +optional + Enabled *bool `json:"enabled,omitempty"` + + // Algorithm selects the speculative algorithm (EAGLE, EAGLE3, Medusa). + // Maps to SGLang --speculative-algorithm flag. + // +kubebuilder:validation:Enum=EAGLE;EAGLE3;Medusa + // +optional + Algorithm string `json:"algorithm,omitempty"` + + // DraftModelPath is the path to the draft model weights (for EAGLE). + // Maps to SGLang --speculative-draft-model-path flag. Required when Enabled. + // +optional + DraftModelPath string `json:"draftModelPath,omitempty"` + + // NumSteps is the number of draft steps per forward pass. + // Maps to SGLang --speculative-num-steps flag. + // +kubebuilder:validation:Minimum=1 + // +optional + NumSteps *int32 `json:"numSteps,omitempty"` + + // EagleTopK is the top-k sampling for EAGLE draft tokens. + // Maps to SGLang --speculative-eagle-topk flag. + // +kubebuilder:validation:Minimum=1 + // +optional + EagleTopK *int32 `json:"eagleTopK,omitempty"` + + // NumDraftTokens is the number of draft tokens proposed per step. + // Maps to SGLang --speculative-num-draft-tokens flag. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default=4 + // +optional + NumDraftTokens *int32 `json:"numDraftTokens,omitempty"` +} + func init() { SchemeBuilder.Register(&InferenceService{}, &InferenceServiceList{}) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index b3216776..d81d64b5 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -644,6 +644,11 @@ func (in *InferenceServiceSpec) DeepCopyInto(out *InferenceServiceSpec) { *out = new(TGIConfig) (*in).DeepCopyInto(*out) } + if in.SGLangConfig != nil { + in, out := &in.SGLangConfig, &out.SGLangConfig + *out = new(SGLangConfig) + (*in).DeepCopyInto(*out) + } if in.ImagePullSecrets != nil { in, out := &in.ImagePullSecrets, &out.ImagePullSecrets *out = make([]v1.LocalObjectReference, len(*in)) @@ -1429,6 +1434,126 @@ func (in *RuleRoute) DeepCopy() *RuleRoute { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SGLangConfig) DeepCopyInto(out *SGLangConfig) { + *out = *in + if in.TensorParallelSize != nil { + in, out := &in.TensorParallelSize, &out.TensorParallelSize + *out = new(int32) + **out = **in + } + if in.ExpertParallelSize != nil { + in, out := &in.ExpertParallelSize, &out.ExpertParallelSize + *out = new(int32) + **out = **in + } + if in.DataParallelSize != nil { + in, out := &in.DataParallelSize, &out.DataParallelSize + *out = new(int32) + **out = **in + } + if in.ContextLength != nil { + in, out := &in.ContextLength, &out.ContextLength + *out = new(int32) + **out = **in + } + if in.MemFractionStatic != nil { + in, out := &in.MemFractionStatic, &out.MemFractionStatic + *out = new(float64) + **out = **in + } + if in.ChunkedPrefillSize != nil { + in, out := &in.ChunkedPrefillSize, &out.ChunkedPrefillSize + *out = new(int32) + **out = **in + } + if in.MaxRunningRequests != nil { + in, out := &in.MaxRunningRequests, &out.MaxRunningRequests + *out = new(int32) + **out = **in + } + if in.KVCacheDtype != nil { + in, out := &in.KVCacheDtype, &out.KVCacheDtype + *out = new(string) + **out = **in + } + if in.EnablePrefixCaching != nil { + in, out := &in.EnablePrefixCaching, &out.EnablePrefixCaching + *out = new(bool) + **out = **in + } + if in.Speculative != nil { + in, out := &in.Speculative, &out.Speculative + *out = new(SGLangSpeculativeConfig) + (*in).DeepCopyInto(*out) + } + if in.LoraModules != nil { + in, out := &in.LoraModules, &out.LoraModules + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MaxLoraRank != nil { + in, out := &in.MaxLoraRank, &out.MaxLoraRank + *out = new(int32) + **out = **in + } + if in.LoraTargetModules != nil { + in, out := &in.LoraTargetModules, &out.LoraTargetModules + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.HFTokenSecretRef != nil { + in, out := &in.HFTokenSecretRef, &out.HFTokenSecretRef + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SGLangConfig. +func (in *SGLangConfig) DeepCopy() *SGLangConfig { + if in == nil { + return nil + } + out := new(SGLangConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SGLangSpeculativeConfig) DeepCopyInto(out *SGLangSpeculativeConfig) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.NumSteps != nil { + in, out := &in.NumSteps, &out.NumSteps + *out = new(int32) + **out = **in + } + if in.EagleTopK != nil { + in, out := &in.EagleTopK, &out.EagleTopK + *out = new(int32) + **out = **in + } + if in.NumDraftTokens != nil { + in, out := &in.NumDraftTokens, &out.NumDraftTokens + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SGLangSpeculativeConfig. +func (in *SGLangSpeculativeConfig) DeepCopy() *SGLangSpeculativeConfig { + if in == nil { + return nil + } + out := new(SGLangSpeculativeConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SpeculativeConfig) DeepCopyInto(out *SpeculativeConfig) { *out = *in diff --git a/charts/llmkube/templates/crds/inferenceservices.yaml b/charts/llmkube/templates/crds/inferenceservices.yaml index 178bc152..29d18dc4 100644 --- a/charts/llmkube/templates/crds/inferenceservices.yaml +++ b/charts/llmkube/templates/crds/inferenceservices.yaml @@ -2472,11 +2472,13 @@ spec: "personaplex": NVIDIA PersonaPlex (Moshi) speech-to-speech server. "vllm": vLLM OpenAI-compatible server with PagedAttention. "tgi": HuggingFace Text Generation Inference server. + "sglang": SGLang OpenAI-compatible server with RadixAttention prefix caching. enum: - llamacpp - personaplex - vllm - tgi + - sglang - generic type: string runtimeClassName: @@ -2682,6 +2684,227 @@ spec: type: string type: object type: object + sglangConfig: + description: |- + SGLangConfig holds configuration for the SGLang runtime. + Only used when Runtime is "sglang". + properties: + attentionBackend: + description: |- + Attention + AttentionBackend selects the attention implementation. flashinfer is + fastest on recent NVIDIA GPUs; flash_attn is portable; torch_native is + the fallback. Maps to SGLang --attention-backend flag. + enum: + - flashinfer + - flash_attn + - torch_native + type: string + chatTemplate: + description: |- + ChatTemplate overrides the model's bundled chat template. Maps to + SGLang --chat-template flag. + type: string + chunkedPrefillSize: + description: |- + Batching + ChunkedPrefillSize sets the chunk size for chunked prefill (tokens). + Maps to SGLang --chunked-prefill-size flag. + format: int32 + minimum: 512 + type: integer + contextLength: + description: |- + Memory & context + ContextLength sets the maximum model context length. + Maps to SGLang --context-length flag. + format: int32 + minimum: 128 + type: integer + dataParallelSize: + description: |- + DataParallelSize sets the number of data-parallel replicas (SGLang-side controller). + Maps to SGLang --dp flag. Not auto-derived; set explicitly. + format: int32 + minimum: 1 + type: integer + enablePrefixCaching: + description: |- + EnablePrefixCaching turns on RadixAttention automatic prefix caching. + Headline feature for agentic workloads with shared system-prompt + + tool-definition + repo-context prefixes. Maps to SGLang --enable-prefix-caching. + type: boolean + expertParallelSize: + description: |- + ExpertParallelSize sets the number of GPUs for expert parallelism (MoE models). + Maps to SGLang --ep flag. Not auto-derived; set explicitly. + format: int32 + minimum: 1 + type: integer + hfTokenSecretRef: + description: |- + HFTokenSecretRef references a Secret containing the HuggingFace token. + Injected as HF_TOKEN env var. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + kvCacheCustomDtype: + description: |- + KVCacheCustomDtype sets a custom SGLang KV cache type not in the standard + enum. Maps to SGLang --kv-cache-dtype flag. Takes precedence over + KVCacheDtype when both are set. LLMKube does not validate the string. + type: string + kvCacheDtype: + default: auto + description: |- + KVCacheDtype selects the KV cache element type. auto follows dtype. + fp8_e5m2 / fp8_e4m3 cut KV memory roughly in half. Custom values not + in the enum (e.g., TurboQuant) go in KVCacheCustomDtype. + Maps to SGLang --kv-cache-dtype flag. + enum: + - auto + - fp8_e5m2 + - fp8_e4m3 + type: string + loraModules: + description: |- + LoRA (basic) + LoraModules is a JSON array of LoRA specs (model_id -> path). Maps to + SGLang --lora-modules flag. + items: + type: string + type: array + loraTargetModules: + description: |- + LoraTargetModules lists the modules LoRA adapters may target (e.g., + "q_proj", "k_proj"). Maps to SGLang --lora-target-modules flag. + items: + type: string + type: array + maxLoraRank: + description: |- + MaxLoraRank sets the maximum LoRA rank accepted at load time. Maps to + SGLang --max-lora-rank flag. + format: int32 + minimum: 1 + type: integer + maxRunningRequests: + description: |- + MaxRunningRequests caps concurrent in-flight requests. Maps to SGLang + --max-running-requests flag. Spec.parallelSlots on the llama.cpp runtime + is the analog; SGLang uses its own name. + format: int32 + minimum: 1 + type: integer + memFractionStatic: + description: |- + MemFractionStatic sets the fraction of GPU memory used for static state + (model weights + KV cache). Range 0.1-0.99. Requires GPU. + Maps to SGLang --mem-fraction-static flag. + maximum: 0.99 + minimum: 0.1 + type: number + quantization: + description: |- + Quantization & KV cache + Quantization sets the quantization method. SGLang accepts fp8/awq/gptq/modelopt. + Maps to SGLang --quantization flag. + type: string + reasoningParser: + description: |- + ReasoningParser selects the reasoning-content extraction format. For + thinking models (qwen3, deepseek-r1). Maps to SGLang --reasoning-parser. + enum: + - qwen3 + - deepseek-r1 + type: string + speculative: + description: Speculative configures speculative decoding (EAGLE + / EAGLE3 / Medusa). + properties: + algorithm: + description: |- + Algorithm selects the speculative algorithm (EAGLE, EAGLE3, Medusa). + Maps to SGLang --speculative-algorithm flag. + enum: + - EAGLE + - EAGLE3 + - Medusa + type: string + draftModelPath: + description: |- + DraftModelPath is the path to the draft model weights (for EAGLE). + Maps to SGLang --speculative-draft-model-path flag. Required when Enabled. + type: string + eagleTopK: + description: |- + EagleTopK is the top-k sampling for EAGLE draft tokens. + Maps to SGLang --speculative-eagle-topk flag. + format: int32 + minimum: 1 + type: integer + enabled: + description: |- + Enabled toggles speculative decoding on. When false or nil, no flags + are emitted regardless of other fields. + type: boolean + numDraftTokens: + default: 4 + description: |- + NumDraftTokens is the number of draft tokens proposed per step. + Maps to SGLang --speculative-num-draft-tokens flag. + format: int32 + minimum: 1 + type: integer + numSteps: + description: |- + NumSteps is the number of draft steps per forward pass. + Maps to SGLang --speculative-num-steps flag. + format: int32 + minimum: 1 + type: integer + type: object + tensorParallelSize: + description: |- + Sharding + TensorParallelSize sets the number of GPUs for tensor parallelism. + Maps to SGLang --tp flag. + format: int32 + minimum: 1 + type: integer + toolCallParser: + description: |- + Agentic glue + ToolCallParser selects the tool-call extraction format. For foreman + tool-loop workloads. Maps to SGLang --tool-call-parser flag. + enum: + - llama3 + - qwen3 + - qwen25 + - hermes + - functionary + - mistral + type: string + type: object skipModelInit: description: |- SkipModelInit disables the model-downloader init container. diff --git a/config/crd/bases/inference.llmkube.dev_inferenceservices.yaml b/config/crd/bases/inference.llmkube.dev_inferenceservices.yaml index 03e01b34..675b59a5 100644 --- a/config/crd/bases/inference.llmkube.dev_inferenceservices.yaml +++ b/config/crd/bases/inference.llmkube.dev_inferenceservices.yaml @@ -2468,11 +2468,13 @@ spec: "personaplex": NVIDIA PersonaPlex (Moshi) speech-to-speech server. "vllm": vLLM OpenAI-compatible server with PagedAttention. "tgi": HuggingFace Text Generation Inference server. + "sglang": SGLang OpenAI-compatible server with RadixAttention prefix caching. enum: - llamacpp - personaplex - vllm - tgi + - sglang - generic type: string runtimeClassName: @@ -2678,6 +2680,227 @@ spec: type: string type: object type: object + sglangConfig: + description: |- + SGLangConfig holds configuration for the SGLang runtime. + Only used when Runtime is "sglang". + properties: + attentionBackend: + description: |- + Attention + AttentionBackend selects the attention implementation. flashinfer is + fastest on recent NVIDIA GPUs; flash_attn is portable; torch_native is + the fallback. Maps to SGLang --attention-backend flag. + enum: + - flashinfer + - flash_attn + - torch_native + type: string + chatTemplate: + description: |- + ChatTemplate overrides the model's bundled chat template. Maps to + SGLang --chat-template flag. + type: string + chunkedPrefillSize: + description: |- + Batching + ChunkedPrefillSize sets the chunk size for chunked prefill (tokens). + Maps to SGLang --chunked-prefill-size flag. + format: int32 + minimum: 512 + type: integer + contextLength: + description: |- + Memory & context + ContextLength sets the maximum model context length. + Maps to SGLang --context-length flag. + format: int32 + minimum: 128 + type: integer + dataParallelSize: + description: |- + DataParallelSize sets the number of data-parallel replicas (SGLang-side controller). + Maps to SGLang --dp flag. Not auto-derived; set explicitly. + format: int32 + minimum: 1 + type: integer + enablePrefixCaching: + description: |- + EnablePrefixCaching turns on RadixAttention automatic prefix caching. + Headline feature for agentic workloads with shared system-prompt + + tool-definition + repo-context prefixes. Maps to SGLang --enable-prefix-caching. + type: boolean + expertParallelSize: + description: |- + ExpertParallelSize sets the number of GPUs for expert parallelism (MoE models). + Maps to SGLang --ep flag. Not auto-derived; set explicitly. + format: int32 + minimum: 1 + type: integer + hfTokenSecretRef: + description: |- + HFTokenSecretRef references a Secret containing the HuggingFace token. + Injected as HF_TOKEN env var. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + kvCacheCustomDtype: + description: |- + KVCacheCustomDtype sets a custom SGLang KV cache type not in the standard + enum. Maps to SGLang --kv-cache-dtype flag. Takes precedence over + KVCacheDtype when both are set. LLMKube does not validate the string. + type: string + kvCacheDtype: + default: auto + description: |- + KVCacheDtype selects the KV cache element type. auto follows dtype. + fp8_e5m2 / fp8_e4m3 cut KV memory roughly in half. Custom values not + in the enum (e.g., TurboQuant) go in KVCacheCustomDtype. + Maps to SGLang --kv-cache-dtype flag. + enum: + - auto + - fp8_e5m2 + - fp8_e4m3 + type: string + loraModules: + description: |- + LoRA (basic) + LoraModules is a JSON array of LoRA specs (model_id -> path). Maps to + SGLang --lora-modules flag. + items: + type: string + type: array + loraTargetModules: + description: |- + LoraTargetModules lists the modules LoRA adapters may target (e.g., + "q_proj", "k_proj"). Maps to SGLang --lora-target-modules flag. + items: + type: string + type: array + maxLoraRank: + description: |- + MaxLoraRank sets the maximum LoRA rank accepted at load time. Maps to + SGLang --max-lora-rank flag. + format: int32 + minimum: 1 + type: integer + maxRunningRequests: + description: |- + MaxRunningRequests caps concurrent in-flight requests. Maps to SGLang + --max-running-requests flag. Spec.parallelSlots on the llama.cpp runtime + is the analog; SGLang uses its own name. + format: int32 + minimum: 1 + type: integer + memFractionStatic: + description: |- + MemFractionStatic sets the fraction of GPU memory used for static state + (model weights + KV cache). Range 0.1-0.99. Requires GPU. + Maps to SGLang --mem-fraction-static flag. + maximum: 0.99 + minimum: 0.1 + type: number + quantization: + description: |- + Quantization & KV cache + Quantization sets the quantization method. SGLang accepts fp8/awq/gptq/modelopt. + Maps to SGLang --quantization flag. + type: string + reasoningParser: + description: |- + ReasoningParser selects the reasoning-content extraction format. For + thinking models (qwen3, deepseek-r1). Maps to SGLang --reasoning-parser. + enum: + - qwen3 + - deepseek-r1 + type: string + speculative: + description: Speculative configures speculative decoding (EAGLE + / EAGLE3 / Medusa). + properties: + algorithm: + description: |- + Algorithm selects the speculative algorithm (EAGLE, EAGLE3, Medusa). + Maps to SGLang --speculative-algorithm flag. + enum: + - EAGLE + - EAGLE3 + - Medusa + type: string + draftModelPath: + description: |- + DraftModelPath is the path to the draft model weights (for EAGLE). + Maps to SGLang --speculative-draft-model-path flag. Required when Enabled. + type: string + eagleTopK: + description: |- + EagleTopK is the top-k sampling for EAGLE draft tokens. + Maps to SGLang --speculative-eagle-topk flag. + format: int32 + minimum: 1 + type: integer + enabled: + description: |- + Enabled toggles speculative decoding on. When false or nil, no flags + are emitted regardless of other fields. + type: boolean + numDraftTokens: + default: 4 + description: |- + NumDraftTokens is the number of draft tokens proposed per step. + Maps to SGLang --speculative-num-draft-tokens flag. + format: int32 + minimum: 1 + type: integer + numSteps: + description: |- + NumSteps is the number of draft steps per forward pass. + Maps to SGLang --speculative-num-steps flag. + format: int32 + minimum: 1 + type: integer + type: object + tensorParallelSize: + description: |- + Sharding + TensorParallelSize sets the number of GPUs for tensor parallelism. + Maps to SGLang --tp flag. + format: int32 + minimum: 1 + type: integer + toolCallParser: + description: |- + Agentic glue + ToolCallParser selects the tool-call extraction format. For foreman + tool-loop workloads. Maps to SGLang --tool-call-parser flag. + enum: + - llama3 + - qwen3 + - qwen25 + - hermes + - functionary + - mistral + type: string + type: object skipModelInit: description: |- SkipModelInit disables the model-downloader init container. diff --git a/docs/contributors/adding-a-runtime.md b/docs/contributors/adding-a-runtime.md index ff435777..c5a6f6c1 100644 --- a/docs/contributors/adding-a-runtime.md +++ b/docs/contributors/adding-a-runtime.md @@ -104,4 +104,5 @@ Add `--runtime yourengine` handling in `pkg/cli/deploy.go`. | `personaplex` | PersonaPlex/Moshi | 8998 | TCP socket | No | — | | `vllm` | vLLM | 8000 | HTTP /health | Yes | vllm:num_requests_running | | `tgi` | TGI | 80 | HTTP /health | No (HF download) | tgi:queue_size | +| `sglang` | SGLang | 30000 | HTTP /health_generate | Yes (curl) | sglang:num_requests_running | | `generic` | Any container | 8080 | TCP socket | No | — | diff --git a/docs/site/concepts/comparison.md b/docs/site/concepts/comparison.md index 4a7d4441..02523bb9 100644 --- a/docs/site/concepts/comparison.md +++ b/docs/site/concepts/comparison.md @@ -18,6 +18,8 @@ Honest comparison, not a sales pitch. We use vLLM, llama.cpp, and TGI as runtime **Inference engines** ([vLLM](https://github.com/vllm-project/vllm), [llama.cpp](https://github.com/ggerganov/llama.cpp), [TGI](https://github.com/huggingface/text-generation-inference), [SGLang](https://github.com/sgl-project/sglang), [MLX](https://github.com/ml-explore/mlx)) run a single model behind an HTTP server. They're the substrate; LLMKube wraps them. +- **LLMKube runtimes:** `llamacpp`, `vllm`, `tgi`, `sglang`, `personaplex`, `generic`. SGLang landed as a first-class runtime in v0.9.x ([#974](https://github.com/defilantech/LLMKube/issues/974)). + **Single-node runners** ([Ollama](https://ollama.com/), LM Studio) are best-in-class for "run a model on your laptop." Not a fit when you need Kubernetes-native scheduling, NetworkPolicy, HPA, multi-tenant namespaces, or a fleet of GPU nodes. **Kubernetes operators and platforms** ([KubeAI](https://www.kubeai.org/), [llm-d](https://llm-d.ai/), [NVIDIA NIM Operator](https://docs.nvidia.com/nim-operator/latest/)) are the actual peers. The table below focuses here. diff --git a/docs/site/guides/model-matrix.md b/docs/site/guides/model-matrix.md index f91f7940..45873004 100644 --- a/docs/site/guides/model-matrix.md +++ b/docs/site/guides/model-matrix.md @@ -260,3 +260,24 @@ The local LLM ecosystem is one of the fastest-moving spaces in software. Between - License terms occasionally change (especially Llama and Gemma). Use this as a structured starting point. **Always click through to the upstream HuggingFace model card before downloading a large file.** If you find this doc is meaningfully wrong, a PR with the corrected row is appreciated, but please do not assume any single number here is current the day you read it. + +## Runtime backends + +### sglang + +GPU-only. Choose when: + +- Your workload re-sends a large shared prefix every turn (foreman tool loops, + multi-turn chat with a stable system prompt + tool definitions). SGLang's + RadixAttention (`--enable-prefix-caching`) serves the shared prefix from a + tree-structured cache rather than reprocessing it. +- You want speculative decoding (EAGLE / EAGLE3) or tool-call / reasoning + parsers that vLLM doesn't ship with first-class CRD fields. + +Unlike llama.cpp, there's no meaningful CPU path — even small embedding models +require a GPU. AMD is supported via the ROCm image (`lmsysorg/sglang:...-rocm720-mi30x`), +picked automatically by the controller when `model.spec.hardware.gpu.vendor` +is `amd`. + +Defaults to the OpenAI-compatible `/v1` surface, so downstream tooling +(litellm, the foreman agents) needs no change. diff --git a/internal/controller/deployment_builder.go b/internal/controller/deployment_builder.go index be820fc4..beecd196 100644 --- a/internal/controller/deployment_builder.go +++ b/internal/controller/deployment_builder.go @@ -44,18 +44,32 @@ import ( // it for vLLM where the v0.20+ env-var validator turns it into log noise. // resolveRuntimeImage returns the container image for the runtime, making the // otherwise vendor-blind backend.DefaultImage() vendor- and runtime-aware where -// it matters. Today the only divergence is the llama.cpp backend with an -// AMD/Vulkan Model: it uses LLMKube's pinned Vulkan image instead of the stock -// upstream image. Every other backend/vendor/runtime combination falls through -// to backend.DefaultImage(). An explicit InferenceService.spec.image still wins +// it matters. Today there are two divergences: +// - LlamaCppBackend with AMD + Vulkan Model → LLMKube's pinned Vulkan image +// - SGLangBackend with AMD (ROCm) Model → SGLang ROCm image +// +// Every other backend/vendor/runtime combination falls through to +// backend.DefaultImage(). An explicit InferenceService.spec.image still wins // over whatever this returns (handled by the caller). func resolveRuntimeImage(backend RuntimeBackend, model *inferencev1alpha1.Model) string { if _, ok := backend.(*LlamaCppBackend); ok && isVulkanAMDModel(model) { return llamaCppVulkanImage } + if _, ok := backend.(*SGLangBackend); ok && isAMDROCmModel(model) { + return sglangROCmImage + } return backend.DefaultImage() } +// isAMDROCmModel reports whether the Model requests the AMD vendor. ROCm vs +// Vulkan is not distinguished here — SGLang ships ROCm images, not Vulkan. +func isAMDROCmModel(model *inferencev1alpha1.Model) bool { + if model == nil || model.Spec.Hardware == nil || model.Spec.Hardware.GPU == nil { + return false + } + return strings.EqualFold(strings.TrimSpace(model.Spec.Hardware.GPU.Vendor), "amd") +} + // isVulkanAMDModel reports whether the Model requests the AMD vendor with the // Vulkan GPU runtime. func isVulkanAMDModel(model *inferencev1alpha1.Model) bool { diff --git a/internal/controller/deployment_builder_test.go b/internal/controller/deployment_builder_test.go index 4d0c735e..b644a699 100644 --- a/internal/controller/deployment_builder_test.go +++ b/internal/controller/deployment_builder_test.go @@ -208,6 +208,48 @@ func TestResolveRuntimeImage(t *testing.T) { model: &inferencev1alpha1.Model{}, expected: stockLlamaCpp, }, + { + name: "sglang no GPU vendor falls back to CUDA image", + backend: &SGLangBackend{}, + model: &inferencev1alpha1.Model{}, + expected: sglangCUDAImage, + }, + { + name: "sglang NVIDIA vendor picks CUDA image", + backend: &SGLangBackend{}, + model: &inferencev1alpha1.Model{ + Spec: inferencev1alpha1.ModelSpec{ + Hardware: &inferencev1alpha1.HardwareSpec{ + GPU: &inferencev1alpha1.GPUSpec{Vendor: "nvidia"}, + }, + }, + }, + expected: sglangCUDAImage, + }, + { + name: "sglang AMD vendor picks ROCm image", + backend: &SGLangBackend{}, + model: &inferencev1alpha1.Model{ + Spec: inferencev1alpha1.ModelSpec{ + Hardware: &inferencev1alpha1.HardwareSpec{ + GPU: &inferencev1alpha1.GPUSpec{Vendor: "amd"}, + }, + }, + }, + expected: sglangROCmImage, + }, + { + name: "sglang AMD vendor uppercase maps to ROCm image", + backend: &SGLangBackend{}, + model: &inferencev1alpha1.Model{ + Spec: inferencev1alpha1.ModelSpec{ + Hardware: &inferencev1alpha1.HardwareSpec{ + GPU: &inferencev1alpha1.GPUSpec{Vendor: "AMD"}, + }, + }, + }, + expected: sglangROCmImage, + }, } for _, tc := range cases { diff --git a/internal/controller/inferenceservice_controller.go b/internal/controller/inferenceservice_controller.go index 695d7288..f0acf339 100644 --- a/internal/controller/inferenceservice_controller.go +++ b/internal/controller/inferenceservice_controller.go @@ -297,11 +297,12 @@ func (r *InferenceServiceReconciler) reconcileDeployment(ctx context.Context, is return nil, snap.ReadyReplicas, snap, nil, nil } - // Surface non-fatal vLLM spec problems as a status condition before we + // Surface non-fatal runtime spec problems as a status condition before we // build the Deployment. A failure here never blocks reconciliation — the // Deployment is still produced with the offending flags silently skipped - // (see VLLMBackend.BuildArgs). + // (see VLLMBackend.BuildArgs, SGLangBackend.BuildArgs). r.reconcileVLLMSpecCondition(isvc) + r.reconcileSGLangSpecCondition(isvc) deployment := r.constructDeployment(isvc, model, desiredReplicas) if err := setControllerReferenceUnblocked(isvc, deployment, r.Scheme); err != nil { diff --git a/internal/controller/runtime.go b/internal/controller/runtime.go index 0df11ba2..eb7f94a1 100644 --- a/internal/controller/runtime.go +++ b/internal/controller/runtime.go @@ -68,6 +68,8 @@ func resolveBackend(isvc *inferencev1alpha1.InferenceService) RuntimeBackend { return &VLLMBackend{} case "tgi": return &TGIBackend{} + case RuntimeSGLANG: + return &SGLangBackend{} case "generic": return &GenericBackend{} default: diff --git a/internal/controller/runtime_sglang.go b/internal/controller/runtime_sglang.go new file mode 100644 index 00000000..0928a5db --- /dev/null +++ b/internal/controller/runtime_sglang.go @@ -0,0 +1,223 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package controller — sglang runtime backend. Mirrors the vLLM backend +// (runtime_vllm.go + runtime_vllm_args.go) but emits SGLang's CLI flags. +// SGLang is GPU-only; CPU-only deployments will fail to start the container. +// The `sglangLog` package logger carries construction-time warnings from +// BuildArgs that aren't fatal to reconciliation. + +package controller + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + inferencev1alpha1 "github.com/defilantech/llmkube/api/v1alpha1" +) + +var sglangLog = logf.Log.WithName("runtime.sglang") + +// RuntimeSGLANG is the InferenceService.Spec.Runtime value that selects the +// SGLang backend. +const RuntimeSGLANG = "sglang" + +// ConditionSGLangSpecValid is the metav1.Condition type set by the reconciler +// when the SGLangConfig portion of the spec is structurally invalid but not +// fatal to reconciliation (e.g., speculative enabled without an algorithm). +const ConditionSGLangSpecValid = "SGLangSpecValid" + +// sglangCUDAImage is the pinned default CUDA image. Tag verified at PR time +// against Docker Hub; bump in a follow-up if a newer SGLang release is required. +const sglangCUDAImage = "lmsysorg/sglang:v0.5.15-cu129" + +// sglangROCmImage is the AMD ROCm variant. Selected by resolveRuntimeImage +// when model.Spec.Hardware.GPU.Vendor == "amd". +const sglangROCmImage = "lmsysorg/sglang:v0.5.15-rocm720-mi30x" + +// SGLangBackend generates container configuration for the SGLang inference +// server (https://github.com/sgl-project/sglang). +type SGLangBackend struct{} + +func (b *SGLangBackend) ContainerName() string { return "sglang" } +func (b *SGLangBackend) DefaultImage() string { return sglangCUDAImage } +func (b *SGLangBackend) DefaultPort() int32 { return 30000 } +func (b *SGLangBackend) NeedsModelInit() bool { return true } +func (b *SGLangBackend) DefaultHPAMetric() string { return "sglang:num_requests_running" } + +// BuildArgs generates the SGLang server CLI arguments. Order: +// 1. --model-path, --host, --port (always) +// 2. --served-model-name (auto-derived from ModelRef / model name) +// 3. typed SGLangConfig flags in declaration order +// 4. --tp auto-derived from gpuCount when unset +// 5. --is-embedding when spec.mode == "embedding" (skip if user set it) +// 6. spec.extraArgs last (user wins on collision) +// +// GPU-only flags (memFractionStatic) log a warning when set on a CPU model. +func (b *SGLangBackend) BuildArgs(isvc *inferencev1alpha1.InferenceService, model *inferencev1alpha1.Model, modelPath string, port int32) []string { + source := modelPath + if source == "" { + source = normalizeHFSource(model.Spec.Source) + } + args := []string{ + "--model-path", source, + // Bind the dual-stack wildcard so pods are reachable on IPv6-only + // clusters (#972). SGLang keeps the last occurrence of a repeated + // flag, so users can override via extraArgs ("--host", "0.0.0.0"). + "--host", "::", + "--port", fmt.Sprintf("%d", port), + } + + // --served-model-name: skip if user already set it in extraArgs. + servedName := isvc.Spec.ModelRef + if servedName == "" && model != nil { + servedName = model.Name + } + if servedName != "" && !hasMatchingExtraArg(isvc.Spec.ExtraArgs, "served-model-name") { + args = append(args, "--served-model-name", servedName) + } + + cfg := isvc.Spec.SGLangConfig + gpuCount := resolveGPUCount(isvc, model) + if cfg != nil { + args = sglangAppendTensorParallelSize(args, cfg.TensorParallelSize) + args = sglangAppendExpertParallelSize(args, cfg.ExpertParallelSize) + args = sglangAppendDataParallelSize(args, cfg.DataParallelSize) + args = sglangAppendContextLength(args, cfg.ContextLength) + if cfg.MemFractionStatic != nil && gpuCount == 0 { + sglangLog.Info( + "spec.sglangConfig.memFractionStatic is defined with no GPU hardware; skipping --mem-fraction-static", + "inferenceService", isvc.Name, + "namespace", isvc.Namespace, + ) + } else { + args = sglangAppendMemFractionStatic(args, cfg.MemFractionStatic) + } + args = sglangAppendChunkedPrefillSize(args, cfg.ChunkedPrefillSize) + args = sglangAppendMaxRunningRequests(args, cfg.MaxRunningRequests) + args = sglangAppendQuantization(args, cfg.Quantization) + args = sglangAppendKVCacheDtype(args, cfg.KVCacheDtype, cfg.KVCacheCustomDtype) + args = sglangAppendAttentionBackend(args, cfg.AttentionBackend) + args = sglangAppendEnablePrefixCaching(args, cfg.EnablePrefixCaching) + args = sglangAppendToolCallParser(args, cfg.ToolCallParser) + args = sglangAppendReasoningParser(args, cfg.ReasoningParser) + args = sglangAppendChatTemplate(args, cfg.ChatTemplate) + args = sglangAppendSpeculative(args, cfg.Speculative) + args = sglangAppendLoraModules(args, cfg.LoraModules) + args = sglangAppendMaxLoraRank(args, cfg.MaxLoraRank) + args = sglangAppendLoraTargetModules(args, cfg.LoraTargetModules) + } + + // Auto-derive --tp when user didn't set it. + if gpuCount > 1 && (cfg == nil || cfg.TensorParallelSize == nil) { + args = append(args, "--tp", fmt.Sprintf("%d", gpuCount)) + } + + // Mode handling: --is-embedding (skip if user already set it). + if isvc.Spec.Mode == "embedding" && !hasMatchingExtraArg(isvc.Spec.ExtraArgs, "is-embedding") { + args = append(args, "--is-embedding") + } + + // ExtraArgs last (user wins). + if len(isvc.Spec.ExtraArgs) > 0 { + args = append(args, isvc.Spec.ExtraArgs...) + } + + return args +} + +// ValidateSGLangConfig checks the SGLangConfig for structurally invalid +// combinations that are non-fatal to reconciliation but should be surfaced +// as a status condition. Returns (reason, message) when invalid; empty +// strings when fine. +func ValidateSGLangConfig(isvc *inferencev1alpha1.InferenceService) (reason, message string) { + if isvc == nil || isvc.Spec.SGLangConfig == nil { + return "", "" + } + cfg := isvc.Spec.SGLangConfig + if cfg.Speculative != nil && cfg.Speculative.Enabled != nil && *cfg.Speculative.Enabled { + if cfg.Speculative.Algorithm == "" || cfg.Speculative.DraftModelPath == "" { + return "SpeculativeMissingConfig", + "spec.sglangConfig.speculative.enabled is true but algorithm/draftModelPath is empty; speculative decoding flags will be skipped" + } + } + return "", "" +} + +// BuildCommand returns the entrypoint for the SGLang container. SGLang +// launches via a Python module rather than a bare binary, mirroring +// PersonaPlexBackend. +func (b *SGLangBackend) BuildCommand() []string { + return []string{"python3", "-m", "sglang.launch_server"} +} + +// BuildProbes returns startup, liveness, and readiness probes. SGLang +// exposes /health (cheap liveness) and /health_generate (runs a token, +// accurate readiness but slow on cold start). Startup tolerates 180 +// failures (~30 minutes at 10s period) to cover model load + warmup. +func (b *SGLangBackend) BuildProbes(port int32) (*corev1.Probe, *corev1.Probe, *corev1.Probe) { + startup := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health_generate", + Port: intstr.FromInt32(port), + }, + }, + PeriodSeconds: 10, + TimeoutSeconds: 5, + FailureThreshold: 180, + } + liveness := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromInt32(port), + }, + }, + PeriodSeconds: 15, + TimeoutSeconds: 5, + FailureThreshold: 3, + } + readiness := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health_generate", + Port: intstr.FromInt32(port), + }, + }, + PeriodSeconds: 10, + TimeoutSeconds: 5, + FailureThreshold: 3, + } + return startup, liveness, readiness +} + +// BuildEnv returns HF_TOKEN from SGLangConfig.HFTokenSecretRef when set. +// SGLang reads HF_TOKEN from the environment to authenticate gated-model +// downloads from HuggingFace Hub. +func (b *SGLangBackend) BuildEnv(isvc *inferencev1alpha1.InferenceService) []corev1.EnvVar { + cfg := isvc.Spec.SGLangConfig + if cfg != nil && cfg.HFTokenSecretRef != nil { + return []corev1.EnvVar{{ + Name: "HF_TOKEN", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: cfg.HFTokenSecretRef}, + }} + } + return nil +} diff --git a/internal/controller/runtime_sglang_args.go b/internal/controller/runtime_sglang_args.go new file mode 100644 index 00000000..9f244994 --- /dev/null +++ b/internal/controller/runtime_sglang_args.go @@ -0,0 +1,195 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + "strconv" + "strings" + + inferencev1alpha1 "github.com/defilantech/llmkube/api/v1alpha1" +) + +// Argument builders for the sglang runtime. Each helper takes the current +// args slice plus the relevant CRD field and returns the appended slice (or +// the unchanged slice when the field is unset or not applicable). Prefixed +// with sglangAppend to avoid name collisions with vLLM helpers in the same +// package. Mirrors runtime_vllm_args.go. + +func sglangAppendTensorParallelSize(args []string, size *int32) []string { + if size != nil && *size >= 1 { + return append(args, "--tp", fmt.Sprintf("%d", *size)) + } + return args +} + +func sglangAppendExpertParallelSize(args []string, size *int32) []string { + if size != nil && *size >= 1 { + return append(args, "--ep", fmt.Sprintf("%d", *size)) + } + return args +} + +func sglangAppendDataParallelSize(args []string, size *int32) []string { + if size != nil && *size >= 1 { + return append(args, "--dp", fmt.Sprintf("%d", *size)) + } + return args +} + +func sglangAppendContextLength(args []string, ctx *int32) []string { + if ctx != nil { + return append(args, "--context-length", fmt.Sprintf("%d", *ctx)) + } + return args +} + +// sglangAppendMemFractionStatic: GPU-only. Caller logs a warning when set on a +// CPU model; this helper just emits the flag. +func sglangAppendMemFractionStatic(args []string, frac *float64) []string { + if frac != nil { + return append(args, "--mem-fraction-static", strconv.FormatFloat(*frac, 'f', -1, 64)) + } + return args +} + +func sglangAppendChunkedPrefillSize(args []string, size *int32) []string { + if size != nil { + return append(args, "--chunked-prefill-size", fmt.Sprintf("%d", *size)) + } + return args +} + +func sglangAppendMaxRunningRequests(args []string, max *int32) []string { + if max != nil { + return append(args, "--max-running-requests", fmt.Sprintf("%d", *max)) + } + return args +} + +func sglangAppendQuantization(args []string, q string) []string { + if q != "" { + return append(args, "--quantization", q) + } + return args +} + +// sglangResolveKVCacheDtype: custom wins; "" / nil / "auto" → empty (don't emit). +func sglangResolveKVCacheDtype(custom string, standard *string) string { + if custom != "" { + return custom + } + if standard == nil { + return "" + } + return *standard +} + +func sglangAppendKVCacheDtype(args []string, kvCacheDtype *string, custom string) []string { + if resolved := sglangResolveKVCacheDtype(custom, kvCacheDtype); resolved != "" && resolved != "auto" { + return append(args, "--kv-cache-dtype", resolved) + } + return args +} + +func sglangAppendAttentionBackend(args []string, backend string) []string { + if backend != "" { + return append(args, "--attention-backend", backend) + } + return args +} + +// sglangAppendEnablePrefixCaching: only emit when user explicitly opted in (true). +// SGLang's own default handles nil/false. +func sglangAppendEnablePrefixCaching(args []string, enabled *bool) []string { + if enabled != nil && *enabled { + return append(args, "--enable-prefix-caching") + } + return args +} + +func sglangAppendToolCallParser(args []string, parser string) []string { + if parser != "" { + return append(args, "--tool-call-parser", parser) + } + return args +} + +func sglangAppendReasoningParser(args []string, parser string) []string { + if parser != "" { + return append(args, "--reasoning-parser", parser) + } + return args +} + +func sglangAppendChatTemplate(args []string, tmpl string) []string { + if tmpl != "" { + return append(args, "--chat-template", tmpl) + } + return args +} + +// sglangAppendSpeculative: enabled+algorithm+draft-model required. Silent-skip with +// a log line when misconfigured; ValidateSGLangConfig surfaces a status condition. +func sglangAppendSpeculative(args []string, cfg *inferencev1alpha1.SGLangSpeculativeConfig) []string { + if cfg == nil || cfg.Enabled == nil || !*cfg.Enabled { + return args + } + if cfg.Algorithm == "" || cfg.DraftModelPath == "" { + sglangLog.Info( + "speculative decoding enabled but algorithm/draft-model-path empty; skipping speculative flags", + ) + return args + } + args = append(args, "--speculative-algorithm", cfg.Algorithm) + args = append(args, "--speculative-draft-model-path", cfg.DraftModelPath) + if cfg.NumSteps != nil { + args = append(args, "--speculative-num-steps", fmt.Sprintf("%d", *cfg.NumSteps)) + } + if cfg.EagleTopK != nil { + args = append(args, "--speculative-eagle-topk", fmt.Sprintf("%d", *cfg.EagleTopK)) + } + if cfg.NumDraftTokens != nil { + args = append(args, "--speculative-num-draft-tokens", fmt.Sprintf("%d", *cfg.NumDraftTokens)) + } + return args +} + +func sglangAppendLoraModules(args []string, modules []string) []string { + if len(modules) == 0 { + return args + } + // SGLang's --lora-modules accepts a comma-separated list of + // = or JSON entries. Join the CRD slice into a single string. + return append(args, "--lora-modules", strings.Join(modules, ",")) +} + +func sglangAppendMaxLoraRank(args []string, rank *int32) []string { + if rank != nil { + return append(args, "--max-lora-rank", fmt.Sprintf("%d", *rank)) + } + return args +} + +// SGLang's --lora-target-modules accepts a comma-separated list of module +// names. Join the CRD slice into a single string with commas. +func sglangAppendLoraTargetModules(args []string, modules []string) []string { + if len(modules) == 0 { + return args + } + return append(args, "--lora-target-modules", strings.Join(modules, ",")) +} diff --git a/internal/controller/runtime_sglang_test.go b/internal/controller/runtime_sglang_test.go new file mode 100644 index 00000000..2988bce3 --- /dev/null +++ b/internal/controller/runtime_sglang_test.go @@ -0,0 +1,805 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + inferencev1alpha1 "github.com/defilantech/llmkube/api/v1alpha1" +) + +// sglangFlagsNeverInBase is the list of SGLang-specific flags that must NOT +// appear in BuildArgs output when SGLangConfig is nil or empty. Shared across +// TestSGLangBuildArgs_NilConfig and TestSGLangBuildArgs_EmptyConfig. +var sglangFlagsNeverInBase = []string{ + "--tp", "--ep", "--dp", "--context-length", "--mem-fraction-static", + "--chunked-prefill-size", "--max-running-requests", "--quantization", + "--kv-cache-dtype", "--attention-backend", "--enable-prefix-caching", + "--tool-call-parser", "--reasoning-parser", "--chat-template", + "--speculative-algorithm", "--lora-modules", "--max-lora-rank", + "--lora-target-modules", "--is-embedding", +} + +// TestSGLangBackendDefaults locks in the trivial-method contracts that every +// runtime backend exposes (image, port, container name, model-init flag, +// HPA metric). Mirrors the structure of VLLMBackend tests. +func TestSGLangBackendDefaults(t *testing.T) { + b := &SGLangBackend{} + if got := b.ContainerName(); got != "sglang" { + t.Errorf("ContainerName() = %q, want %q", got, "sglang") + } + if got := b.DefaultImage(); got != sglangCUDAImage { + t.Errorf("DefaultImage() = %q, want %q", got, sglangCUDAImage) + } + if got := b.DefaultPort(); got != 30000 { + t.Errorf("DefaultPort() = %d, want 30000", got) + } + if !b.NeedsModelInit() { + t.Error("NeedsModelInit() = false, want true") + } + if got := b.DefaultHPAMetric(); got != "sglang:num_requests_running" { + t.Errorf("DefaultHPAMetric() = %q, want %q", got, "sglang:num_requests_running") + } +} + +// TestSGLangBuildArgs_NilConfig asserts the base arg emission when no +// SGLangConfig is provided. +func TestSGLangBuildArgs_NilConfig(t *testing.T) { + backend := &SGLangBackend{} + model := &inferencev1alpha1.Model{ + ObjectMeta: metav1.ObjectMeta{Name: "test-model", Namespace: "default"}, + } + isvc := &inferencev1alpha1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "isvc", Namespace: "default"}, + Spec: inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + ModelRef: "test-model", + }, + } + args := backend.BuildArgs(isvc, model, "/models/test", 30000) + + mustContain := []FlagCheck{ + {"--model-path", "/models/test"}, + {"--host", "::"}, + {"--port", "30000"}, + } + for _, fc := range mustContain { + if !containsArg(args, fc.flag, fc.value) { + t.Errorf("expected %q %q in args, got: %v", fc.flag, fc.value, args) + } + } + for _, f := range sglangFlagsNeverInBase { + if containsArg(args, f, "") { + t.Errorf("expected %q NOT in args, got: %v", f, args) + } + } +} + +// TestSGLangBuildArgs_EmptyConfig asserts the same base flags when an empty +// (non-nil) SGLangConfig is provided. +func TestSGLangBuildArgs_EmptyConfig(t *testing.T) { + backend := &SGLangBackend{} + model := &inferencev1alpha1.Model{ + ObjectMeta: metav1.ObjectMeta{Name: "test-model", Namespace: "default"}, + } + isvc := &inferencev1alpha1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "isvc", Namespace: "default"}, + Spec: inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + ModelRef: "test-model", + SGLangConfig: &inferencev1alpha1.SGLangConfig{}, + }, + } + args := backend.BuildArgs(isvc, model, "/models/test", 30000) + + for _, fc := range []FlagCheck{{"--model-path", "/models/test"}, {"--host", "::"}, {"--port", "30000"}} { + if !containsArg(args, fc.flag, fc.value) { + t.Errorf("expected %q %q in args, got: %v", fc.flag, fc.value, args) + } + } + for _, f := range sglangFlagsNeverInBase { + if containsArg(args, f, "") { + t.Errorf("expected %q NOT in args, got: %v", f, args) + } + } +} + +// sglangGPUModel returns a Model with one NVIDIA GPU enabled (helper for the +// memFractionStatic GPU-only check). +func sglangGPUModel() *inferencev1alpha1.Model { + return &inferencev1alpha1.Model{ + Spec: inferencev1alpha1.ModelSpec{ + Hardware: &inferencev1alpha1.HardwareSpec{ + GPU: &inferencev1alpha1.GPUSpec{Enabled: true, Count: 1}, + }, + }, + ObjectMeta: metav1.ObjectMeta{Name: "gpu-model", Namespace: "default"}, + } +} + +// sglangMultiGPUModel returns a Model with two NVIDIA GPUs for tp-auto tests. +func sglangMultiGPUModel() *inferencev1alpha1.Model { + return &inferencev1alpha1.Model{ + Spec: inferencev1alpha1.ModelSpec{ + Hardware: &inferencev1alpha1.HardwareSpec{ + GPU: &inferencev1alpha1.GPUSpec{Enabled: true, Count: 2}, + }, + }, + ObjectMeta: metav1.ObjectMeta{Name: "multi-gpu-model", Namespace: "default"}, + } +} + +func TestSGLangBuildArgs(t *testing.T) { + backend := &SGLangBackend{} + const modelPath = "/models/test" + const port = int32(30000) + + cases := []struct { + contains []FlagCheck + notContains []string + model *inferencev1alpha1.Model + name string + spec *inferencev1alpha1.InferenceServiceSpec + }{ + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "test-model", Namespace: "default"}}, + name: "served-model-name defaults to ModelRef", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + ModelRef: "test-model", + }, + contains: []FlagCheck{{"--served-model-name", "test-model"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "fallback-name"}}, + name: "served-model-name falls back to model.Name when ModelRef empty", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + }, + contains: []FlagCheck{{"--served-model-name", "fallback-name"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "served-model-name not emitted when extraArgs already has it", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + ModelRef: "should-not-appear", + ExtraArgs: []string{"--served-model-name", "custom"}, + }, + contains: []FlagCheck{{"--served-model-name", "custom"}}, + notContains: []string{"should-not-appear"}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "mode=embedding emits --is-embedding", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + Mode: "embedding", + }, + contains: []FlagCheck{{"--is-embedding", ""}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "mode=embedding not double-emitted when extraArgs already has it", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + Mode: "embedding", + ExtraArgs: []string{"--is-embedding=false"}, + }, + contains: []FlagCheck{{"--is-embedding=false", ""}}, + }, + { + model: sglangMultiGPUModel(), + name: "gpuCount>1 auto-derives --tp", + spec: &inferencev1alpha1.InferenceServiceSpec{Runtime: "sglang"}, + contains: []FlagCheck{{"--tp", "2"}}, + }, + { + model: sglangMultiGPUModel(), + name: "explicit tensorParallelSize overrides gpuCount auto-derive", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{TensorParallelSize: ptrInt32(4)}, + }, + contains: []FlagCheck{{"--tp", "4"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "gpuCount==1 does not auto-emit --tp", + spec: &inferencev1alpha1.InferenceServiceSpec{Runtime: "sglang"}, + notContains: []string{"--tp"}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "expertParallelSize emits --ep", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ExpertParallelSize: ptrInt32(2)}, + }, + contains: []FlagCheck{{"--ep", "2"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "dataParallelSize emits --dp", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{DataParallelSize: ptrInt32(4)}, + }, + contains: []FlagCheck{{"--dp", "4"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "contextLength emits --context-length", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ContextLength: ptrInt32(131072)}, + }, + contains: []FlagCheck{{"--context-length", "131072"}}, + }, + { + model: sglangGPUModel(), + name: "memFractionStatic emits --mem-fraction-static on GPU model", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{MemFractionStatic: ptrFloat64(0.85)}, + }, + contains: []FlagCheck{{"--mem-fraction-static", "0.85"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "memFractionStatic on CPU model logs warning, no flag", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{MemFractionStatic: ptrFloat64(0.85)}, + }, + notContains: []string{"--mem-fraction-static"}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "chunkedPrefillSize emits --chunked-prefill-size", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ChunkedPrefillSize: ptrInt32(8192)}, + }, + contains: []FlagCheck{{"--chunked-prefill-size", "8192"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "maxRunningRequests emits --max-running-requests", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{MaxRunningRequests: ptrInt32(64)}, + }, + contains: []FlagCheck{{"--max-running-requests", "64"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "quantization emits --quantization", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{Quantization: "fp8"}, + }, + contains: []FlagCheck{{"--quantization", "fp8"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "kvCacheDtype=auto does not emit flag", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{KVCacheDtype: ptrString("auto")}, + }, + notContains: []string{"--kv-cache-dtype"}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "kvCacheDtype=fp8_e5m2 emits --kv-cache-dtype", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{KVCacheDtype: ptrString("fp8_e5m2")}, + }, + contains: []FlagCheck{{"--kv-cache-dtype", "fp8_e5m2"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "kvCacheCustomDtype wins over kvCacheDtype", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + KVCacheDtype: ptrString("fp8_e4m3"), + KVCacheCustomDtype: "turbo2", + }, + }, + contains: []FlagCheck{{"--kv-cache-dtype", "turbo2"}}, + notContains: []string{"fp8_e4m3"}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "attentionBackend=flashinfer emits --attention-backend", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{AttentionBackend: "flashinfer"}, + }, + contains: []FlagCheck{{"--attention-backend", "flashinfer"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "enablePrefixCaching=true emits flag", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{EnablePrefixCaching: ptrBool(true)}, + }, + contains: []FlagCheck{{"--enable-prefix-caching", ""}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "enablePrefixCaching=false does not emit flag", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{EnablePrefixCaching: ptrBool(false)}, + }, + notContains: []string{"--enable-prefix-caching"}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "toolCallParser=llama3 emits --tool-call-parser", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ToolCallParser: "llama3"}, + }, + contains: []FlagCheck{{"--tool-call-parser", "llama3"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "reasoningParser=qwen3 emits --reasoning-parser", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ReasoningParser: "qwen3"}, + }, + contains: []FlagCheck{{"--reasoning-parser", "qwen3"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "chatTemplate emits --chat-template", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ChatTemplate: "/path/to/template.jinja"}, + }, + contains: []FlagCheck{{"--chat-template", "/path/to/template.jinja"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "speculative enabled without algorithm skips all speculative flags", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + Speculative: &inferencev1alpha1.SGLangSpeculativeConfig{ + Enabled: ptrBool(true), + Algorithm: "", + DraftModelPath: "/models/draft", + }, + }, + }, + notContains: []string{"--speculative-algorithm", "--speculative-draft-model-path", "--speculative-num-steps", "--speculative-eagle-topk", "--speculative-num-draft-tokens"}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "speculative enabled+configured emits all flags", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + Speculative: &inferencev1alpha1.SGLangSpeculativeConfig{ + Enabled: ptrBool(true), + Algorithm: "EAGLE3", + DraftModelPath: "/models/draft", + NumSteps: ptrInt32(3), + EagleTopK: ptrInt32(8), + NumDraftTokens: ptrInt32(5), + }, + }, + }, + contains: []FlagCheck{ + {"--speculative-algorithm", "EAGLE3"}, + {"--speculative-draft-model-path", "/models/draft"}, + {"--speculative-num-steps", "3"}, + {"--speculative-eagle-topk", "8"}, + {"--speculative-num-draft-tokens", "5"}, + }, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "loraModules emits --lora-modules", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + LoraModules: []string{`{"name":"loraA","path":"/loras/a"}`}, + }, + }, + contains: []FlagCheck{{"--lora-modules", `{"name":"loraA","path":"/loras/a"}`}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "maxLoraRank emits --max-lora-rank", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{MaxLoraRank: ptrInt32(64)}, + }, + contains: []FlagCheck{{"--max-lora-rank", "64"}}, + }, + { + model: &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}}, + name: "loraTargetModules emits --lora-target-modules", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + LoraTargetModules: []string{"q_proj", "k_proj"}, + }, + }, + contains: []FlagCheck{{"--lora-target-modules", "q_proj,k_proj"}}, + }, + { + model: sglangGPUModel(), + name: "full agentic config emits all flags together", + spec: &inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + TensorParallelSize: ptrInt32(2), + ContextLength: ptrInt32(131072), + MemFractionStatic: ptrFloat64(0.85), + ChunkedPrefillSize: ptrInt32(8192), + MaxRunningRequests: ptrInt32(64), + Quantization: "fp8", + KVCacheDtype: ptrString("fp8_e5m2"), + AttentionBackend: "flashinfer", + EnablePrefixCaching: ptrBool(true), + ToolCallParser: "qwen3", + ReasoningParser: "qwen3", + }, + }, + contains: []FlagCheck{ + {"--tp", "2"}, + {"--context-length", "131072"}, + {"--mem-fraction-static", "0.85"}, + {"--chunked-prefill-size", "8192"}, + {"--max-running-requests", "64"}, + {"--quantization", "fp8"}, + {"--kv-cache-dtype", "fp8_e5m2"}, + {"--attention-backend", "flashinfer"}, + {"--enable-prefix-caching", ""}, + {"--tool-call-parser", "qwen3"}, + {"--reasoning-parser", "qwen3"}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + isvc := &inferencev1alpha1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "isvc", Namespace: "default"}, + Spec: *tc.spec, + } + args := backend.BuildArgs(isvc, tc.model, modelPath, port) + for _, fc := range tc.contains { + if !containsArg(args, fc.flag, fc.value) { + t.Errorf("expected %q %q in args, got: %v", fc.flag, fc.value, args) + } + } + for _, f := range tc.notContains { + if containsArg(args, f, "") { + t.Errorf("expected %q NOT in args, got: %v", f, args) + } + } + }) + } +} + +// TestSGLangBuildArgsDeterministic verifies BuildArgs emits flags in the +// same order across calls so Deployment .spec diffs stay quiet. +func TestSGLangBuildArgsDeterministic(t *testing.T) { + backend := &SGLangBackend{} + model := &inferencev1alpha1.Model{ObjectMeta: metav1.ObjectMeta{Name: "m"}} + isvc := &inferencev1alpha1.InferenceService{ + ObjectMeta: metav1.ObjectMeta{Name: "svc"}, + Spec: inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + TensorParallelSize: ptrInt32(2), + KVCacheDtype: ptrString("fp8_e5m2"), + EnablePrefixCaching: ptrBool(true), + ContextLength: ptrInt32(131072), + }, + }, + } + first := backend.BuildArgs(isvc, model, "/models/x", 30000) + for i := 0; i < 10; i++ { + got := backend.BuildArgs(isvc, model, "/models/x", 30000) + if len(got) != len(first) { + t.Fatalf("iteration %d: length differs: got %d want %d", i, len(got), len(first)) + } + for j := range got { + if got[j] != first[j] { + t.Fatalf("iteration %d pos %d: %q != %q", i, j, got[j], first[j]) + } + } + } +} + +func TestSGLangBuildCommand(t *testing.T) { + b := &SGLangBackend{} + want := []string{"python3", "-m", "sglang.launch_server"} + got := b.BuildCommand() + if !reflect.DeepEqual(got, want) { + t.Errorf("BuildCommand() = %v, want %v", got, want) + } +} + +func TestSGLangProbes(t *testing.T) { + b := &SGLangBackend{} + startup, liveness, readiness := b.BuildProbes(30000) + + if startup == nil || startup.HTTPGet == nil || startup.HTTPGet.Path != "/health_generate" { + t.Errorf("startup probe should hit /health_generate, got %+v", startup) + } + if startup.FailureThreshold != 180 { + t.Errorf("startup FailureThreshold = %d, want 180 (cold-start tolerance)", startup.FailureThreshold) + } + + if liveness == nil || liveness.HTTPGet == nil || liveness.HTTPGet.Path != "/health" { + t.Errorf("liveness probe should hit /health, got %+v", liveness) + } + if liveness.FailureThreshold != 3 { + t.Errorf("liveness FailureThreshold = %d, want 3", liveness.FailureThreshold) + } + + if readiness == nil || readiness.HTTPGet == nil || readiness.HTTPGet.Path != "/health_generate" { + t.Errorf("readiness probe should hit /health_generate, got %+v", readiness) + } + if readiness.FailureThreshold != 3 { + t.Errorf("readiness FailureThreshold = %d, want 3", readiness.FailureThreshold) + } +} + +func TestSGLangBuildEnv(t *testing.T) { + b := &SGLangBackend{} + + // nil when no HFTokenSecretRef. + if got := b.BuildEnv(&inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{Runtime: "sglang"}, + }); got != nil { + t.Errorf("BuildEnv() with no HFTokenSecretRef = %v, want nil", got) + } + + // HF_TOKEN env when SecretRef is set. + isvc := &inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + HFTokenSecretRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "hf-secret"}, + Key: "HF_TOKEN", + }, + }, + }, + } + got := b.BuildEnv(isvc) + if len(got) != 1 { + t.Fatalf("BuildEnv() = %v, want one env var", got) + } + if got[0].Name != "HF_TOKEN" { + t.Errorf("env[0].Name = %q, want %q", got[0].Name, "HF_TOKEN") + } + if got[0].ValueFrom == nil || got[0].ValueFrom.SecretKeyRef == nil { + t.Errorf("env[0].ValueFrom = nil, want SecretKeyRef") + } +} + +func TestValidateSGLangConfig(t *testing.T) { + cases := []struct { + name string + isvc *inferencev1alpha1.InferenceService + wantReason string + }{ + { + name: "nil isvc is valid", + isvc: nil, + wantReason: "", + }, + { + name: "nil sglang config is valid", + isvc: &inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{Runtime: "sglang"}, + }, + wantReason: "", + }, + { + name: "speculative disabled is valid", + isvc: &inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + Speculative: &inferencev1alpha1.SGLangSpeculativeConfig{Enabled: ptrBool(false)}, + }, + }, + }, + wantReason: "", + }, + { + name: "speculative enabled+configured is valid", + isvc: &inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + Speculative: &inferencev1alpha1.SGLangSpeculativeConfig{ + Enabled: ptrBool(true), + Algorithm: "EAGLE3", + DraftModelPath: "/models/draft", + }, + }, + }, + }, + wantReason: "", + }, + { + name: "speculative enabled without algorithm reports SpeculativeMissingConfig", + isvc: &inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + Speculative: &inferencev1alpha1.SGLangSpeculativeConfig{ + Enabled: ptrBool(true), + DraftModelPath: "/models/draft", + }, + }, + }, + }, + wantReason: "SpeculativeMissingConfig", + }, + { + name: "speculative enabled without draft-model-path reports SpeculativeMissingConfig", + isvc: &inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + Speculative: &inferencev1alpha1.SGLangSpeculativeConfig{ + Enabled: ptrBool(true), + Algorithm: "EAGLE3", + }, + }, + }, + }, + wantReason: "SpeculativeMissingConfig", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + reason, message := ValidateSGLangConfig(tc.isvc) + if reason != tc.wantReason { + t.Errorf("reason: got %q want %q (message=%q)", reason, tc.wantReason, message) + } + if reason != "" && message == "" { + t.Errorf("expected non-empty message when reason is set, got empty") + } + }) + } +} + +func TestReconcileSGLangSpecCondition(t *testing.T) { + cases := []struct { + name string + isvc *inferencev1alpha1.InferenceService + existingConds []metav1.Condition + wantTypeExists bool + wantStatus metav1.ConditionStatus + wantReason string + }{ + { + name: "non-sglang runtime removes condition", + isvc: &inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{Runtime: "vllm"}, + ObjectMeta: metav1.ObjectMeta{Name: "test", Generation: 1}, + Status: inferencev1alpha1.InferenceServiceStatus{ + Conditions: []metav1.Condition{{ + Type: ConditionSGLangSpecValid, + Status: metav1.ConditionFalse, + }}, + }, + }, + wantTypeExists: false, + }, + { + name: "valid config with no prior condition does nothing", + isvc: &inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{Runtime: "sglang"}, + ObjectMeta: metav1.ObjectMeta{Name: "test", Generation: 1}, + }, + wantTypeExists: false, + }, + { + name: "valid config clears prior False condition", + isvc: &inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{Runtime: "sglang"}, + ObjectMeta: metav1.ObjectMeta{Name: "test", Generation: 1}, + Status: inferencev1alpha1.InferenceServiceStatus{ + Conditions: []metav1.Condition{{ + Type: ConditionSGLangSpecValid, + Status: metav1.ConditionFalse, + }}, + }, + }, + existingConds: []metav1.Condition{{ + Type: ConditionSGLangSpecValid, + Status: metav1.ConditionFalse, + }}, + wantTypeExists: true, + wantStatus: metav1.ConditionTrue, + wantReason: "ConfigValid", + }, + { + name: "invalid speculative config sets False condition", + isvc: &inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{ + Runtime: "sglang", + SGLangConfig: &inferencev1alpha1.SGLangConfig{ + Speculative: &inferencev1alpha1.SGLangSpeculativeConfig{ + Enabled: ptrBool(true), + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{Name: "test", Generation: 2}, + }, + wantTypeExists: true, + wantStatus: metav1.ConditionFalse, + wantReason: "SpeculativeMissingConfig", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := &InferenceServiceReconciler{} + r.reconcileSGLangSpecCondition(tc.isvc) + + cond := findCondition(tc.isvc.Status.Conditions, ConditionSGLangSpecValid) + if tc.wantTypeExists { + if cond == nil { + t.Fatalf("expected condition %q to exist", ConditionSGLangSpecValid) + } + if cond.Status != tc.wantStatus { + t.Errorf("status: got %q want %q", cond.Status, tc.wantStatus) + } + if cond.Reason != tc.wantReason { + t.Errorf("reason: got %q want %q", cond.Reason, tc.wantReason) + } + } else { + if cond != nil { + t.Errorf("expected condition %q to not exist, got status=%q reason=%q", ConditionSGLangSpecValid, cond.Status, cond.Reason) + } + } + }) + } +} + +func findCondition(conds []metav1.Condition, ctype string) *metav1.Condition { + for i := range conds { + if conds[i].Type == ctype { + return &conds[i] + } + } + return nil +} diff --git a/internal/controller/runtime_test.go b/internal/controller/runtime_test.go index ba8ffe14..ac01f8be 100644 --- a/internal/controller/runtime_test.go +++ b/internal/controller/runtime_test.go @@ -51,6 +51,7 @@ func TestRuntimeNameLabel(t *testing.T) { // untouched: the label is the user-declared identifier, not a // validated enum, so new backends do not need to update this map. {name: "unknown runtime passes through verbatim", runtime: "future-thing", expected: "future-thing"}, + {name: "sglang passes through", runtime: "sglang", expected: "sglang"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -139,3 +140,13 @@ func TestResolveGPUCount(t *testing.T) { }) } } + +func TestResolveBackend_SGLang(t *testing.T) { + isvc := &inferencev1alpha1.InferenceService{ + Spec: inferencev1alpha1.InferenceServiceSpec{Runtime: "sglang"}, + } + backend := resolveBackend(isvc) + if _, ok := backend.(*SGLangBackend); !ok { + t.Errorf("resolveBackend(sglang) = %T, want *SGLangBackend", backend) + } +} diff --git a/internal/controller/status_builder.go b/internal/controller/status_builder.go index 8c08cdf1..defacbb1 100644 --- a/internal/controller/status_builder.go +++ b/internal/controller/status_builder.go @@ -33,7 +33,7 @@ import ( // Status-subresource writes. This file owns the path from an in-memory // phase/condition decision to the actual Status().Update call, including: -// - VLLMSpecValid condition maintenance (informational) +// - VLLMSpecValid and SGLangSpecValid condition maintenance (informational) // - cluster-local endpoint URL construction // - the omnibus updateStatusWithSchedulingInfo that writes phase, // replica counts, endpoint, scheduling diagnostics, priority, @@ -76,6 +76,40 @@ func (r *InferenceServiceReconciler) reconcileVLLMSpecCondition(isvc *inferencev }) } +// reconcileSGLangSpecCondition sets or clears the SGLangSpecValid status condition +// based on ValidateSGLangConfig. This is informational only — it does not block +// Deployment creation. The controller's main Status().Update at the end of +// reconcile persists the condition. +func (r *InferenceServiceReconciler) reconcileSGLangSpecCondition(isvc *inferencev1alpha1.InferenceService) { + if isvc.Spec.Runtime != RuntimeSGLANG { + meta.RemoveStatusCondition(&isvc.Status.Conditions, ConditionSGLangSpecValid) + return + } + reason, message := ValidateSGLangConfig(isvc) + now := metav1.NewTime(time.Now()) + if reason == "" { + if existing := meta.FindStatusCondition(isvc.Status.Conditions, ConditionSGLangSpecValid); existing != nil { + meta.SetStatusCondition(&isvc.Status.Conditions, metav1.Condition{ + Type: ConditionSGLangSpecValid, + Status: metav1.ConditionTrue, + ObservedGeneration: isvc.Generation, + LastTransitionTime: now, + Reason: "ConfigValid", + Message: "SGLang configuration is valid", + }) + } + return + } + meta.SetStatusCondition(&isvc.Status.Conditions, metav1.Condition{ + Type: ConditionSGLangSpecValid, + Status: metav1.ConditionFalse, + ObservedGeneration: isvc.Generation, + LastTransitionTime: now, + Reason: reason, + Message: message, + }) +} + func (r *InferenceServiceReconciler) constructEndpoint(isvc *inferencev1alpha1.InferenceService, svc *corev1.Service) string { port := int32(8080) path := "/v1/chat/completions"