diff --git a/cmd/aima/diskcheck.go b/cmd/aima/diskcheck.go new file mode 100644 index 00000000..74bb13ad --- /dev/null +++ b/cmd/aima/diskcheck.go @@ -0,0 +1,28 @@ +package main + +// diskReserveMiB is the free space kept beyond a model download so the node +// does not cross k3s's disk-pressure eviction threshold (kubelet defaults evict +// at nodefs<10% / imagefs<15%, which taints the node NoSchedule and leaves pods +// stuck Pending). Reserve 10% of the volume, with a 2 GiB floor for small disks. +func diskReserveMiB(totalMiB int64) int64 { + reserve := totalMiB / 10 + if reserve < 2048 { + reserve = 2048 + } + return reserve +} + +// enoughDiskForDownload reports whether a download of requiredMiB fits on a +// filesystem with freeMiB free / totalMiB total while preserving the reserve. +// requiredMiB<=0 means the size is unknown, in which case it cannot gate and +// returns true (best-effort). Returns the shortfall in MiB when it does not fit. +func enoughDiskForDownload(requiredMiB, freeMiB, totalMiB int64) (ok bool, shortfallMiB int64) { + if requiredMiB <= 0 { + return true, 0 + } + need := requiredMiB + diskReserveMiB(totalMiB) + if freeMiB >= need { + return true, 0 + } + return false, need - freeMiB +} diff --git a/cmd/aima/diskcheck_test.go b/cmd/aima/diskcheck_test.go new file mode 100644 index 00000000..805f5329 --- /dev/null +++ b/cmd/aima/diskcheck_test.go @@ -0,0 +1,59 @@ +package main + +import "testing" + +func TestEnoughDiskForDownload(t *testing.T) { + tests := []struct { + name string + requiredMiB int64 + freeMiB int64 + totalMiB int64 + wantOK bool + wantShortfallMiB int64 + }{ + { + name: "plenty of space", + requiredMiB: 5000, freeMiB: 200000, totalMiB: 500000, + wantOK: true, + }, + { + // 5 GB download, only 6 GB free on a 100 GB disk: reserve is 10% + // (10 GB), so it needs 15 GB but has 6 GB — would trip disk-pressure. + name: "download would trip disk-pressure", + requiredMiB: 5000, freeMiB: 6000, totalMiB: 100000, + wantOK: false, wantShortfallMiB: 9000, + }, + { + // Small disk: reserve floors at 2048 MiB rather than 10%. + name: "reserve floor on small disk", + requiredMiB: 1000, freeMiB: 2500, totalMiB: 10000, + wantOK: false, wantShortfallMiB: 548, + }, + { + // Unknown download size: cannot gate, must not false-block. + name: "unknown size does not gate", + requiredMiB: 0, freeMiB: 100, totalMiB: 1000, + wantOK: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ok, shortfall := enoughDiskForDownload(tt.requiredMiB, tt.freeMiB, tt.totalMiB) + if ok != tt.wantOK { + t.Errorf("ok = %v, want %v", ok, tt.wantOK) + } + if !ok && shortfall != tt.wantShortfallMiB { + t.Errorf("shortfall = %d, want %d", shortfall, tt.wantShortfallMiB) + } + }) + } +} + +func TestDiskReserveMiB(t *testing.T) { + if got := diskReserveMiB(500000); got != 50000 { + t.Errorf("diskReserveMiB(500000) = %d, want 50000 (10%%)", got) + } + if got := diskReserveMiB(10000); got != 2048 { + t.Errorf("diskReserveMiB(10000) = %d, want 2048 (floor)", got) + } +} diff --git a/cmd/aima/main.go b/cmd/aima/main.go index bbd1445c..6243171e 100644 --- a/cmd/aima/main.go +++ b/cmd/aima/main.go @@ -1517,6 +1517,10 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { // Step 3: Pull model (non-fatal — may be local or pre-installed). Use // pullModelCore directly so byte-level progress can flow back to the // caller via onModelProgress (the deps.PullModel closure swallows it). + // A disk guard aborts the moment the download's total size is known to + // not fit (keeping a disk-pressure reserve), turning a disk-filling hang + // — which taints the k3s node NoSchedule so the pod never starts — into + // a clear, fatal, pre-transfer error. if !noPull { notify("pulling_model", model) modelStatus := func(phase, msg string) { @@ -1524,7 +1528,31 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { notify("pulling_model", msg) } } - if err := pullModelCore(ctx, model, modelStatus, onModelProgress); err != nil { + freeMiB, totalMiB := hal.DiskUsage(dataDir) + pullCtx, cancelPull := context.WithCancel(ctx) + var diskErr error + diskChecked := false + guardedProgress := func(downloaded, total int64) { + if !diskChecked && total > 0 && freeMiB > 0 { + diskChecked = true + requiredMiB := total / (1024 * 1024) + if ok, shortfall := enoughDiskForDownload(requiredMiB, freeMiB, totalMiB); !ok { + diskErr = fmt.Errorf("not enough disk space to download %s: needs ~%d MiB plus reserve but only %d MiB free on %s (short by ~%d MiB); free up space or choose a smaller model", + model, requiredMiB, freeMiB, dataDir, shortfall) + cancelPull() + return + } + } + if onModelProgress != nil { + onModelProgress(downloaded, total) + } + } + err := pullModelCore(pullCtx, model, modelStatus, guardedProgress) + cancelPull() + if diskErr != nil { + return nil, diskErr + } + if err != nil { notify("model_skip", err.Error()) } } diff --git a/internal/hal/detect.go b/internal/hal/detect.go index 12425c4d..35438348 100644 --- a/internal/hal/detect.go +++ b/internal/hal/detect.go @@ -142,6 +142,12 @@ func detectStorage() StorageInfo { } } +// DiskUsage returns free and total space (MiB) on the filesystem containing +// path, or (0, 0) if it cannot be determined. Cross-platform via diskStats. +func DiskUsage(path string) (freeMiB, totalMiB int64) { + return diskStats(path) +} + // collectMetricsWithRunner gathers real-time metrics using given CommandRunner. func collectMetricsWithRunner(ctx context.Context, runner CommandRunner) (*Metrics, error) { m := &Metrics{} diff --git a/internal/knowledge/offload_test.go b/internal/knowledge/offload_test.go new file mode 100644 index 00000000..977b17dd --- /dev/null +++ b/internal/knowledge/offload_test.go @@ -0,0 +1,104 @@ +package knowledge + +import "testing" + +// baseOffloadInputs builds a resolved/engine/variant/hw set representing a +// llamacpp GGUF variant that declares "fits any GPU" (vram_min_mib=0) with a +// full-GPU-offload default (n_gpu_layers=999) and a real RAM footprint. +func baseOffloadInputs() (*ResolvedConfig, *EngineAsset, *ModelVariant) { + resolved := &ResolvedConfig{ + Config: map[string]any{"n_gpu_layers": 999}, + Provenance: map[string]string{"n_gpu_layers": "L0"}, + } + engine := &EngineAsset{Amplifier: EngineAmplifier{OffloadConfigKey: "n_gpu_layers"}} + variant := &ModelVariant{Hardware: ModelVariantHardware{RAMMinMiB: 7168}} + return resolved, engine, variant +} + +func TestCapGPUOffloadDowngradesOnSmallDiscreteGPU(t *testing.T) { + resolved, engine, variant := baseOffloadInputs() + hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: false} + + capGPUOffloadForVRAM(resolved, engine, variant, hw) + + if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 0 { + t.Errorf("n_gpu_layers = %v, want 0 (CPU) when model (7168 MiB) exceeds GPU VRAM (4096 MiB)", got) + } + if !resolved.OffloadPath { + t.Error("OffloadPath = false, want true after CPU downgrade") + } + if len(resolved.Warnings) != 1 { + t.Errorf("Warnings = %d, want 1 explaining the CPU fallback", len(resolved.Warnings)) + } +} + +func TestCapGPUOffloadLeavesUnifiedMemoryAlone(t *testing.T) { + resolved, engine, variant := baseOffloadInputs() + // Unified-memory APU/GB10/Apple: the GPU shares system RAM, so full offload + // is correct even when the footprint exceeds nominal VRAM. + hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: true} + + capGPUOffloadForVRAM(resolved, engine, variant, hw) + + if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 999 { + t.Errorf("n_gpu_layers = %v, want 999 (untouched) on unified memory", got) + } + if len(resolved.Warnings) != 0 { + t.Errorf("Warnings = %d, want 0 on unified memory", len(resolved.Warnings)) + } +} + +func TestCapGPUOffloadLeavesFittingGPUAlone(t *testing.T) { + resolved, engine, variant := baseOffloadInputs() + hw := HardwareInfo{GPUVRAMMiB: 49152, UnifiedMemory: false} // 48 GB card fits 7 GB model + + capGPUOffloadForVRAM(resolved, engine, variant, hw) + + if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 999 { + t.Errorf("n_gpu_layers = %v, want 999 (untouched) when model fits VRAM", got) + } +} + +func TestCapGPUOffloadRespectsUserOverride(t *testing.T) { + resolved, engine, variant := baseOffloadInputs() + resolved.Config["n_gpu_layers"] = 50 + resolved.Provenance["n_gpu_layers"] = "L1" // user set it explicitly + hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: false} + + capGPUOffloadForVRAM(resolved, engine, variant, hw) + + if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 50 { + t.Errorf("n_gpu_layers = %v, want 50 (user override respected)", got) + } +} + +func TestCapGPUOffloadNoopWithoutOffloadKey(t *testing.T) { + resolved, engine, variant := baseOffloadInputs() + engine.Amplifier.OffloadConfigKey = "" // engine declares no offload knob + hw := HardwareInfo{GPUVRAMMiB: 4096, UnifiedMemory: false} + + capGPUOffloadForVRAM(resolved, engine, variant, hw) + + if got := toFloat64(resolved.Config["n_gpu_layers"]); got != 999 { + t.Errorf("n_gpu_layers = %v, want 999 (untouched) when no offload_config_key", got) + } +} + +func TestCheckFitSurfacesResolvedWarnings(t *testing.T) { + resolved := &ResolvedConfig{ + Config: map[string]any{}, + Warnings: []string{"model needs ~7168 MiB but this GPU has 4096 MiB VRAM"}, + } + + fit := CheckFit(resolved, HardwareInfo{}) + + found := false + for _, w := range fit.Warnings { + if w == resolved.Warnings[0] { + found = true + } + } + if !found { + t.Errorf("CheckFit did not surface resolve-time warning; got %v", fit.Warnings) + } +} diff --git a/internal/knowledge/resolver.go b/internal/knowledge/resolver.go index 906a61c0..49067f96 100644 --- a/internal/knowledge/resolver.go +++ b/internal/knowledge/resolver.go @@ -97,6 +97,10 @@ type ResolvedConfig struct { // Performance reference (K4 — historical data or YAML estimate) PerformanceRef *PerformanceReference + + // Warnings collected during resolution (e.g. GPU-offload downgrade). Surfaced + // to the user via CheckFit's FitReport so resolve-time decisions are visible. + Warnings []string } // PerformanceReference attaches known performance data to a resolved config. @@ -295,6 +299,7 @@ func (c *Catalog) Resolve(hw HardwareInfo, modelName, engineType string, userOve resolved.EstimatedVRAMMiB = variant.Hardware.VRAMMinMiB } resolved.ResourceEstimate = estimateResources(engine, variant, hw) + capGPUOffloadForVRAM(resolved, engine, variant, hw) } // Container memory guardrail. On unified-memory hosts (GB10, Apple M-series, @@ -1596,6 +1601,47 @@ func (c *Catalog) UpsertSyntheticModel(ma ModelAsset) { c.ModelAssets = append(c.ModelAssets, ma) } +// capGPUOffloadForVRAM downgrades an engine's GPU-offload knob to CPU when a +// discrete GPU cannot hold the model. llama.cpp's default n_gpu_layers=999 +// ("offload every layer") assumes the model fits VRAM; on a discrete GPU with +// less VRAM than the model needs it forces an impossible allocation → CUDA OOM +// at load, which surfaces to users as a deployment that never becomes ready. +// Unified-memory hosts (APU/GB10/Apple) share system RAM with the GPU, so full +// offload is correct there and left untouched. The knob is identified by the +// engine-declared offload_config_key, so this stays engine-agnostic (INV-1/2): +// no engine name or config key is hardcoded here. +func capGPUOffloadForVRAM(resolved *ResolvedConfig, engine *EngineAsset, variant *ModelVariant, hw HardwareInfo) { + if resolved == nil || engine == nil || variant == nil { + return + } + key := engine.Amplifier.OffloadConfigKey + if key == "" || hw.UnifiedMemory || hw.GPUVRAMMiB <= 0 { + return + } + // Model memory footprint. RAMMinMiB is the catalog's footprint proxy, used + // precisely when vram_min_mib=0 (a GGUF variant declared to "fit any GPU" + // via CPU offload); fall back to any explicit VRAM estimate. + footprint := variant.Hardware.RAMMinMiB + if resolved.EstimatedVRAMMiB > footprint { + footprint = resolved.EstimatedVRAMMiB + } + if footprint <= 0 || footprint <= hw.GPUVRAMMiB { + return // fits, or footprint unknown — full GPU offload is correct + } + if resolved.Provenance[key] == "L1" { + return // user set the offload knob explicitly — respect it + } + if cur, ok := resolved.Config[key]; ok && toFloat64(cur) == 0 { + return // already CPU-only + } + resolved.Config[key] = 0 + resolved.Provenance[key] = "L0-fit" + resolved.OffloadPath = true + resolved.Warnings = append(resolved.Warnings, fmt.Sprintf( + "model needs ~%d MiB but this GPU has %d MiB VRAM; disabling GPU offload and running on CPU (much slower). Use a smaller model or a larger-VRAM GPU for GPU acceleration.", + footprint, hw.GPUVRAMMiB)) +} + // estimateResources computes cost(path, R) — the full resource consumption estimate. func estimateResources(engine *EngineAsset, variant *ModelVariant, hw HardwareInfo) *ResourceEstimate { perf := variant.ParsedExpectedPerf() @@ -1644,6 +1690,10 @@ var gmuKeys = []string{"gpu_memory_utilization", "mem_fraction_static"} func CheckFit(resolved *ResolvedConfig, hw HardwareInfo) *FitReport { r := &FitReport{Fit: true, Adjustments: make(map[string]any)} + // Surface warnings collected during resolution (e.g. GPU-offload downgrade) + // even if a hard-fail check below returns early. + r.Warnings = append(r.Warnings, resolved.Warnings...) + // Unified memory guard: GPU allocation directly reduces available system memory. // Enforce minimum OS reserve to prevent starvation / swap thrashing. if hw.UnifiedMemory && hw.RAMTotalMiB > 0 {