From f44b32e216dc4045d42c53ab75fec6dda6ad47de Mon Sep 17 00:00:00 2001 From: Bob Date: Sun, 5 Jul 2026 05:47:54 +0800 Subject: [PATCH 1/5] feat(report): rebuild artifact from run directory via bench report --artifact CreateOrAppend treats an empty existing file as a fresh artifact and rejects directories. --- internal/artifact/sqlite.go | 9 +++- internal/benchcli/benchcli.go | 62 +++++++++++++++++++++-- internal/vllmbench/model_artifact_test.go | 21 ++++++++ internal/vllmbench/sqlite_artifact.go | 4 +- 4 files changed, 90 insertions(+), 6 deletions(-) diff --git a/internal/artifact/sqlite.go b/internal/artifact/sqlite.go index 02bd129..f9a588f 100644 --- a/internal/artifact/sqlite.go +++ b/internal/artifact/sqlite.go @@ -47,9 +47,16 @@ func Create(path, schema string) (*sql.DB, error) { // The existing file must be a valid artifact of the supported format; // anything else is an error rather than something to silently overwrite. func CreateOrAppend(path, schema string) (*sql.DB, error) { - if _, err := os.Stat(path); err != nil { + info, err := os.Stat(path) + if err != nil { return Create(path, schema) } + if info.IsDir() { + return nil, fmt.Errorf("%s is a directory", path) + } + if info.Size() == 0 { + return openWithSchema(path, schema) + } db, err := OpenWritable(path) if err != nil { return nil, err diff --git a/internal/benchcli/benchcli.go b/internal/benchcli/benchcli.go index cb72656..3a65094 100644 --- a/internal/benchcli/benchcli.go +++ b/internal/benchcli/benchcli.go @@ -34,9 +34,64 @@ var rootHandlers = commandHandlers{ } var artifactHandlers = commandHandlers{ - "check": runArtifactCheck, - "render": runArtifactRender, - "merge": runArtifactMerge, + "check": runArtifactCheck, + "render": runArtifactRender, + "merge": runArtifactMerge, + "rebuild": runArtifactRebuild, +} + +// runArtifactRebuild reconstructs a SQLite artifact from a run directory's +// raw data (events, results, summary.json) — the recovery path when a run +// finished without an artifact or the artifact was lost. +func runArtifactRebuild(args []string) { + flags := flag.NewFlagSet("artifact rebuild", flag.ExitOnError) + runDir := flags.String("run-dir", "", "run directory with events.jsonl, results, and summary.json") + into := flags.String("into", "", "SQLite artifact path to rebuild/append; defaults to .sqlite") + specPath := flags.String("spec", "", "optional original spec path for provenance; defaults to /spec.normalized.json") + _ = flags.Parse(args) + if strings.TrimSpace(*runDir) == "" { + fmt.Fprintln(os.Stderr, "missing --run-dir") + os.Exit(2) + } + artifactPath := artifact.Path(*runDir, *into) + if err := rebuildArtifactFromRunDir(*runDir, artifactPath, *specPath); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Printf("artifact: %s\n", artifactPath) +} + +func rebuildArtifactFromRunDir(runDir, artifactPath, originalSpecPath string) error { + specLoadPath := originalSpecPath + if strings.TrimSpace(specLoadPath) == "" { + specLoadPath = filepath.Join(runDir, "spec.normalized.json") + } + spec, err := vllmbench.LoadSpec(specLoadPath) + if err != nil { + return err + } + summary, err := loadRunSummary(runDir) + if err != nil { + return err + } + summary.RunDir = runDir + summary.ArtifactPath = artifactPath + if strings.TrimSpace(summary.SpecPath) == "" { + summary.SpecPath = filepath.Join(runDir, "spec.normalized.json") + } + return vllmbench.WriteSQLiteArtifact(runDir, artifactPath, spec, summary, vllmbench.BuildPlan(spec, runDir), originalSpecPath) +} + +func loadRunSummary(runDir string) (vllmbench.RunSummary, error) { + data, err := os.ReadFile(filepath.Join(runDir, "summary.json")) + if err != nil { + return vllmbench.RunSummary{}, err + } + var summary vllmbench.RunSummary + if err := json.Unmarshal(data, &summary); err != nil { + return vllmbench.RunSummary{}, err + } + return summary, nil } // runArtifactMerge combines run artifacts into one model-level SQLite file; @@ -767,6 +822,7 @@ func usageRoot() { localperf artifact check runs/example.sqlite localperf artifact render runs/example.sqlite [--output runs/example.html] [--store] localperf artifact merge --into runs/models/model.sqlite src1.sqlite [src2.sqlite ...] + localperf artifact rebuild --run-dir runs/example [--into runs/example.sqlite] [--spec spec.json] localperf sweep plan --model model-id [--contexts 8k,16k,32k,64k] [--concurrency 1,4,8,16,32] [--repeats 1] [--reference] [--stress] [--vllm-command /path/to/vllm] [--gpu-memory-utilization 0.4] [--kv-cache-memory-bytes N] [--trim 64k=8:'reason'] [--out spec.json] localperf view runs/model.sqlite [runs/other.sqlite ...] [--addr 127.0.0.1:0] [--open]`) } diff --git a/internal/vllmbench/model_artifact_test.go b/internal/vllmbench/model_artifact_test.go index 1124fd1..65667f6 100644 --- a/internal/vllmbench/model_artifact_test.go +++ b/internal/vllmbench/model_artifact_test.go @@ -44,6 +44,27 @@ func TestModelLevelArtifactAppendsRuns(t *testing.T) { assertRunCount(t, artifactPath, 2) } +func TestModelLevelArtifactInitializesEmptyExistingPath(t *testing.T) { + dir := t.TempDir() + artifactPath := filepath.Join(dir, "model.sqlite") + if err := os.WriteFile(artifactPath, nil, 0o644); err != nil { + t.Fatal(err) + } + spec := testSpec() + spec.OutputDir = dir + if _, err := Execute(context.Background(), spec, RunOptions{ + DryRun: true, + RunDir: filepath.Join(dir, "run"), + ArtifactPath: artifactPath, + }); err != nil { + t.Fatal(err) + } + if err := artifact.Check(artifactPath); err != nil { + t.Fatalf("empty-path artifact check failed: %v", err) + } + assertRunCount(t, artifactPath, 1) +} + func TestArtifactMergeCombinesRunsAndSkipsDuplicates(t *testing.T) { dir := t.TempDir() makeSingle := func(name string) string { diff --git a/internal/vllmbench/sqlite_artifact.go b/internal/vllmbench/sqlite_artifact.go index db8429f..3ff95a0 100644 --- a/internal/vllmbench/sqlite_artifact.go +++ b/internal/vllmbench/sqlite_artifact.go @@ -30,8 +30,8 @@ func WriteSQLiteArtifact(runDir, artifactPath string, spec Spec, summary RunSumm if strings.TrimSpace(artifactPath) == "" { return nil } - _, statErr := os.Stat(artifactPath) - createdFresh := statErr != nil + info, statErr := os.Stat(artifactPath) + createdFresh := statErr != nil || (statErr == nil && info.Size() == 0) db, err := createSQLiteArtifact(artifactPath) if err != nil { return err From fc5e3b09f78ffb22b6898b5d9fdb329ec9a2df66 Mon Sep 17 00:00:00 2001 From: Bob Date: Sun, 5 Jul 2026 05:51:04 +0800 Subject: [PATCH 2/5] test(artifact): cover WriteSQLiteArtifact no-op and failed-write cleanup --- internal/benchcli/benchcli.go | 48 +++++++++++++++------ internal/sweepplan/sweepplan.go | 52 +++++++++++++++++++++-- internal/sweepplan/sweepplan_test.go | 32 ++++++++++++++ internal/vllmbench/model_artifact_test.go | 23 ++++++++++ 4 files changed, 139 insertions(+), 16 deletions(-) diff --git a/internal/benchcli/benchcli.go b/internal/benchcli/benchcli.go index 3a65094..4608e7a 100644 --- a/internal/benchcli/benchcli.go +++ b/internal/benchcli/benchcli.go @@ -147,6 +147,12 @@ func runSweepPlan(args []string) { var trims trimFlags flags.Var(&trims, "trim", "cap a context's ladder with a reason, e.g. 64k=8:'12 GiB KV budget'; repeatable") out := flags.String("out", "", "output spec path (default stdout)") + var profileArgs repeatedStringFlag + var profileEngineArgs repeatedStringFlag + var omitProfileEngineFlags repeatedStringFlag + flags.Var(&profileArgs, "profile-arg", "extra generated profile arg; repeat once per arg") + flags.Var(&profileEngineArgs, "profile-engine-arg", "extra generated profile engine arg; repeat once per arg") + flags.Var(&omitProfileEngineFlags, "omit-profile-engine-flag", "engine flag to remove from generated profile engine args; repeatable") _ = flags.Parse(args) contextValues, err := parseTokenList(*contexts) if err != nil { @@ -159,19 +165,22 @@ func runSweepPlan(args []string) { os.Exit(2) } spec, err := sweepplan.Plan(sweepplan.PlanRequest{ - Model: *model, - Contexts: contextValues, - Concurrency: concurrencyValues, - Repeats: *repeats, - NumPrompts: *numPrompts, - PromptsPerUser: *promptsPerUser, - IncludeReference: *reference, - IncludeStress: *stress, - MinMemAvailableGiB: *memFloor, - VLLMCommand: *vllmCommand, - GPUMemoryUtilization: *gpuMemUtil, - KVCacheMemoryBytes: *kvCacheBytes, - Trims: trims.values, + Model: *model, + Contexts: contextValues, + Concurrency: concurrencyValues, + Repeats: *repeats, + NumPrompts: *numPrompts, + PromptsPerUser: *promptsPerUser, + IncludeReference: *reference, + IncludeStress: *stress, + ProfileArgs: profileArgs.values, + ProfileEngineArgs: profileEngineArgs.values, + OmitProfileEngineFlags: omitProfileEngineFlags.values, + MinMemAvailableGiB: *memFloor, + VLLMCommand: *vllmCommand, + GPUMemoryUtilization: *gpuMemUtil, + KVCacheMemoryBytes: *kvCacheBytes, + Trims: trims.values, }) if err != nil { fmt.Fprintln(os.Stderr, err) @@ -234,6 +243,19 @@ func (flags *trimFlags) Set(value string) error { return nil } +type repeatedStringFlag struct { + values []string +} + +func (flag *repeatedStringFlag) Set(value string) error { + flag.values = append(flag.values, value) + return nil +} + +func (flag *repeatedStringFlag) String() string { + return strings.Join(flag.values, ",") +} + // parseTokenList parses values such as "8k,16k,32768" into token counts. func parseTokenList(value string) ([]int, error) { var values []int diff --git a/internal/sweepplan/sweepplan.go b/internal/sweepplan/sweepplan.go index d48d2fb..a89a978 100644 --- a/internal/sweepplan/sweepplan.go +++ b/internal/sweepplan/sweepplan.go @@ -37,6 +37,14 @@ type PlanRequest struct { // 32k c4 and 64k c1/c4 when those contexts are in the ladder) and the // 128k points at c1/c4. IncludeStress bool `json:"include_stress,omitempty"` + // ProfileArgs are appended to every generated profile as serve CLI args. + ProfileArgs []string `json:"profile_args,omitempty"` + // ProfileEngineArgs are appended to every generated profile as engine + // CLI args. Use OmitProfileEngineFlags to remove unsafe inherited flags. + ProfileEngineArgs []string `json:"profile_engine_args,omitempty"` + // OmitProfileEngineFlags removes matching "--flag value" and + // "--flag=value" entries from ProfileEngineArgs before emitting profiles. + OmitProfileEngineFlags []string `json:"omit_profile_engine_flags,omitempty"` // MinMemAvailableGiB is the safety memory floor; defaults to 40. MinMemAvailableGiB float64 `json:"min_mem_available_gib,omitempty"` // BasePort is the first server port; defaults to 8100. @@ -168,7 +176,7 @@ func Plan(request PlanRequest) (vllmbench.Spec, error) { maxNumSeqs := maxConcurrencyOf(request.Concurrency) port := request.BasePort if request.IncludeReference { - spec.Profiles = append(spec.Profiles, sweepProfile("4k-reference", referenceContext, port, maxNumSeqs)) + spec.Profiles = append(spec.Profiles, sweepProfile("4k-reference", referenceContext, port, maxNumSeqs, request)) port++ spec.Workloads = append(spec.Workloads, sweepWorkload( "max-throughput-reference", "decode", "4k-reference", @@ -181,7 +189,7 @@ func Plan(request PlanRequest) (vllmbench.Spec, error) { } for _, context := range contexts { label := vllmbench.TokenCountLabel(context) - spec.Profiles = append(spec.Profiles, sweepProfile(label, context, port, contextMaxSeqs(context, request))) + spec.Profiles = append(spec.Profiles, sweepProfile(label, context, port, contextMaxSeqs(context, request), request)) port++ prefillInput, prefillOutput := PrefillShape(context) spec.Workloads = append(spec.Workloads, sweepWorkload( @@ -263,7 +271,7 @@ func stampSpec(spec *vllmbench.Spec, request PlanRequest) error { return nil } -func sweepProfile(name string, maxModelLen, port, maxNumSeqs int) vllmbench.Profile { +func sweepProfile(name string, maxModelLen, port, maxNumSeqs int, request PlanRequest) vllmbench.Profile { return vllmbench.Profile{ Name: name, Host: "127.0.0.1", @@ -271,9 +279,47 @@ func sweepProfile(name string, maxModelLen, port, maxNumSeqs int) vllmbench.Prof Managed: true, MaxModelLen: maxModelLen, MaxNumSeqs: maxNumSeqs, + Args: append([]string{}, request.ProfileArgs...), + EngineArgs: omittedFlagValues(request.ProfileEngineArgs, request.OmitProfileEngineFlags), } } +func omittedFlagValues(args, omittedFlags []string) []string { + if len(args) == 0 { + return nil + } + if len(omittedFlags) == 0 { + return append([]string{}, args...) + } + omitted := map[string]struct{}{} + for _, flag := range omittedFlags { + if strings.TrimSpace(flag) != "" { + omitted[flag] = struct{}{} + } + } + filtered := make([]string, 0, len(args)) + for index := 0; index < len(args); index++ { + arg := args[index] + if _, ok := omitted[arg]; ok { + if index+1 < len(args) { + index++ + } + continue + } + skip := false + for flag := range omitted { + if strings.HasPrefix(arg, flag+"=") { + skip = true + break + } + } + if !skip { + filtered = append(filtered, arg) + } + } + return filtered +} + // stressWorkloads adds long-output decode spot checks for ladder contexts // that have them: 4096-token output at 32k c4 and 64k c1/c4. Kept out of the // default grid because they dominate sweep wall time. diff --git a/internal/sweepplan/sweepplan_test.go b/internal/sweepplan/sweepplan_test.go index e1c4310..6970f27 100644 --- a/internal/sweepplan/sweepplan_test.go +++ b/internal/sweepplan/sweepplan_test.go @@ -98,6 +98,38 @@ func TestPlanGolden(t *testing.T) { } } +func TestPlanProfileArgsCanOmitUnsafeEngineFlags(t *testing.T) { + spec, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{8192}, + Concurrency: []int{1}, + ProfileArgs: []string{"--trust-remote-code"}, + ProfileEngineArgs: []string{ + "--kv-cache-memory-bytes", "21474836480", + "--attention-backend", "flashinfer", + "--unsafe-flag=1", + }, + OmitProfileEngineFlags: []string{"--kv-cache-memory-bytes", "--unsafe-flag"}, + }) + if err != nil { + t.Fatal(err) + } + if len(spec.Profiles) != 1 { + t.Fatalf("profiles = %d, want 1", len(spec.Profiles)) + } + profile := spec.Profiles[0] + if strings.Join(profile.Args, " ") != "--trust-remote-code" { + t.Fatalf("args = %v, want trust remote code", profile.Args) + } + got := strings.Join(profile.EngineArgs, " ") + if strings.Contains(got, "kv-cache-memory-bytes") || strings.Contains(got, "unsafe-flag") { + t.Fatalf("engine args = %v, want omitted flags removed", profile.EngineArgs) + } + if got != "--attention-backend flashinfer" { + t.Fatalf("engine args = %q, want retained safe args", got) + } +} + func TestStressProfilesAdmitSpotCheckConcurrency(t *testing.T) { spec, err := Plan(PlanRequest{ Model: "example/model", diff --git a/internal/vllmbench/model_artifact_test.go b/internal/vllmbench/model_artifact_test.go index 65667f6..d5c19a3 100644 --- a/internal/vllmbench/model_artifact_test.go +++ b/internal/vllmbench/model_artifact_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/dutifuldev/localperf/internal/artifact" ) @@ -231,3 +232,25 @@ func assertRunCount(t *testing.T, path string, want int) { t.Fatalf("run rows = %d, want %d", runs, want) } } + +func TestWriteSQLiteArtifactNoopWithoutPath(t *testing.T) { + if err := WriteSQLiteArtifact(t.TempDir(), "", testSpec(), RunSummary{}, nil, ""); err != nil { + t.Fatalf("WriteSQLiteArtifact without path = %v, want nil", err) + } +} + +func TestWriteSQLiteArtifactRemovesFreshFileOnFailedWrite(t *testing.T) { + dir := t.TempDir() + artifactPath := filepath.Join(dir, "model.sqlite") + spec := testSpec() + // Duplicate workload names violate the artifact's unique constraint, + // failing the first write; the fresh file must not be left behind. + spec.Workloads = append(spec.Workloads, spec.Workloads[0]) + err := WriteSQLiteArtifact(filepath.Join(dir, "run"), artifactPath, spec, RunSummary{StartedAt: time.Now()}, nil, "") + if err == nil { + t.Fatal("write with duplicate workloads succeeded") + } + if _, statErr := os.Stat(artifactPath); statErr == nil { + t.Fatal("failed first write left a schema-only artifact behind") + } +} From 587865f69b55fa08526a09d89ee0bf48b32b013d Mon Sep 17 00:00:00 2001 From: Bob Date: Sun, 5 Jul 2026 13:18:57 +0800 Subject: [PATCH 3/5] fix(sweep): omit fixed qwen kv cache --- internal/sweepplan/sweepplan.go | 12 +++++++- internal/sweepplan/sweepplan_test.go | 41 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/internal/sweepplan/sweepplan.go b/internal/sweepplan/sweepplan.go index a89a978..c8f83fa 100644 --- a/internal/sweepplan/sweepplan.go +++ b/internal/sweepplan/sweepplan.go @@ -76,6 +76,7 @@ const ( // that the KV budget makes higher concurrency an hours-long stress // exercise, which belongs in --stress, not the default grid. highContextConcurrencyCap = 4 + fixedKVCacheBytesFlag = "--kv-cache-memory-bytes" ) // headroom absorbs chat template and tokenizer drift between requested and @@ -272,6 +273,10 @@ func stampSpec(spec *vllmbench.Spec, request PlanRequest) error { } func sweepProfile(name string, maxModelLen, port, maxNumSeqs int, request PlanRequest) vllmbench.Profile { + omittedEngineFlags := append([]string{}, request.OmitProfileEngineFlags...) + if shouldOmitFixedKVCache(request.Model) { + omittedEngineFlags = append(omittedEngineFlags, fixedKVCacheBytesFlag) + } return vllmbench.Profile{ Name: name, Host: "127.0.0.1", @@ -280,10 +285,15 @@ func sweepProfile(name string, maxModelLen, port, maxNumSeqs int, request PlanRe MaxModelLen: maxModelLen, MaxNumSeqs: maxNumSeqs, Args: append([]string{}, request.ProfileArgs...), - EngineArgs: omittedFlagValues(request.ProfileEngineArgs, request.OmitProfileEngineFlags), + EngineArgs: omittedFlagValues(request.ProfileEngineArgs, omittedEngineFlags), } } +func shouldOmitFixedKVCache(model string) bool { + normalized := strings.ToLower(model) + return strings.Contains(normalized, "qwen") +} + func omittedFlagValues(args, omittedFlags []string) []string { if len(args) == 0 { return nil diff --git a/internal/sweepplan/sweepplan_test.go b/internal/sweepplan/sweepplan_test.go index 6970f27..17ba93e 100644 --- a/internal/sweepplan/sweepplan_test.go +++ b/internal/sweepplan/sweepplan_test.go @@ -130,6 +130,47 @@ func TestPlanProfileArgsCanOmitUnsafeEngineFlags(t *testing.T) { } } +func TestPlanDropsFixedKVCacheForQwenModels(t *testing.T) { + spec, err := Plan(PlanRequest{ + Model: "nvidia/Qwen3.6-27B-NVFP4", + Contexts: []int{8192}, + Concurrency: []int{1}, + ProfileEngineArgs: []string{ + "--kv-cache-memory-bytes", "21474836480", + "--attention-backend", "flashinfer", + }, + }) + if err != nil { + t.Fatal(err) + } + got := strings.Join(spec.Profiles[0].EngineArgs, " ") + if strings.Contains(got, "kv-cache-memory-bytes") || strings.Contains(got, "21474836480") { + t.Fatalf("engine args = %q, want fixed KV cache omitted for Qwen", got) + } + if got != "--attention-backend flashinfer" { + t.Fatalf("engine args = %q, want retained non-KV args", got) + } +} + +func TestPlanKeepsFixedKVCacheForNonQwenModels(t *testing.T) { + spec, err := Plan(PlanRequest{ + Model: "google/gemma-4-26b-it", + Contexts: []int{8192}, + Concurrency: []int{1}, + ProfileEngineArgs: []string{ + "--kv-cache-memory-bytes", "12884901888", + "--attention-backend", "flashinfer", + }, + }) + if err != nil { + t.Fatal(err) + } + got := strings.Join(spec.Profiles[0].EngineArgs, " ") + if !strings.Contains(got, "--kv-cache-memory-bytes 12884901888") { + t.Fatalf("engine args = %q, want fixed KV cache retained for non-Qwen", got) + } +} + func TestStressProfilesAdmitSpotCheckConcurrency(t *testing.T) { spec, err := Plan(PlanRequest{ Model: "example/model", From 31d725d5ff1ef1cb20e95105c1e61194ab3e46bc Mon Sep 17 00:00:00 2001 From: Bob Date: Sun, 5 Jul 2026 14:04:10 +0800 Subject: [PATCH 4/5] fix(sweep): include active 4k by default --- AGENTS.md | 6 +-- README.md | 2 +- docs/2026-07-02-default-inference-sweep.md | 10 +++-- .../2026-07-02-reporting-completeness-plan.md | 4 +- internal/benchcli/benchcli.go | 6 +-- internal/sweepplan/sweepplan.go | 8 ++-- internal/sweepplan/sweepplan_test.go | 38 +++++++++++++++++++ 7 files changed, 59 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index aa5fe44..7436e24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,9 +36,9 @@ remove guardrails to make a run pass. When the user asks for a default context/concurrency sweep, follow `docs/2026-07-02-default-inference-sweep.md`. Use the documented `4k` -max-throughput reference plus the practical `8k`, `16k`, `32k`, `64k`, `128k` -context ladder with concurrency `1`, `4`, `8`, `16`, and `32`, extending by -powers of two only when the hardware can safely take it. +max-throughput reference plus the regular active-context `4k`, `8k`, `16k`, +`32k`, `64k`, `128k` ladder with concurrency `1`, `4`, `8`, `16`, and `32`, +extending by powers of two only when the hardware can safely take it. For repeated benchmark runs of the same model, keep results in one model-level SQLite artifact and render 1 HTML report per model. Do not split retry runs, diff --git a/README.md b/README.md index 391c928..e68c318 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ the grid: ```sh localperf sweep plan \ --model nvidia/diffusiongemma-26B-A4B-it-NVFP4 \ - --contexts 8k,16k,32k --concurrency 1,4,8 \ + --contexts 4k,8k,16k,32k --concurrency 1,4,8 \ --out spec.json ``` diff --git a/docs/2026-07-02-default-inference-sweep.md b/docs/2026-07-02-default-inference-sweep.md index 41598b8..3e5c310 100644 --- a/docs/2026-07-02-default-inference-sweep.md +++ b/docs/2026-07-02-default-inference-sweep.md @@ -34,12 +34,16 @@ Run two benchmark families: - `max-throughput-reference`: minimum `4k` context, intentionally optimized for maximum aggregate token throughput even if the setting is not practically useful. These are capacity points (`context_semantics: "capacity"`). -- `practical-context-sweep`: **active** context points `8k`, `16k`, `32k`, - and `64k`, with concurrency `1`, `4`, `8`, `16`, and `32` where the +- `practical-context-sweep`: **active** context points `4k`, `8k`, `16k`, + `32k`, and `64k`, with concurrency `1`, `4`, `8`, `16`, and `32` where the hardware can safely run them. `128k` is opt-in and capped at `c4`; at that KV budget, higher concurrency is an hours-long stress exercise, not a default grid point. +`4k-reference` is not a substitute for the active `4k` sweep point. A default +sweep should include both: the reference row for toy/max-throughput comparison +and active `4k` prefill/decode rows for the regular context ladder. + The ladder points mean active context, not server capacity: a `32k` point must actually move ~32k tokens per request through the KV cache. Setting `max_model_len=32768` while requesting a ~1k prompt is a capacity experiment, @@ -77,7 +81,7 @@ Default sweeps must be generated with `localperf sweep plan`, for example: ```sh localperf sweep plan --model \ - --contexts 8k,16k,32k,64k --concurrency 1,4,8,16,32 \ + --contexts 4k,8k,16k,32k,64k --concurrency 1,4,8,16,32 \ --out spec.json ``` diff --git a/docs/2026-07-02-reporting-completeness-plan.md b/docs/2026-07-02-reporting-completeness-plan.md index aafe2f8..40b1c73 100644 --- a/docs/2026-07-02-reporting-completeness-plan.md +++ b/docs/2026-07-02-reporting-completeness-plan.md @@ -397,7 +397,7 @@ Pure generator, no I/O in the core: type PlanRequest struct { Model string Engine string // "vllm-managed" first - Contexts []int // active-context ladder, e.g. 8k..128k + Contexts []int // active-context ladder, e.g. 4k..128k Concurrency []int // e.g. 1,4,8,16,32 Repeats int } @@ -421,7 +421,7 @@ validator can never drift. CLI: a `sweep plan` subcommand in `internal/benchcli` (the dispatch that already routes `bench run` and `artifact check`), flags `--model`, -`--engine`, `--contexts 8k,16k,...`, `--concurrency 1,4,...`, +`--engine`, `--contexts 4k,8k,16k,...`, `--concurrency 1,4,...`, `--repeats`, `--out spec.json` (default stdout). Golden-file test: fixed request in, byte-stable spec out. diff --git a/internal/benchcli/benchcli.go b/internal/benchcli/benchcli.go index 4608e7a..29137bc 100644 --- a/internal/benchcli/benchcli.go +++ b/internal/benchcli/benchcli.go @@ -133,7 +133,7 @@ func runSweep(args []string) { func runSweepPlan(args []string) { flags := flag.NewFlagSet("sweep plan", flag.ExitOnError) model := flags.String("model", "", "model identifier to benchmark (required)") - contexts := flags.String("contexts", "8k,16k,32k,64k", "comma-separated active-context ladder (e.g. 8k,16k,32k); 128k and above are capped at c4") + contexts := flags.String("contexts", "4k,8k,16k,32k,64k", "comma-separated active-context ladder (e.g. 4k,8k,16k,32k); 128k and above are capped at c4") concurrency := flags.String("concurrency", "1,4,8,16,32", "comma-separated concurrency levels") repeats := flags.Int("repeats", 1, "repeats per measurement") numPrompts := flags.Int("num-prompts", 0, "fixed prompts per measurement; default scales with concurrency") @@ -256,7 +256,7 @@ func (flag *repeatedStringFlag) String() string { return strings.Join(flag.values, ",") } -// parseTokenList parses values such as "8k,16k,32768" into token counts. +// parseTokenList parses values such as "4k,8k,32768" into token counts. func parseTokenList(value string) ([]int, error) { var values []int for _, part := range strings.Split(value, ",") { @@ -845,7 +845,7 @@ func usageRoot() { localperf artifact render runs/example.sqlite [--output runs/example.html] [--store] localperf artifact merge --into runs/models/model.sqlite src1.sqlite [src2.sqlite ...] localperf artifact rebuild --run-dir runs/example [--into runs/example.sqlite] [--spec spec.json] - localperf sweep plan --model model-id [--contexts 8k,16k,32k,64k] [--concurrency 1,4,8,16,32] [--repeats 1] [--reference] [--stress] [--vllm-command /path/to/vllm] [--gpu-memory-utilization 0.4] [--kv-cache-memory-bytes N] [--trim 64k=8:'reason'] [--out spec.json] + localperf sweep plan --model model-id [--contexts 4k,8k,16k,32k,64k] [--concurrency 1,4,8,16,32] [--repeats 1] [--reference] [--stress] [--vllm-command /path/to/vllm] [--gpu-memory-utilization 0.4] [--kv-cache-memory-bytes N] [--trim 64k=8:'reason'] [--out spec.json] localperf view runs/model.sqlite [runs/other.sqlite ...] [--addr 127.0.0.1:0] [--open]`) } diff --git a/internal/sweepplan/sweepplan.go b/internal/sweepplan/sweepplan.go index c8f83fa..3e05a12 100644 --- a/internal/sweepplan/sweepplan.go +++ b/internal/sweepplan/sweepplan.go @@ -19,7 +19,7 @@ import ( type PlanRequest struct { // Model is the model identifier to benchmark (required). Model string `json:"model"` - // Contexts is the active-context ladder in tokens, e.g. 8192..131072. + // Contexts is the active-context ladder in tokens, e.g. 4096..131072. Contexts []int `json:"contexts,omitempty"` // Concurrency levels per point, e.g. 1,4,8,16,32. Concurrency []int `json:"concurrency,omitempty"` @@ -235,9 +235,11 @@ func applyRuntimeIntent(spec *vllmbench.Spec, request PlanRequest) { if request.GPUMemoryUtilization > 0 { spec.Profiles[index].GPUMemoryUtilization = request.GPUMemoryUtilization } - if request.KVCacheMemoryBytes > 0 { + // Qwen runtimes reject a fixed KV cache size; the same rule that + // strips the inherited engine flag applies to the intent flag. + if request.KVCacheMemoryBytes > 0 && !shouldOmitFixedKVCache(request.Model) { spec.Profiles[index].Args = append(spec.Profiles[index].Args, - "--kv-cache-memory-bytes", strconv.FormatInt(request.KVCacheMemoryBytes, 10)) + fixedKVCacheBytesFlag, strconv.FormatInt(request.KVCacheMemoryBytes, 10)) } } } diff --git a/internal/sweepplan/sweepplan_test.go b/internal/sweepplan/sweepplan_test.go index 17ba93e..df3ffb8 100644 --- a/internal/sweepplan/sweepplan_test.go +++ b/internal/sweepplan/sweepplan_test.go @@ -98,6 +98,44 @@ func TestPlanGolden(t *testing.T) { } } +func TestPlanKeeps4KReferenceSeparateFromActive4K(t *testing.T) { + spec, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{4096}, + Concurrency: []int{1, 4}, + IncludeReference: true, + }) + if err != nil { + t.Fatal(err) + } + profiles := map[string]vllmbench.Profile{} + for _, profile := range spec.Profiles { + profiles[profile.Name] = profile + } + if _, ok := profiles["4k-reference"]; !ok { + t.Fatal("missing 4k-reference profile") + } + if _, ok := profiles["4k"]; !ok { + t.Fatal("missing active 4k profile") + } + workloads := map[string]vllmbench.Workload{} + for _, workload := range spec.Workloads { + workloads[workload.Name] = workload + } + reference := workloads["max-throughput-reference"] + if reference.ContextSemantics != vllmbench.ContextSemanticsCapacity || reference.Profiles[0] != "4k-reference" { + t.Fatalf("reference workload = %+v, want capacity workload on 4k-reference", reference) + } + prefill := workloads["prefill-4k"] + if prefill.ContextSemantics != vllmbench.ContextSemanticsActive || prefill.Phase != "prefill" || prefill.Profiles[0] != "4k" { + t.Fatalf("prefill-4k = %+v, want active prefill workload on 4k", prefill) + } + decode := workloads["decode-4k"] + if decode.ContextSemantics != vllmbench.ContextSemanticsActive || decode.Phase != "decode" || decode.Profiles[0] != "4k" { + t.Fatalf("decode-4k = %+v, want active decode workload on 4k", decode) + } +} + func TestPlanProfileArgsCanOmitUnsafeEngineFlags(t *testing.T) { spec, err := Plan(PlanRequest{ Model: "example/model", From f5e76a22545e9b0fb2a8421cd10a46d97f971f5d Mon Sep 17 00:00:00 2001 From: Bob Date: Sun, 5 Jul 2026 19:26:31 +0800 Subject: [PATCH 5/5] fix(sweep,artifact): rebuild from normalized spec; merge cleans empty destinations; boolean flag omission Codex review round 1: artifact rebuild reconstructs from the run's normalized spec (--spec is provenance-only); a failed merge into a zero-byte destination removes it like a fresh file; omitting a boolean engine flag no longer swallows the following flag. --- internal/artifact/merge.go | 7 +++-- internal/benchcli/benchcli.go | 9 +++---- internal/sweepplan/sweepplan.go | 4 ++- internal/sweepplan/sweepplan_test.go | 11 ++++++++ internal/vllmbench/model_artifact_test.go | 32 +++++++++++++++++++++++ 5 files changed, 55 insertions(+), 8 deletions(-) diff --git a/internal/artifact/merge.go b/internal/artifact/merge.go index 8bc9313..4512ddd 100644 --- a/internal/artifact/merge.go +++ b/internal/artifact/merge.go @@ -29,8 +29,11 @@ func Merge(dstPath string, srcPaths []string) (MergeSummary, error) { return summary, fmt.Errorf("merge source %s: %w", srcPath, err) } } - _, statErr := os.Stat(dstPath) - createdFresh := statErr != nil + info, statErr := os.Stat(dstPath) + // An empty destination file is initialized by CreateOrAppend; on + // failure it must be removed like a missing one, not left behind as a + // schema-only artifact. + createdFresh := statErr != nil || (info != nil && info.Size() == 0) db, err := CreateOrAppend(dstPath, Schema) if err != nil { return summary, err diff --git a/internal/benchcli/benchcli.go b/internal/benchcli/benchcli.go index 29137bc..a471a8b 100644 --- a/internal/benchcli/benchcli.go +++ b/internal/benchcli/benchcli.go @@ -62,11 +62,10 @@ func runArtifactRebuild(args []string) { } func rebuildArtifactFromRunDir(runDir, artifactPath, originalSpecPath string) error { - specLoadPath := originalSpecPath - if strings.TrimSpace(specLoadPath) == "" { - specLoadPath = filepath.Join(runDir, "spec.normalized.json") - } - spec, err := vllmbench.LoadSpec(specLoadPath) + // Reconstruction always follows the normalized spec the run actually + // executed (filters applied); --spec supplies only the original file + // for provenance. + spec, err := vllmbench.LoadSpec(filepath.Join(runDir, "spec.normalized.json")) if err != nil { return err } diff --git a/internal/sweepplan/sweepplan.go b/internal/sweepplan/sweepplan.go index 3e05a12..4fbc7e4 100644 --- a/internal/sweepplan/sweepplan.go +++ b/internal/sweepplan/sweepplan.go @@ -313,7 +313,9 @@ func omittedFlagValues(args, omittedFlags []string) []string { for index := 0; index < len(args); index++ { arg := args[index] if _, ok := omitted[arg]; ok { - if index+1 < len(args) { + // Skip a separate value ("--flag value") but never a following + // flag: boolean flags carry no value token. + if index+1 < len(args) && !strings.HasPrefix(args[index+1], "--") { index++ } continue diff --git a/internal/sweepplan/sweepplan_test.go b/internal/sweepplan/sweepplan_test.go index df3ffb8..11faedf 100644 --- a/internal/sweepplan/sweepplan_test.go +++ b/internal/sweepplan/sweepplan_test.go @@ -315,3 +315,14 @@ func TestDefaultGridUsesShortDecodeAndScaledPrompts(t *testing.T) { t.Fatalf("resolved prompts = %v, want map[1:8 4:8 16:32]", prompts) } } + +func TestOmittedFlagValuesKeepsFollowingFlags(t *testing.T) { + got := omittedFlagValues( + []string{"--disable-log-requests", "--attention-backend", "flashinfer", "--kv-cache-memory-bytes", "123"}, + []string{"--disable-log-requests", "--kv-cache-memory-bytes"}, + ) + want := []string{"--attention-backend", "flashinfer"} + if len(got) != len(want) || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("omitted args = %v, want %v: a boolean flag must not swallow the next flag", got, want) + } +} diff --git a/internal/vllmbench/model_artifact_test.go b/internal/vllmbench/model_artifact_test.go index d5c19a3..c63a6fd 100644 --- a/internal/vllmbench/model_artifact_test.go +++ b/internal/vllmbench/model_artifact_test.go @@ -254,3 +254,35 @@ func TestWriteSQLiteArtifactRemovesFreshFileOnFailedWrite(t *testing.T) { t.Fatal("failed first write left a schema-only artifact behind") } } + +func TestMergeCleansUpEmptyDestinationOnCollision(t *testing.T) { + dir := t.TempDir() + makeSingle := func(parent string) string { + spec := testSpec() + spec.OutputDir = dir + path := filepath.Join(dir, parent+".sqlite") + if _, err := Execute(context.Background(), spec, RunOptions{ + DryRun: true, + RunDir: filepath.Join(dir, parent, "run"), + ArtifactPath: path, + }); err != nil { + t.Fatal(err) + } + return path + } + first := makeSingle("c") + second := makeSingle("d") + dst := filepath.Join(dir, "empty-model.sqlite") + // A pre-existing zero-byte destination is initialized by the merge; a + // later collision must remove it like a freshly created file, not + // leave a partially merged artifact behind. + if err := os.WriteFile(dst, nil, 0o644); err != nil { + t.Fatal(err) + } + if _, err := artifact.Merge(dst, []string{first, second}); err == nil { + t.Fatal("expected provenance collision failure") + } + if _, err := os.Stat(dst); !os.IsNotExist(err) { + t.Fatalf("destination stat = %v, want removed after failed merge into an empty file", err) + } +}