feat(sweep): profile args, qwen KV rule, 4k active default, artifact rebuild

AGENTS.md

@@ -36,9 +36,9 @@ remove guardrails to make a run pass.
 
 When the user asks for a default context/concurrency sweep, follow
 `docs/2026-07-02-default-inference-sweep.md`. Use the documented `4k`
-max-throughput reference plus the practical `8k`, `16k`, `32k`, `64k`, `128k`
-context ladder with concurrency `1`, `4`, `8`, `16`, and `32`, extending by
-powers of two only when the hardware can safely take it.
+max-throughput reference plus the regular active-context `4k`, `8k`, `16k`,
+`32k`, `64k`, `128k` ladder with concurrency `1`, `4`, `8`, `16`, and `32`,
+extending by powers of two only when the hardware can safely take it.
 
 For repeated benchmark runs of the same model, keep results in one model-level
 SQLite artifact and render 1 HTML report per model. Do not split retry runs,

README.md

@@ -48,7 +48,7 @@ the grid:
 ```sh
 localperf sweep plan \
   --model nvidia/diffusiongemma-26B-A4B-it-NVFP4 \
-  --contexts 8k,16k,32k --concurrency 1,4,8 \
+  --contexts 4k,8k,16k,32k --concurrency 1,4,8 \
   --out spec.json
 ```
 

docs/2026-07-02-default-inference-sweep.md

@@ -34,12 +34,16 @@ Run two benchmark families:
 - `max-throughput-reference`: minimum `4k` context, intentionally optimized for
   maximum aggregate token throughput even if the setting is not practically
   useful. These are capacity points (`context_semantics: "capacity"`).
-- `practical-context-sweep`: **active** context points `8k`, `16k`, `32k`,
-  and `64k`, with concurrency `1`, `4`, `8`, `16`, and `32` where the
+- `practical-context-sweep`: **active** context points `4k`, `8k`, `16k`,
+  `32k`, and `64k`, with concurrency `1`, `4`, `8`, `16`, and `32` where the
   hardware can safely run them. `128k` is opt-in and capped at `c4`; at that
   KV budget, higher concurrency is an hours-long stress exercise, not a
   default grid point.
 
+`4k-reference` is not a substitute for the active `4k` sweep point. A default
+sweep should include both: the reference row for toy/max-throughput comparison
+and active `4k` prefill/decode rows for the regular context ladder.
+
 The ladder points mean active context, not server capacity: a `32k` point must
 actually move ~32k tokens per request through the KV cache. Setting
 `max_model_len=32768` while requesting a ~1k prompt is a capacity experiment,
@@ -77,7 +81,7 @@ Default sweeps must be generated with `localperf sweep plan`, for example:
 
 ```sh
 localperf sweep plan --model <model-id> \
-  --contexts 8k,16k,32k,64k --concurrency 1,4,8,16,32 \
+  --contexts 4k,8k,16k,32k,64k --concurrency 1,4,8,16,32 \
   --out spec.json
 ```
 

docs/2026-07-02-reporting-completeness-plan.md

@@ -397,7 +397,7 @@ Pure generator, no I/O in the core:
 type PlanRequest struct {
     Model       string
     Engine      string // "vllm-managed" first
-    Contexts    []int  // active-context ladder, e.g. 8k..128k
+    Contexts    []int  // active-context ladder, e.g. 4k..128k
     Concurrency []int  // e.g. 1,4,8,16,32
     Repeats     int
 }
@@ -421,7 +421,7 @@ validator can never drift.
 
 CLI: a `sweep plan` subcommand in `internal/benchcli` (the dispatch that
 already routes `bench run` and `artifact check`), flags `--model`,
-`--engine`, `--contexts 8k,16k,...`, `--concurrency 1,4,...`,
+`--engine`, `--contexts 4k,8k,16k,...`, `--concurrency 1,4,...`,
 `--repeats`, `--out spec.json` (default stdout). Golden-file test: fixed
 request in, byte-stable spec out.
 

internal/artifact/merge.go

@@ -29,8 +29,11 @@ func Merge(dstPath string, srcPaths []string) (MergeSummary, error) {
 			return summary, fmt.Errorf("merge source %s: %w", srcPath, err)
 		}
 	}
-	_, statErr := os.Stat(dstPath)
-	createdFresh := statErr != nil
+	info, statErr := os.Stat(dstPath)
+	// An empty destination file is initialized by CreateOrAppend; on
+	// failure it must be removed like a missing one, not left behind as a
+	// schema-only artifact.
+	createdFresh := statErr != nil || (info != nil && info.Size() == 0)
 	db, err := CreateOrAppend(dstPath, Schema)
 	if err != nil {
 		return summary, err

internal/artifact/sqlite.go

@@ -47,9 +47,16 @@ func Create(path, schema string) (*sql.DB, error) {
 // The existing file must be a valid artifact of the supported format;
 // anything else is an error rather than something to silently overwrite.
 func CreateOrAppend(path, schema string) (*sql.DB, error) {
-	if _, err := os.Stat(path); err != nil {
+	info, err := os.Stat(path)
+	if err != nil {
 		return Create(path, schema)
 	}
+	if info.IsDir() {
+		return nil, fmt.Errorf("%s is a directory", path)
+	}
+	if info.Size() == 0 {
+		return openWithSchema(path, schema)
+	}
 	db, err := OpenWritable(path)
 	if err != nil {
 		return nil, err

internal/benchcli/benchcli.go

@@ -34,9 +34,63 @@ var rootHandlers = commandHandlers{
 }
 
 var artifactHandlers = commandHandlers{
-	"check":  runArtifactCheck,
-	"render": runArtifactRender,
-	"merge":  runArtifactMerge,
+	"check":   runArtifactCheck,
+	"render":  runArtifactRender,
+	"merge":   runArtifactMerge,
+	"rebuild": runArtifactRebuild,
+}
+
+// runArtifactRebuild reconstructs a SQLite artifact from a run directory's
+// raw data (events, results, summary.json) — the recovery path when a run
+// finished without an artifact or the artifact was lost.
+func runArtifactRebuild(args []string) {
+	flags := flag.NewFlagSet("artifact rebuild", flag.ExitOnError)
+	runDir := flags.String("run-dir", "", "run directory with events.jsonl, results, and summary.json")
+	into := flags.String("into", "", "SQLite artifact path to rebuild/append; defaults to <run-dir>.sqlite")
+	specPath := flags.String("spec", "", "optional original spec path for provenance; defaults to <run-dir>/spec.normalized.json")
+	_ = flags.Parse(args)
+	if strings.TrimSpace(*runDir) == "" {
+		fmt.Fprintln(os.Stderr, "missing --run-dir")
+		os.Exit(2)
+	}
+	artifactPath := artifact.Path(*runDir, *into)
+	if err := rebuildArtifactFromRunDir(*runDir, artifactPath, *specPath); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	fmt.Printf("artifact: %s\n", artifactPath)
+}
+
+func rebuildArtifactFromRunDir(runDir, artifactPath, originalSpecPath string) error {
+	// Reconstruction always follows the normalized spec the run actually
+	// executed (filters applied); --spec supplies only the original file
+	// for provenance.
+	spec, err := vllmbench.LoadSpec(filepath.Join(runDir, "spec.normalized.json"))
+	if err != nil {
+		return err
+	}
+	summary, err := loadRunSummary(runDir)
+	if err != nil {
+		return err
+	}
+	summary.RunDir = runDir
+	summary.ArtifactPath = artifactPath
+	if strings.TrimSpace(summary.SpecPath) == "" {
+		summary.SpecPath = filepath.Join(runDir, "spec.normalized.json")
+	}
+	return vllmbench.WriteSQLiteArtifact(runDir, artifactPath, spec, summary, vllmbench.BuildPlan(spec, runDir), originalSpecPath)
+}
+
+func loadRunSummary(runDir string) (vllmbench.RunSummary, error) {
+	data, err := os.ReadFile(filepath.Join(runDir, "summary.json"))
+	if err != nil {
+		return vllmbench.RunSummary{}, err
+	}
+	var summary vllmbench.RunSummary
+	if err := json.Unmarshal(data, &summary); err != nil {
+		return vllmbench.RunSummary{}, err
+	}
+	return summary, nil
 }
 
 // runArtifactMerge combines run artifacts into one model-level SQLite file;
@@ -78,7 +132,7 @@ func runSweep(args []string) {
 func runSweepPlan(args []string) {
 	flags := flag.NewFlagSet("sweep plan", flag.ExitOnError)
 	model := flags.String("model", "", "model identifier to benchmark (required)")
-	contexts := flags.String("contexts", "8k,16k,32k,64k", "comma-separated active-context ladder (e.g. 8k,16k,32k); 128k and above are capped at c4")
+	contexts := flags.String("contexts", "4k,8k,16k,32k,64k", "comma-separated active-context ladder (e.g. 4k,8k,16k,32k); 128k and above are capped at c4")
 	concurrency := flags.String("concurrency", "1,4,8,16,32", "comma-separated concurrency levels")
 	repeats := flags.Int("repeats", 1, "repeats per measurement")
 	numPrompts := flags.Int("num-prompts", 0, "fixed prompts per measurement; default scales with concurrency")
@@ -92,6 +146,12 @@ func runSweepPlan(args []string) {
 	var trims trimFlags
 	flags.Var(&trims, "trim", "cap a context's ladder with a reason, e.g. 64k=8:'12 GiB KV budget'; repeatable")
 	out := flags.String("out", "", "output spec path (default stdout)")
+	var profileArgs repeatedStringFlag
+	var profileEngineArgs repeatedStringFlag
+	var omitProfileEngineFlags repeatedStringFlag
+	flags.Var(&profileArgs, "profile-arg", "extra generated profile arg; repeat once per arg")
+	flags.Var(&profileEngineArgs, "profile-engine-arg", "extra generated profile engine arg; repeat once per arg")
+	flags.Var(&omitProfileEngineFlags, "omit-profile-engine-flag", "engine flag to remove from generated profile engine args; repeatable")
 	_ = flags.Parse(args)
 	contextValues, err := parseTokenList(*contexts)
 	if err != nil {
@@ -104,19 +164,22 @@ func runSweepPlan(args []string) {
 		os.Exit(2)
 	}
 	spec, err := sweepplan.Plan(sweepplan.PlanRequest{
-		Model:                *model,
-		Contexts:             contextValues,
-		Concurrency:          concurrencyValues,
-		Repeats:              *repeats,
-		NumPrompts:           *numPrompts,
-		PromptsPerUser:       *promptsPerUser,
-		IncludeReference:     *reference,
-		IncludeStress:        *stress,
-		MinMemAvailableGiB:   *memFloor,
-		VLLMCommand:          *vllmCommand,
-		GPUMemoryUtilization: *gpuMemUtil,
-		KVCacheMemoryBytes:   *kvCacheBytes,
-		Trims:                trims.values,
+		Model:                  *model,
+		Contexts:               contextValues,
+		Concurrency:            concurrencyValues,
+		Repeats:                *repeats,
+		NumPrompts:             *numPrompts,
+		PromptsPerUser:         *promptsPerUser,
+		IncludeReference:       *reference,
+		IncludeStress:          *stress,
+		ProfileArgs:            profileArgs.values,
+		ProfileEngineArgs:      profileEngineArgs.values,
+		OmitProfileEngineFlags: omitProfileEngineFlags.values,
+		MinMemAvailableGiB:     *memFloor,
+		VLLMCommand:            *vllmCommand,
+		GPUMemoryUtilization:   *gpuMemUtil,
+		KVCacheMemoryBytes:     *kvCacheBytes,
+		Trims:                  trims.values,
 	})
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
@@ -179,7 +242,20 @@ func (flags *trimFlags) Set(value string) error {
 	return nil
 }
 
-// parseTokenList parses values such as "8k,16k,32768" into token counts.
+type repeatedStringFlag struct {
+	values []string
+}
+
+func (flag *repeatedStringFlag) Set(value string) error {
+	flag.values = append(flag.values, value)
+	return nil
+}
+
+func (flag *repeatedStringFlag) String() string {
+	return strings.Join(flag.values, ",")
+}
+
+// parseTokenList parses values such as "4k,8k,32768" into token counts.
 func parseTokenList(value string) ([]int, error) {
 	var values []int
 	for _, part := range strings.Split(value, ",") {
@@ -767,7 +843,8 @@ func usageRoot() {
   localperf artifact check runs/example.sqlite
   localperf artifact render runs/example.sqlite [--output runs/example.html] [--store]
   localperf artifact merge --into runs/models/model.sqlite src1.sqlite [src2.sqlite ...]
-  localperf sweep plan   --model model-id [--contexts 8k,16k,32k,64k] [--concurrency 1,4,8,16,32] [--repeats 1] [--reference] [--stress] [--vllm-command /path/to/vllm] [--gpu-memory-utilization 0.4] [--kv-cache-memory-bytes N] [--trim 64k=8:'reason'] [--out spec.json]
+  localperf artifact rebuild --run-dir runs/example [--into runs/example.sqlite] [--spec spec.json]
+  localperf sweep plan   --model model-id [--contexts 4k,8k,16k,32k,64k] [--concurrency 1,4,8,16,32] [--repeats 1] [--reference] [--stress] [--vllm-command /path/to/vllm] [--gpu-memory-utilization 0.4] [--kv-cache-memory-bytes N] [--trim 64k=8:'reason'] [--out spec.json]
   localperf view runs/model.sqlite [runs/other.sqlite ...] [--addr 127.0.0.1:0] [--open]`)
 }
 

internal/sweepplan/sweepplan.go

@@ -19,7 +19,7 @@ import (
 type PlanRequest struct {
 	// Model is the model identifier to benchmark (required).
 	Model string `json:"model"`
-	// Contexts is the active-context ladder in tokens, e.g. 8192..131072.
+	// Contexts is the active-context ladder in tokens, e.g. 4096..131072.
 	Contexts []int `json:"contexts,omitempty"`
 	// Concurrency levels per point, e.g. 1,4,8,16,32.
 	Concurrency []int `json:"concurrency,omitempty"`
@@ -37,6 +37,14 @@ type PlanRequest struct {
 	// 32k c4 and 64k c1/c4 when those contexts are in the ladder) and the
 	// 128k points at c1/c4.
 	IncludeStress bool `json:"include_stress,omitempty"`
+	// ProfileArgs are appended to every generated profile as serve CLI args.
+	ProfileArgs []string `json:"profile_args,omitempty"`
+	// ProfileEngineArgs are appended to every generated profile as engine
+	// CLI args. Use OmitProfileEngineFlags to remove unsafe inherited flags.
+	ProfileEngineArgs []string `json:"profile_engine_args,omitempty"`
+	// OmitProfileEngineFlags removes matching "--flag value" and
+	// "--flag=value" entries from ProfileEngineArgs before emitting profiles.
+	OmitProfileEngineFlags []string `json:"omit_profile_engine_flags,omitempty"`
 	// MinMemAvailableGiB is the safety memory floor; defaults to 40.
 	MinMemAvailableGiB float64 `json:"min_mem_available_gib,omitempty"`
 	// BasePort is the first server port; defaults to 8100.
@@ -68,6 +76,7 @@ const (
 	// that the KV budget makes higher concurrency an hours-long stress
 	// exercise, which belongs in --stress, not the default grid.
 	highContextConcurrencyCap = 4
+	fixedKVCacheBytesFlag     = "--kv-cache-memory-bytes"
 )
 
 // headroom absorbs chat template and tokenizer drift between requested and
@@ -168,7 +177,7 @@ func Plan(request PlanRequest) (vllmbench.Spec, error) {
 	maxNumSeqs := maxConcurrencyOf(request.Concurrency)
 	port := request.BasePort
 	if request.IncludeReference {
-		spec.Profiles = append(spec.Profiles, sweepProfile("4k-reference", referenceContext, port, maxNumSeqs))
+		spec.Profiles = append(spec.Profiles, sweepProfile("4k-reference", referenceContext, port, maxNumSeqs, request))
 		port++
 		spec.Workloads = append(spec.Workloads, sweepWorkload(
 			"max-throughput-reference", "decode", "4k-reference",
@@ -181,7 +190,7 @@ func Plan(request PlanRequest) (vllmbench.Spec, error) {
 	}
 	for _, context := range contexts {
 		label := vllmbench.TokenCountLabel(context)
-		spec.Profiles = append(spec.Profiles, sweepProfile(label, context, port, contextMaxSeqs(context, request)))
+		spec.Profiles = append(spec.Profiles, sweepProfile(label, context, port, contextMaxSeqs(context, request), request))
 		port++
 		prefillInput, prefillOutput := PrefillShape(context)
 		spec.Workloads = append(spec.Workloads, sweepWorkload(
@@ -226,9 +235,11 @@ func applyRuntimeIntent(spec *vllmbench.Spec, request PlanRequest) {
 		if request.GPUMemoryUtilization > 0 {
 			spec.Profiles[index].GPUMemoryUtilization = request.GPUMemoryUtilization
 		}
-		if request.KVCacheMemoryBytes > 0 {
+		// Qwen runtimes reject a fixed KV cache size; the same rule that
+		// strips the inherited engine flag applies to the intent flag.
+		if request.KVCacheMemoryBytes > 0 && !shouldOmitFixedKVCache(request.Model) {
 			spec.Profiles[index].Args = append(spec.Profiles[index].Args,
-				"--kv-cache-memory-bytes", strconv.FormatInt(request.KVCacheMemoryBytes, 10))
+				fixedKVCacheBytesFlag, strconv.FormatInt(request.KVCacheMemoryBytes, 10))
 		}
 	}
 }
@@ -263,15 +274,64 @@ func stampSpec(spec *vllmbench.Spec, request PlanRequest) error {
 	return nil
 }
 
-func sweepProfile(name string, maxModelLen, port, maxNumSeqs int) vllmbench.Profile {
+func sweepProfile(name string, maxModelLen, port, maxNumSeqs int, request PlanRequest) vllmbench.Profile {
+	omittedEngineFlags := append([]string{}, request.OmitProfileEngineFlags...)
+	if shouldOmitFixedKVCache(request.Model) {
+		omittedEngineFlags = append(omittedEngineFlags, fixedKVCacheBytesFlag)
+	}
 	return vllmbench.Profile{
 		Name:        name,
 		Host:        "127.0.0.1",
 		Port:        port,
 		Managed:     true,
 		MaxModelLen: maxModelLen,
 		MaxNumSeqs:  maxNumSeqs,
+		Args:        append([]string{}, request.ProfileArgs...),
+		EngineArgs:  omittedFlagValues(request.ProfileEngineArgs, omittedEngineFlags),
+	}
+}
+
+func shouldOmitFixedKVCache(model string) bool {
+	normalized := strings.ToLower(model)
+	return strings.Contains(normalized, "qwen")
+}
+
+func omittedFlagValues(args, omittedFlags []string) []string {
+	if len(args) == 0 {
+		return nil
+	}
+	if len(omittedFlags) == 0 {
+		return append([]string{}, args...)
+	}
+	omitted := map[string]struct{}{}
+	for _, flag := range omittedFlags {
+		if strings.TrimSpace(flag) != "" {
+			omitted[flag] = struct{}{}
+		}
+	}
+	filtered := make([]string, 0, len(args))
+	for index := 0; index < len(args); index++ {
+		arg := args[index]
+		if _, ok := omitted[arg]; ok {
+			// Skip a separate value ("--flag value") but never a following
+			// flag: boolean flags carry no value token.
+			if index+1 < len(args) && !strings.HasPrefix(args[index+1], "--") {
+				index++
+			}
+			continue
+		}
+		skip := false
+		for flag := range omitted {
+			if strings.HasPrefix(arg, flag+"=") {
+				skip = true
+				break
+			}
+		}
+		if !skip {
+			filtered = append(filtered, arg)
+		}
 	}
+	return filtered
 }
 
 // stressWorkloads adds long-output decode spot checks for ladder contexts