Benchmark results database

.gitignore

@@ -6,6 +6,7 @@ obj/
 # Python
 __pycache__
 *.pyc
+uv.lock
 .venv/
 
 # Temporary / artifacts

Clip.Benchmarks/FastConfig.cs

@@ -9,51 +9,70 @@
 namespace Clip.Benchmarks;
 
 //
-// Modes (env vars):
-//   BENCH_MODE=fast (default) — InProcess, 1 warmup, 3 × 100 ms iterations (~5 min)
-//   BENCH_MODE=full — InProcess, 2 warmups, 5 × 250 ms iterations (~10-15 min)
-//   BENCH_CONFIG=asm — out-of-process + DisassemblyDiagnoser, artifacts in tmp/BenchmarkDotNet.AsmArtifacts/
+// BENCH_MODE env var:
+//   fast (default) — InProcess, 2 warmups, 5 × 200 ms iterations
+//   full — out-of-process, 3 warmups, 50 × 1000 ms iterations
+//   asm — out-of-process + DisassemblyDiagnoser, artifacts in tmp/BenchmarkDotNet.AsmArtifacts/
 //
 
 internal sealed class FastConfig : ManualConfig
 {
     public FastConfig()
     {
-        var asm = string.Equals(
-            Environment.GetEnvironmentVariable("BENCH_CONFIG"),
-            "asm", StringComparison.OrdinalIgnoreCase);
+        var mode = (Environment.GetEnvironmentVariable("BENCH_MODE") ?? "fast")
+            .ToLowerInvariant();
 
-        ArtifactsPath = Path.Combine("tmp", asm
+        ArtifactsPath = Path.Combine("tmp", mode == "asm"
             ? "BenchmarkDotNet.AsmArtifacts"
             : "BenchmarkDotNet.Artifacts");
 
-        if (asm)
+        switch (mode)
         {
-            AddJob(Job.Default.WithWarmupCount(1).WithIterationCount(2));
-            AddDiagnoser(new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig(3)));
-        }
-        else
-        {
-            var full = string.Equals(
-                Environment.GetEnvironmentVariable("BENCH_MODE"),
-                "full", StringComparison.OrdinalIgnoreCase);
-
-            AddJob(Job.Default
-                .WithToolchain(InProcessEmitToolchain.Instance)
-                .WithWarmupCount(full
-                    ? 2
-                    : 1)
-                .WithIterationCount(full
-                    ? 5
-                    : 3)
-                .WithIterationTime(TimeInterval.FromMilliseconds(full
-                    ? 250
-                    : 100)));
+            case "asm": ConfigureAsm(); break;
+            case "full": ConfigureFull(); break;
+            default: ConfigureFast(); break;
         }
 
         AddExporter(MarkdownExporter.GitHub);
         AddDiagnoser(MemoryDiagnoser.Default);
         AddColumn(CategoriesColumn.Default);
         AddLogicalGroupRules(BenchmarkLogicalGroupRule.ByCategory);
     }
+
+    //
+    // Fast — quick iteration during development
+    //
+
+    private void ConfigureFast()
+    {
+        AddJob(Job.Default
+            .WithToolchain(InProcessEmitToolchain.Instance)
+            .WithWarmupCount(2)
+            .WithIterationCount(5)
+            .WithIterationTime(TimeInterval.FromMilliseconds(200)));
+    }
+
+    //
+    // Full — publication-quality, out-of-process
+    //
+
+    private void ConfigureFull()
+    {
+        AddJob(Job.Default
+            .WithWarmupCount(3)
+            .WithIterationCount(50)
+            .WithIterationTime(TimeInterval.FromMilliseconds(1000)));
+    }
+
+    //
+    // Asm — JIT disassembly
+    //
+
+    private void ConfigureAsm()
+    {
+        AddJob(Job.Default
+            .WithWarmupCount(1)
+            .WithIterationCount(2));
+        AddDiagnoser(new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig(3)));
+    }
 }

Makefile

@@ -1,6 +1,6 @@
 .PHONY: help build test check
 .PHONY: fmt fmt-cs fmt-py
-.PHONY: bench bench-full bench-asm archive-bench
+.PHONY: bench bench-full bench-clip bench-clip-full bench-asm bench-update archive-bench
 .PHONY: charts docs pdf usage
 .PHONY: demo demo-console demo-json
 .PHONY: pkg
@@ -33,26 +33,41 @@ format-cs:
 
 ## Format Python scripts
 format-py:
-	.venv/bin/black scripts/
+	uv run ruff format scripts/
 
-## Run fast benchmarks (~40 min)
+## Run fast benchmarks — all loggers (~40 min, 5 data points)
 bench:
 	BENCH_MODE=fast dotnet run -c Release --project $(BENCH_PROJECT) -- --filter '*ConsoleBenchmarks*' '*JsonBenchmarks*' '*FilteredBenchmarks*'
 	@rm -f tmp/BenchmarkDotNet.Artifacts/*.log
-	@$(MAKE) archive-bench
-	@$(MAKE) docs
+	@$(MAKE) bench-update
 
-## Run full benchmarks (~90 min, publication-quality)
+## Run full benchmarks — all loggers (~1h45m, 50 data points)
 bench-full:
 	BENCH_MODE=full dotnet run -c Release --project $(BENCH_PROJECT) -- --filter '*ConsoleBenchmarks*' '*JsonBenchmarks*' '*FilteredBenchmarks*'
 	@rm -f tmp/BenchmarkDotNet.Artifacts/*.log
+	@$(MAKE) bench-update
+
+## Run fast benchmarks — Clip only (~12 min, 5 data points)
+bench-clip:
+	BENCH_MODE=fast dotnet run -c Release --project $(BENCH_PROJECT) -- --filter '*_Clip' '*_ClipZero' '*_ClipMEL'
+	@rm -f tmp/BenchmarkDotNet.Artifacts/*.log
+	@$(MAKE) bench-update
+
+## Run full benchmarks — Clip only (~30 min, 50 data points)
+bench-clip-full:
+	BENCH_MODE=full dotnet run -c Release --project $(BENCH_PROJECT) -- --filter '*_Clip' '*_ClipZero' '*_ClipMEL'
+	@rm -f tmp/BenchmarkDotNet.Artifacts/*.log
+	@$(MAKE) bench-update
+
+## Import results into DB and archive raw artifacts
+bench-update:
+	@uv run python3 scripts/benchdb.py import
 	@$(MAKE) archive-bench
-	@$(MAKE) docs
 
 ## Dump JIT assembly for Clip hot paths
 bench-asm:
-	BENCH_CONFIG=asm dotnet run -c Release --project $(BENCH_PROJECT) -- --filter '*FiveFields_Clip*'
-	BENCH_CONFIG=asm dotnet run -c Release --project $(BENCH_PROJECT) -- --filter '*WithContext_Clip*'
+	BENCH_MODE=asm dotnet run -c Release --project $(BENCH_PROJECT) -- --filter '*FiveFields_Clip*'
+	BENCH_MODE=asm dotnet run -c Release --project $(BENCH_PROJECT) -- --filter '*WithContext_Clip*'
 	@rm -f tmp/BenchmarkDotNet.AsmArtifacts/*.log
 
 ## Archive benchmark results to tmp/bench-history/
@@ -65,22 +80,22 @@ archive-bench:
 	cp tmp/BenchmarkDotNet.Artifacts/results/*.md tmp/BenchmarkDotNet.Artifacts/results/*.csv "$$dir/" 2>/dev/null; \
 	echo "Archived to $$dir"
 
-## Generate bar charts from BDN artifacts
+## Generate bar charts from benchmark database
 charts:
-	.venv/bin/python3 scripts/chart.py
+	uv run python3 scripts/chart.py
 
 ## Generate docs/COMPARE.md (depends on charts)
 docs: charts
-	.venv/bin/python3 scripts/compare.py
+	uv run python3 scripts/compare.py
 
 ## Generate PDFs from docs
 pdf: docs
-	.venv/bin/python3 scripts/pdf.py
+	uv run python3 scripts/pdf.py
 
 ## Generate docs/USAGE.md (code + output for all loggers)
 usage:
 	dotnet run -c Release --project Clip.ComparisonDemo > tmp/raw/usage.txt
-	.venv/bin/python3 scripts/usage.py tmp/raw/usage.txt
+	uv run python3 scripts/usage.py tmp/raw/usage.txt
 
 ## Run the demo app (console output)
 demo: demo-console
@@ -104,10 +119,9 @@ pkg:
 	@echo ""
 	@ls -lh pkg/*.nupkg
 
-## Create venv and install Python dependencies
+## Install Python dependencies
 setup:
-	python3 -m venv .venv
-	.venv/bin/pip install -r scripts/requirements.txt
+	uv sync
 
 ## Remove build artifacts and benchmark results
 clean: