feat(matrix): benchmark multiple wallarm builds side-by-side

gateways/wallarm/README.md

@@ -22,8 +22,30 @@ fails loudly with "`WALLARM_IMAGE must be set`" — this is intentional;
 an earlier iteration pinned the public `0.2.0` image as a default, but
 that release lacks `jwt_validation` (p02, p11,
 `p03-jwks-rs256-basic`) and the full body-rewrite policy surface (p09,
-p10) the benchmark exercises, so we dropped the pin in
-[`.notes/PROGRESS.md § Iteration 23`](../../.notes/PROGRESS.md).
+p10) the benchmark exercises, so we dropped the pin (see the internal
+progress log § Iteration 23).
+
+### Comparing multiple wallarm builds in one sweep
+
+`WALLARM_IMAGE` accepts a comma-separated list — `bench run` expands
+each entry into a distinct column named `wallarm@<variant>` so two (or
+more) builds run side-by-side through the same matrix:
+
+```bash
+WALLARM_IMAGE='wallarm:branch-main,wallarm:branch-other' \
+    orchestrator/bin/bench run --gateways wallarm,nginx --matrix canonical
+# columns: nginx, wallarm@branch-main, wallarm@branch-other
+```
+
+The variant label is derived from the image tag (everything after the
+last `:` in the reference). Duplicate labels get a `-2`/`-3` suffix.
+A single-value `WALLARM_IMAGE` (no comma) keeps the legacy column name
+`wallarm`, so existing pipelines are unaffected.
+
+Variant cells share `gateways/wallarm/docker-compose.yaml` and all the
+profile sub-directories below it — only the image swap and column
+label differ. Per-cell artefacts land at
+`reports/<run-id>/raw/wallarm@<variant>/...`.
 
 | Field        | Value                                                       |
 |--------------|-------------------------------------------------------------|

orchestrator/cmd/run.go

@@ -80,23 +80,32 @@ Use --dry-run first to see the planned cell list.`,
 
 			scenarios := matrix.ParseCSV(scenariosCSV)
 
+			// WALLARM_IMAGE is a CSV when the operator wants to benchmark
+			// multiple wallarm builds against each other in one sweep —
+			// e.g. WALLARM_IMAGE='wallarm:branch-main,wallarm:branch-other'.
+			// With 2+ entries the wallarm column fans out into one
+			// "wallarm@<variant>" column per image; with a single value
+			// the column stays plain "wallarm" (legacy behaviour).
+			wallarmVariants := matrix.ParseWallarmImageEnv(os.Getenv("WALLARM_IMAGE"))
+
 			var cells []matrix.Cell
 			var err error
 			switch matrixMode {
 			case "selection":
 				sel := matrix.Selection{
-					Gateways:    gateways,
-					Policies:    policies,
-					Scenarios:   scenarios,
-					Loads:       loads,
-					Repetitions: repetitions,
+					Gateways:        gateways,
+					Policies:        policies,
+					Scenarios:       scenarios,
+					Loads:           loads,
+					Repetitions:     repetitions,
+					WallarmVariants: wallarmVariants,
 				}
 				cells, err = sel.Expand()
 			case "canonical":
 				if len(scenarios) > 0 || cmd.Flags().Changed("policies") {
 					return fmt.Errorf("--matrix canonical owns policies/scenarios; use --gateways/--loads/--reps to narrow it")
 				}
-				cells, err = matrix.CanonicalReportCells(gateways, loads, repetitions)
+				cells, err = matrix.CanonicalReportCellsWithVariants(gateways, loads, repetitions, wallarmVariants)
 			default:
 				return fmt.Errorf("unknown --matrix %q (valid: selection, canonical)", matrixMode)
 			}

orchestrator/internal/manifest/manifest.go

@@ -132,13 +132,21 @@ func New(ctx context.Context, repoRoot, mode, runID string, seed int64) *Builder
 
 // AddGateway records a gateway reference. Call after the gateway has
 // been brought up so we can resolve its image digest.
+//
+// "wallarm@<variant>" entries share gateways/wallarm/docker-compose.yaml
+// — strip the suffix before resolving the compose path so the manifest
+// still gets a useful image probe.
 func (b *Builder) AddGateway(ctx context.Context, name string) {
-	composePath := filepath.Join(b.repoRoot, "gateways", name, "docker-compose.yaml")
+	base := name
+	if i := strings.IndexByte(base, '@'); i >= 0 {
+		base = base[:i]
+	}
+	composePath := filepath.Join(b.repoRoot, "gateways", base, "docker-compose.yaml")
 	ref := GatewayRef{
 		Name:        name,
 		ComposePath: composePath,
 	}
-	if image, digest, ok := probeGatewayImage(ctx, name); ok {
+	if image, digest, ok := probeGatewayImage(ctx, base); ok {
 		ref.Image = image
 		ref.Digest = digest
 		ref.Source = "registry"