diff --git a/gateways/wallarm/README.md b/gateways/wallarm/README.md index 990219e..4bd841e 100644 --- a/gateways/wallarm/README.md +++ b/gateways/wallarm/README.md @@ -22,8 +22,30 @@ fails loudly with "`WALLARM_IMAGE must be set`" — this is intentional; an earlier iteration pinned the public `0.2.0` image as a default, but that release lacks `jwt_validation` (p02, p11, `p03-jwks-rs256-basic`) and the full body-rewrite policy surface (p09, -p10) the benchmark exercises, so we dropped the pin in -[`.notes/PROGRESS.md § Iteration 23`](../../.notes/PROGRESS.md). +p10) the benchmark exercises, so we dropped the pin (see the internal +progress log § Iteration 23). + +### Comparing multiple wallarm builds in one sweep + +`WALLARM_IMAGE` accepts a comma-separated list — `bench run` expands +each entry into a distinct column named `wallarm@` so two (or +more) builds run side-by-side through the same matrix: + +```bash +WALLARM_IMAGE='wallarm:branch-main,wallarm:branch-other' \ + orchestrator/bin/bench run --gateways wallarm,nginx --matrix canonical +# columns: nginx, wallarm@branch-main, wallarm@branch-other +``` + +The variant label is derived from the image tag (everything after the +last `:` in the reference). Duplicate labels get a `-2`/`-3` suffix. +A single-value `WALLARM_IMAGE` (no comma) keeps the legacy column name +`wallarm`, so existing pipelines are unaffected. + +Variant cells share `gateways/wallarm/docker-compose.yaml` and all the +profile sub-directories below it — only the image swap and column +label differ. Per-cell artefacts land at +`reports//raw/wallarm@/...`. | Field | Value | |--------------|-------------------------------------------------------------| diff --git a/orchestrator/cmd/run.go b/orchestrator/cmd/run.go index fc8b0ad..779df3e 100644 --- a/orchestrator/cmd/run.go +++ b/orchestrator/cmd/run.go @@ -80,23 +80,32 @@ Use --dry-run first to see the planned cell list.`, scenarios := matrix.ParseCSV(scenariosCSV) + // WALLARM_IMAGE is a CSV when the operator wants to benchmark + // multiple wallarm builds against each other in one sweep — + // e.g. WALLARM_IMAGE='wallarm:branch-main,wallarm:branch-other'. + // With 2+ entries the wallarm column fans out into one + // "wallarm@" column per image; with a single value + // the column stays plain "wallarm" (legacy behaviour). + wallarmVariants := matrix.ParseWallarmImageEnv(os.Getenv("WALLARM_IMAGE")) + var cells []matrix.Cell var err error switch matrixMode { case "selection": sel := matrix.Selection{ - Gateways: gateways, - Policies: policies, - Scenarios: scenarios, - Loads: loads, - Repetitions: repetitions, + Gateways: gateways, + Policies: policies, + Scenarios: scenarios, + Loads: loads, + Repetitions: repetitions, + WallarmVariants: wallarmVariants, } cells, err = sel.Expand() case "canonical": if len(scenarios) > 0 || cmd.Flags().Changed("policies") { return fmt.Errorf("--matrix canonical owns policies/scenarios; use --gateways/--loads/--reps to narrow it") } - cells, err = matrix.CanonicalReportCells(gateways, loads, repetitions) + cells, err = matrix.CanonicalReportCellsWithVariants(gateways, loads, repetitions, wallarmVariants) default: return fmt.Errorf("unknown --matrix %q (valid: selection, canonical)", matrixMode) } diff --git a/orchestrator/internal/manifest/manifest.go b/orchestrator/internal/manifest/manifest.go index 1008ba5..0cc2b8b 100644 --- a/orchestrator/internal/manifest/manifest.go +++ b/orchestrator/internal/manifest/manifest.go @@ -132,13 +132,21 @@ func New(ctx context.Context, repoRoot, mode, runID string, seed int64) *Builder // AddGateway records a gateway reference. Call after the gateway has // been brought up so we can resolve its image digest. +// +// "wallarm@" entries share gateways/wallarm/docker-compose.yaml +// — strip the suffix before resolving the compose path so the manifest +// still gets a useful image probe. func (b *Builder) AddGateway(ctx context.Context, name string) { - composePath := filepath.Join(b.repoRoot, "gateways", name, "docker-compose.yaml") + base := name + if i := strings.IndexByte(base, '@'); i >= 0 { + base = base[:i] + } + composePath := filepath.Join(b.repoRoot, "gateways", base, "docker-compose.yaml") ref := GatewayRef{ Name: name, ComposePath: composePath, } - if image, digest, ok := probeGatewayImage(ctx, name); ok { + if image, digest, ok := probeGatewayImage(ctx, base); ok { ref.Image = image ref.Digest = digest ref.Source = "registry" diff --git a/orchestrator/internal/matrix/matrix.go b/orchestrator/internal/matrix/matrix.go index 4b593c7..611a74e 100644 --- a/orchestrator/internal/matrix/matrix.go +++ b/orchestrator/internal/matrix/matrix.go @@ -94,12 +94,25 @@ var HTTPSScenarios = map[string]string{ // Cell is a single (gateway, policy, scenario, load) coordinate. // Repetition is the 1-indexed pass within a multi-rep run. +// +// Gateway may carry a "@" suffix when the operator passes +// WALLARM_IMAGE as a comma-separated list (one wallarm column per +// image — see ExpandWallarmVariants). The base name (before "@") is +// the directory under gateways/ that owns docker-compose.yaml / +// setup.sh; the suffix only flavours the column label and the per- +// cell raw/ output directory. type Cell struct { Gateway string `json:"gateway"` Policy string `json:"policy"` Scenario string `json:"scenario"` Load string `json:"load"` Repetition int `json:"repetition"` + + // WallarmImage is the image to pull for this cell when Gateway + // names a wallarm variant. Empty for non-wallarm cells and for + // single-variant runs (where WALLARM_IMAGE remains a scalar env + // var honoured by gateways/wallarm/docker-compose.yaml directly). + WallarmImage string `json:"wallarm_image,omitempty"` } // ID returns "///[#repN]" — used @@ -112,6 +125,38 @@ func (c Cell) ID() string { return id } +// GatewayBase returns the gateway name without any "@" +// suffix — i.e. the directory under gateways/ that owns the compose +// file and policy subdirs. For non-suffixed names it is identical to +// Cell.Gateway. +func (c Cell) GatewayBase() string { + return GatewayBase(c.Gateway) +} + +// GatewayVariant returns the "@" suffix part (without the +// "@") or "" when the gateway name has no variant suffix. +func (c Cell) GatewayVariant() string { + return GatewayVariant(c.Gateway) +} + +// GatewayBase returns the gateway name without any "@" +// suffix. Free function so callers that have a plain string (e.g. +// the aggregator reading directory names off disk) can reuse it. +func GatewayBase(name string) string { + if i := strings.IndexByte(name, '@'); i >= 0 { + return name[:i] + } + return name +} + +// GatewayVariant returns the variant suffix or "" when none is set. +func GatewayVariant(name string) string { + if i := strings.IndexByte(name, '@'); i >= 0 { + return name[i+1:] + } + return "" +} + // OutputDir mirrors scripts/load-gateway.sh's per-cell directory // layout: reports//raw//____ // with a #repN suffix for repetitions > 1. @@ -131,6 +176,142 @@ type Selection struct { Scenarios []string // one-per-policy, in order; auto-derived if empty Loads []string Repetitions int + + // WallarmVariants, when set with 2+ entries, expands every + // "wallarm" entry in Gateways into one "wallarm@" entry + // per variant. With 0 or 1 entries the wallarm column stays a + // single "wallarm" — matching legacy single-image behaviour. + WallarmVariants []WallarmVariant +} + +// WallarmVariant pairs the human-readable label that becomes the +// gateway column suffix ("wallarm@") with the docker image +// reference that should be set as WALLARM_IMAGE for cells in that +// column. +type WallarmVariant struct { + Name string + Image string +} + +// ParseWallarmImageEnv parses a WALLARM_IMAGE env value into a list +// of variants. A scalar (no comma) returns a single variant whose +// Name is derived from the image tag and Image is the input verbatim. +// An empty / whitespace-only input returns nil. +// +// The variant name is the last colon-separated segment of the image +// reference, sanitised to docker-name safe characters. Duplicate +// names get a "-N" disambiguator. Examples: +// +// "wallarm:branch-main" +// → [{Name: "branch-main", Image: "wallarm:branch-main"}] +// +// "wallarm:branch-main,wallarm:branch-other" +// → [{Name: "branch-main", Image: "wallarm:branch-main"}, +// {Name: "branch-other", Image: "wallarm:branch-other"}] +func ParseWallarmImageEnv(env string) []WallarmVariant { + parts := ParseCSV(env) + if len(parts) == 0 { + return nil + } + out := make([]WallarmVariant, 0, len(parts)) + seen := make(map[string]int, len(parts)) + for _, image := range parts { + name := variantNameFromImage(image) + if dup, ok := seen[name]; ok { + seen[name] = dup + 1 + name = fmt.Sprintf("%s-%d", name, dup+1) + } else { + seen[name] = 1 + } + out = append(out, WallarmVariant{Name: name, Image: image}) + } + return out +} + +// variantNameFromImage extracts a docker-safe label from an image +// reference. Prefers the tag after the last ':'; falls back to the +// last path segment when the reference has no tag (e.g. bare digest). +func variantNameFromImage(image string) string { + candidate := image + if i := strings.LastIndexByte(candidate, '@'); i >= 0 { + // digest form: drop the digest, keep the tag if any + candidate = candidate[:i] + } + if i := strings.LastIndexByte(candidate, ':'); i >= 0 { + candidate = candidate[i+1:] + } else if i := strings.LastIndexByte(candidate, '/'); i >= 0 { + candidate = candidate[i+1:] + } + candidate = sanitiseVariantName(candidate) + if candidate == "" { + return "variant" + } + return candidate +} + +func sanitiseVariantName(s string) string { + var b strings.Builder + lastDash := false + for _, r := range strings.ToLower(s) { + ok := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' || r == '_' || r == '.' + if ok { + b.WriteRune(r) + lastDash = false + continue + } + if !lastDash { + b.WriteByte('-') + lastDash = true + } + } + return strings.Trim(b.String(), "-") +} + +// ExpandWallarmVariants takes a gateway list and, when 2+ variants +// are supplied, replaces each "wallarm" entry with one +// "wallarm@" entry per variant. 0 or 1 variants is a no-op +// — single-image runs keep the legacy "wallarm" column name. +func ExpandWallarmVariants(gateways []string, variants []WallarmVariant) []string { + if len(variants) < 2 { + return append([]string(nil), gateways...) + } + out := make([]string, 0, len(gateways)+len(variants)-1) + for _, gw := range gateways { + if GatewayBase(gw) != "wallarm" || GatewayVariant(gw) != "" { + out = append(out, gw) + continue + } + for _, v := range variants { + out = append(out, "wallarm@"+v.Name) + } + } + return out +} + +// WallarmImageFor returns the image associated with a gateway name +// (e.g. "wallarm@branch-main" → "wallarm:branch-main"). Returns "" +// when the gateway is not a wallarm variant or no matching variant +// exists in the list. +func WallarmImageFor(gateway string, variants []WallarmVariant) string { + if GatewayBase(gateway) != "wallarm" { + return "" + } + variant := GatewayVariant(gateway) + if variant == "" { + // Scalar wallarm column — only meaningful when a single variant + // was supplied. Return its image so the runner can still inject + // WALLARM_IMAGE per cell rather than relying on the env leak. + if len(variants) == 1 { + return variants[0].Image + } + return "" + } + for _, v := range variants { + if v.Name == variant { + return v.Image + } + } + return "" } // Expand returns every cell implied by the selection, in stable @@ -167,17 +348,20 @@ func (s Selection) Expand() ([]Cell, error) { } } + gateways := ExpandWallarmVariants(s.Gateways, s.WallarmVariants) var cells []Cell - for _, gw := range s.Gateways { + for _, gw := range gateways { + image := WallarmImageFor(gw, s.WallarmVariants) for i, policy := range s.Policies { for _, load := range s.Loads { for rep := 1; rep <= s.Repetitions; rep++ { cells = append(cells, Cell{ - Gateway: gw, - Policy: policy, - Scenario: scenarios[i], - Load: load, - Repetition: rep, + Gateway: gw, + Policy: policy, + Scenario: scenarios[i], + Load: load, + Repetition: rep, + WallarmImage: image, }) } } @@ -189,6 +373,14 @@ func (s Selection) Expand() ([]Cell, error) { // CanonicalReportCells expands the published report matrix: // (11 ranking HTTP profiles + 2 HTTPS scenarios) x loads x gateways x reps. func CanonicalReportCells(gateways, loads []string, repetitions int) ([]Cell, error) { + return CanonicalReportCellsWithVariants(gateways, loads, repetitions, nil) +} + +// CanonicalReportCellsWithVariants is the variant-aware version of +// CanonicalReportCells. When 2+ wallarm variants are passed, each +// "wallarm" entry expands into one "wallarm@" column with +// the corresponding image stamped onto every emitted cell. +func CanonicalReportCellsWithVariants(gateways, loads []string, repetitions int, variants []WallarmVariant) ([]Cell, error) { if len(gateways) == 0 { return nil, fmt.Errorf("matrix: at least one --gateway is required") } @@ -217,17 +409,20 @@ func CanonicalReportCells(gateways, loads []string, repetitions int) ([]Cell, er policyScenario{policy: "p12-full-pipeline", scenario: HTTPSScenarios["p12-full-pipeline"]}, ) + expanded := ExpandWallarmVariants(gateways, variants) var cells []Cell - for _, gw := range gateways { + for _, gw := range expanded { + image := WallarmImageFor(gw, variants) for _, pair := range pairs { for _, load := range loads { for rep := 1; rep <= repetitions; rep++ { cells = append(cells, Cell{ - Gateway: gw, - Policy: pair.policy, - Scenario: pair.scenario, - Load: load, - Repetition: rep, + Gateway: gw, + Policy: pair.policy, + Scenario: pair.scenario, + Load: load, + Repetition: rep, + WallarmImage: image, }) } } @@ -305,6 +500,10 @@ func ResolveAlias(kind, value string) []string { // SortStable sorts a slice in canonical order (CanonicalPolicies for // "policies", CanonicalGateways for "gateways", CanonicalLoads for // "loads"). Unknown items go to the end alphabetically. +// +// For "gateways", a "wallarm@" entry ranks at the same +// position as plain "wallarm"; ties between variants are broken by +// the variant suffix in input order (stable). func SortStable(kind string, items []string) []string { var rank map[string]int switch kind { @@ -320,12 +519,23 @@ func SortStable(kind string, items []string) []string { return out } out := append([]string(nil), items...) + keyFor := func(s string) string { + if kind == "gateways" { + return GatewayBase(s) + } + return s + } sort.SliceStable(out, func(i, j int) bool { - ri, oi := rank[out[i]] - rj, oj := rank[out[j]] + ki, kj := keyFor(out[i]), keyFor(out[j]) + ri, oi := rank[ki] + rj, oj := rank[kj] switch { case oi && oj: - return ri < rj + if ri != rj { + return ri < rj + } + // same base rank — preserve input order for variants. + return false case oi: return true case oj: diff --git a/orchestrator/internal/matrix/matrix_test.go b/orchestrator/internal/matrix/matrix_test.go index aa7888f..2fdf8ea 100644 --- a/orchestrator/internal/matrix/matrix_test.go +++ b/orchestrator/internal/matrix/matrix_test.go @@ -173,3 +173,155 @@ func TestSortStable(t *testing.T) { t.Errorf("SortStable policies: got %v, want %v", out, want) } } + +func TestParseWallarmImageEnv(t *testing.T) { + cases := []struct { + in string + want []WallarmVariant + }{ + {"", nil}, + {"wallarm:branch-main", []WallarmVariant{{Name: "branch-main", Image: "wallarm:branch-main"}}}, + { + "wallarm:branch-main,wallarm:branch-other", + []WallarmVariant{ + {Name: "branch-main", Image: "wallarm:branch-main"}, + {Name: "branch-other", Image: "wallarm:branch-other"}, + }, + }, + { + // Same tag twice — second gets disambiguated with a -2 suffix. + "wallarm:main,registry.example/wallarm:main", + []WallarmVariant{ + {Name: "main", Image: "wallarm:main"}, + {Name: "main-2", Image: "registry.example/wallarm:main"}, + }, + }, + { + // Digest-only references: fall back to image name segment. + "wallarm/api-gateway@sha256:abc", + []WallarmVariant{{Name: "api-gateway", Image: "wallarm/api-gateway@sha256:abc"}}, + }, + } + for _, c := range cases { + got := ParseWallarmImageEnv(c.in) + if !reflect.DeepEqual(got, c.want) { + t.Errorf("ParseWallarmImageEnv(%q) = %#v, want %#v", c.in, got, c.want) + } + } +} + +func TestExpandWallarmVariants(t *testing.T) { + variants := []WallarmVariant{ + {Name: "branch-main", Image: "wallarm:branch-main"}, + {Name: "branch-other", Image: "wallarm:branch-other"}, + } + got := ExpandWallarmVariants([]string{"nginx", "wallarm", "envoy"}, variants) + want := []string{"nginx", "wallarm@branch-main", "wallarm@branch-other", "envoy"} + if !reflect.DeepEqual(got, want) { + t.Errorf("ExpandWallarmVariants 2: got %v, want %v", got, want) + } + + // Single variant: no expansion, legacy column name preserved. + single := []WallarmVariant{{Name: "main", Image: "wallarm:main"}} + got = ExpandWallarmVariants([]string{"nginx", "wallarm"}, single) + want = []string{"nginx", "wallarm"} + if !reflect.DeepEqual(got, want) { + t.Errorf("ExpandWallarmVariants 1: got %v, want %v", got, want) + } + + // Nil variants: no-op. + got = ExpandWallarmVariants([]string{"wallarm", "kong"}, nil) + want = []string{"wallarm", "kong"} + if !reflect.DeepEqual(got, want) { + t.Errorf("ExpandWallarmVariants nil: got %v, want %v", got, want) + } +} + +func TestCellGatewayBaseAndVariant(t *testing.T) { + c := Cell{Gateway: "wallarm@branch-main"} + if c.GatewayBase() != "wallarm" { + t.Errorf("GatewayBase: got %q, want wallarm", c.GatewayBase()) + } + if c.GatewayVariant() != "branch-main" { + t.Errorf("GatewayVariant: got %q, want branch-main", c.GatewayVariant()) + } + c = Cell{Gateway: "nginx"} + if c.GatewayBase() != "nginx" || c.GatewayVariant() != "" { + t.Errorf("plain name: base=%q variant=%q", c.GatewayBase(), c.GatewayVariant()) + } +} + +func TestSelectionExpandWithWallarmVariants(t *testing.T) { + sel := Selection{ + Gateways: []string{"wallarm", "nginx"}, + Policies: []string{"p01-vanilla"}, + Loads: []string{"p1-baseline"}, + WallarmVariants: []WallarmVariant{ + {Name: "branch-main", Image: "wallarm:branch-main"}, + {Name: "branch-other", Image: "wallarm:branch-other"}, + }, + } + cells, err := sel.Expand() + if err != nil { + t.Fatal(err) + } + if len(cells) != 3 { + t.Fatalf("want 3 cells (2 wallarm variants + 1 nginx), got %d", len(cells)) + } + if cells[0].Gateway != "wallarm@branch-main" || cells[0].WallarmImage != "wallarm:branch-main" { + t.Errorf("cell[0] unexpected: %+v", cells[0]) + } + if cells[1].Gateway != "wallarm@branch-other" || cells[1].WallarmImage != "wallarm:branch-other" { + t.Errorf("cell[1] unexpected: %+v", cells[1]) + } + if cells[2].Gateway != "nginx" || cells[2].WallarmImage != "" { + t.Errorf("cell[2] unexpected: %+v", cells[2]) + } +} + +func TestCanonicalReportCellsWithVariants(t *testing.T) { + variants := []WallarmVariant{ + {Name: "a", Image: "wallarm:a"}, + {Name: "b", Image: "wallarm:b"}, + } + cells, err := CanonicalReportCellsWithVariants( + []string{"wallarm"}, []string{"p1-baseline"}, 1, variants, + ) + if err != nil { + t.Fatal(err) + } + // 13 policy/scenario pairs (11 HTTP + 2 HTTPS) × 2 variants + if len(cells) != 26 { + t.Fatalf("want 26 cells, got %d", len(cells)) + } + sawA, sawB := false, false + for _, c := range cells { + switch c.Gateway { + case "wallarm@a": + sawA = true + if c.WallarmImage != "wallarm:a" { + t.Errorf("wallarm@a cell has image %q, want wallarm:a", c.WallarmImage) + } + case "wallarm@b": + sawB = true + if c.WallarmImage != "wallarm:b" { + t.Errorf("wallarm@b cell has image %q, want wallarm:b", c.WallarmImage) + } + default: + t.Errorf("unexpected gateway %q", c.Gateway) + } + } + if !sawA || !sawB { + t.Errorf("missing variant: sawA=%v sawB=%v", sawA, sawB) + } +} + +func TestSortStableGatewayVariants(t *testing.T) { + in := []string{"kong", "wallarm@b", "nginx", "wallarm@a"} + out := SortStable("gateways", in) + // Canonical order: nginx, wallarm*, kong. Variants keep input order. + want := []string{"nginx", "wallarm@b", "wallarm@a", "kong"} + if !reflect.DeepEqual(out, want) { + t.Errorf("SortStable gateways: got %v, want %v", out, want) + } +} diff --git a/orchestrator/internal/report/model.go b/orchestrator/internal/report/model.go index 6738705..b3423e8 100644 --- a/orchestrator/internal/report/model.go +++ b/orchestrator/internal/report/model.go @@ -611,6 +611,11 @@ func nullable(in []float64) []*float64 { // gatewaysIn returns every gateway that appears in the index, sorted // by GatewayStack canonical order (nginx first, then alphabetic for // any unknown extras). +// +// "wallarm@" entries — emitted when WALLARM_IMAGE was passed +// as a CSV — rank at the same canonical position as plain "wallarm" +// and break ties alphabetically by variant so the report columns +// stay deterministic across runs. func gatewaysIn(idx *Index) []string { seen := map[string]struct{}{} for _, byGW := range idx.Buckets { @@ -623,12 +628,22 @@ func gatewaysIn(idx *Index) []string { out = append(out, gw) } rank := indexedSlice(orderedGateways()) + baseOf := func(s string) string { + if i := strings.IndexByte(s, '@'); i >= 0 { + return s[:i] + } + return s + } sort.SliceStable(out, func(i, j int) bool { - ri, oi := rank[out[i]] - rj, oj := rank[out[j]] + bi, bj := baseOf(out[i]), baseOf(out[j]) + ri, oi := rank[bi] + rj, oj := rank[bj] switch { case oi && oj: - return ri < rj + if ri != rj { + return ri < rj + } + return out[i] < out[j] case oi: return true case oj: diff --git a/orchestrator/internal/runner/runner.go b/orchestrator/internal/runner/runner.go index 3dbd321..32a7c91 100644 --- a/orchestrator/internal/runner/runner.go +++ b/orchestrator/internal/runner/runner.go @@ -195,6 +195,31 @@ func (r *Runner) runOnce(ctx context.Context, cell matrix.Cell, attempt int) Res // `gwb-` and writes reports//raw//docker-stats.csv // — the shell sidecar is suppressed via BENCH_SKIP_DOCKER_STATS=1. cellEnv := nilSafeCellEnv(r.CellEnv, cell) + + // When the cell carries a per-variant wallarm image (CSV-form + // WALLARM_IMAGE in the parent env), override WALLARM_IMAGE for + // this cell only. Non-wallarm cells leave the parent env alone — + // their docker-compose files don't reference WALLARM_IMAGE. + if cell.WallarmImage != "" { + cellEnv = append(cellEnv, "WALLARM_IMAGE="+cell.WallarmImage) + } + + // "wallarm@branch-main" violates docker's container_name regex + // ([a-zA-Z0-9][a-zA-Z0-9_.-]*). In serial mode CellEnv is nil so + // load-gateway.sh would otherwise default BENCH_CONTAINER_PREFIX + // to "gwb-wallarm@branch-main" and compose up would fail. Inject + // a sanitised prefix + project name so the variant columns stay + // distinct on disk while the container names stay docker-safe. + // Parallel mode already passes its own slot-scoped prefix; skip + // when it has. + if cell.GatewayVariant() != "" && envValue(cellEnv, "BENCH_CONTAINER_PREFIX") == "" { + safe := "gwb-" + sanitizeDockerName(cell.Gateway) + cellEnv = append(cellEnv, + "BENCH_CONTAINER_PREFIX="+safe, + "BENCH_COMPOSE_PROJECT="+safe, + ) + } + gatewayContainer := envValue(cellEnv, "BENCH_CONTAINER_PREFIX") if gatewayContainer == "" { gatewayContainer = "gwb-" + cell.Gateway @@ -456,3 +481,26 @@ var ansiPattern = regexp.MustCompile(`\x1b\[[0-9;]*[A-Za-z]`) func stripANSI(s string) string { return ansiPattern.ReplaceAllString(s, "") } + +// sanitizeDockerName collapses any character outside [a-z0-9] into +// a single '-' and trims leading/trailing dashes. Mirrors the helper +// in cmd/run.go so wallarm@variant gateway names produce a valid +// container_name / compose project (docker forbids '@' there). +func sanitizeDockerName(s string) string { + s = strings.ToLower(s) + var b strings.Builder + lastDash := false + for _, r := range s { + ok := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + if ok { + b.WriteRune(r) + lastDash = false + continue + } + if !lastDash { + b.WriteByte('-') + lastDash = true + } + } + return strings.Trim(b.String(), "-") +} diff --git a/scripts/aws-clean-cell.sh b/scripts/aws-clean-cell.sh index 65c4236..301eb9c 100755 --- a/scripts/aws-clean-cell.sh +++ b/scripts/aws-clean-cell.sh @@ -35,6 +35,13 @@ done exit 2 } +# Multi-variant wallarm runs pass "wallarm@" as the gateway. +# The variant suffix is only a column label — compose file, setup.sh, +# FEATURE-MISSING, .env, and policy YAMLs all live under +# gateways//. Strip the suffix to find them; keep the full name +# in PROJECT / output paths so the variants stay distinguishable. +GATEWAY_BASE="${GATEWAY%@*}" + : "${AWS_GATEWAY_SSH:?AWS_GATEWAY_SSH is required}" : "${AWS_GATEWAY_PRIVATE_IP:?AWS_GATEWAY_PRIVATE_IP is required}" : "${AWS_BACKEND_PRIVATE_IP:?AWS_BACKEND_PRIVATE_IP is required}" @@ -65,7 +72,7 @@ REMOTE_OUT="/tmp/${PROJECT}-${POLICY}-${LOAD}-${SCENARIO}" REMOTE_OUT="${REMOTE_OUT//[^a-zA-Z0-9_\/.-]/-}" OVERRIDE="/tmp/${PROJECT}-external-backend.yaml" GATEWAY_DEPENDS_ON="[]" -case "${GATEWAY}" in +case "${GATEWAY_BASE}" in tyk) GATEWAY_DEPENDS_ON="[tyk-redis, jwks-server]" ;; @@ -101,20 +108,20 @@ HEARTBEAT_PID=$! cleanup() { set +e kill "${HEARTBEAT_PID}" >/dev/null 2>&1 || true - ssh_gateway "cd /opt/gateway-benchmarks; env_file='gateways/${GATEWAY}/${POLICY}/.env'; env_args=''; if [ -f \"\${env_file}\" ]; then env_args=\"--env-file \${env_file}\"; fi; GATEWAY_IMAGE='${GATEWAY_IMAGE:-}' WALLARM_IMAGE='${WALLARM_IMAGE:-}' BENCH_COMPOSE_PROJECT='${PROJECT}' BENCH_CONTAINER_PREFIX='${PREFIX}' docker compose -p '${PROJECT}' \${env_args} -f gateways/${GATEWAY}/docker-compose.yaml -f '${OVERRIDE}' logs --no-color > '${REMOTE_OUT}/compose.log' 2>&1 || true; GATEWAY_IMAGE='${GATEWAY_IMAGE:-}' WALLARM_IMAGE='${WALLARM_IMAGE:-}' BENCH_COMPOSE_PROJECT='${PROJECT}' BENCH_CONTAINER_PREFIX='${PREFIX}' docker compose -p '${PROJECT}' \${env_args} -f gateways/${GATEWAY}/docker-compose.yaml -f '${OVERRIDE}' down --remove-orphans -v >/dev/null 2>&1 || true" + ssh_gateway "cd /opt/gateway-benchmarks; env_file='gateways/${GATEWAY_BASE}/${POLICY}/.env'; env_args=''; if [ -f \"\${env_file}\" ]; then env_args=\"--env-file \${env_file}\"; fi; GATEWAY_IMAGE='${GATEWAY_IMAGE:-}' WALLARM_IMAGE='${WALLARM_IMAGE:-}' BENCH_COMPOSE_PROJECT='${PROJECT}' BENCH_CONTAINER_PREFIX='${PREFIX}' docker compose -p '${PROJECT}' \${env_args} -f gateways/${GATEWAY_BASE}/docker-compose.yaml -f '${OVERRIDE}' logs --no-color > '${REMOTE_OUT}/compose.log' 2>&1 || true; GATEWAY_IMAGE='${GATEWAY_IMAGE:-}' WALLARM_IMAGE='${WALLARM_IMAGE:-}' BENCH_COMPOSE_PROJECT='${PROJECT}' BENCH_CONTAINER_PREFIX='${PREFIX}' docker compose -p '${PROJECT}' \${env_args} -f gateways/${GATEWAY_BASE}/docker-compose.yaml -f '${OVERRIDE}' down --remove-orphans -v >/dev/null 2>&1 || true" ssh_gateway "cat '${REMOTE_OUT}/compose.log' 2>/dev/null || true" > "${LOGS_DIR}/compose.log" 2>/dev/null || true ssh_gateway "rm -f '${OVERRIDE}'" >/dev/null 2>&1 || true } trap cleanup EXIT -feature_missing="gateways/${GATEWAY}/${POLICY}/FEATURE-MISSING" +feature_missing="gateways/${GATEWAY_BASE}/${POLICY}/FEATURE-MISSING" # Scenario-specific marker — used when only one scenario inside a # policy is unimplemented (e.g. tyk OSS can't run s13/s14 because # `http_server_options.use_ssl` makes ALL listeners TLS-only, but its # HTTP scenario s01 works fine). Checked in addition to the policy- # wide marker so a gateway with partial coverage doesn't lose its HTTP # rows. -feature_missing_scenario="gateways/${GATEWAY}/${POLICY}/FEATURE-MISSING-${SCENARIO}" +feature_missing_scenario="gateways/${GATEWAY_BASE}/${POLICY}/FEATURE-MISSING-${SCENARIO}" if [[ -f "${feature_missing}" ]]; then reason="$(sed -n '1p' "${feature_missing}" 2>/dev/null || true)" jq -cn --arg gateway "${GATEWAY}" --arg policy "${POLICY}" --arg scenario "${SCENARIO}" \ @@ -154,11 +161,11 @@ cd /opt/gateway-benchmarks; # caller's RLIMIT_NOFILE — without this, the per-service ulimit silently # downgrades. See gateways//docker-compose.yaml § ulimits comment. ulimit -n 65536 2>/dev/null || true; -env_file='gateways/${GATEWAY}/${POLICY}/.env'; +env_file='gateways/${GATEWAY_BASE}/${POLICY}/.env'; env_args=''; if [ -f \"\${env_file}\" ]; then env_args=\"--env-file \${env_file}\"; fi; -BENCH_COMPOSE_PROJECT='${PROJECT}' BENCH_CONTAINER_PREFIX='${PREFIX}' GATEWAY_IMAGE='${GATEWAY_IMAGE:-}' WALLARM_IMAGE='${WALLARM_IMAGE:-}' GATEWAY_PROFILE='${POLICY}' GATEWAY_HTTP_PORT=9080 GATEWAY_HTTPS_PORT=9443 GATEWAY_ADMIN_PORT=9081 GATEWAY_ENVOY_ADMIN_PORT=9901 docker compose -p '${PROJECT}' \${env_args} -f gateways/${GATEWAY}/docker-compose.yaml -f '${OVERRIDE}' down --remove-orphans -v >/dev/null 2>&1 || true; -BENCH_COMPOSE_PROJECT='${PROJECT}' BENCH_CONTAINER_PREFIX='${PREFIX}' GATEWAY_IMAGE='${GATEWAY_IMAGE:-}' WALLARM_IMAGE='${WALLARM_IMAGE:-}' GATEWAY_PROFILE='${POLICY}' GATEWAY_HTTP_PORT=9080 GATEWAY_HTTPS_PORT=9443 GATEWAY_ADMIN_PORT=9081 GATEWAY_ENVOY_ADMIN_PORT=9901 docker compose -p '${PROJECT}' \${env_args} -f gateways/${GATEWAY}/docker-compose.yaml -f '${OVERRIDE}' up -d > '${REMOTE_OUT}/compose-up.log' 2>&1; +BENCH_COMPOSE_PROJECT='${PROJECT}' BENCH_CONTAINER_PREFIX='${PREFIX}' GATEWAY_IMAGE='${GATEWAY_IMAGE:-}' WALLARM_IMAGE='${WALLARM_IMAGE:-}' GATEWAY_PROFILE='${POLICY}' GATEWAY_HTTP_PORT=9080 GATEWAY_HTTPS_PORT=9443 GATEWAY_ADMIN_PORT=9081 GATEWAY_ENVOY_ADMIN_PORT=9901 docker compose -p '${PROJECT}' \${env_args} -f gateways/${GATEWAY_BASE}/docker-compose.yaml -f '${OVERRIDE}' down --remove-orphans -v >/dev/null 2>&1 || true; +BENCH_COMPOSE_PROJECT='${PROJECT}' BENCH_CONTAINER_PREFIX='${PREFIX}' GATEWAY_IMAGE='${GATEWAY_IMAGE:-}' WALLARM_IMAGE='${WALLARM_IMAGE:-}' GATEWAY_PROFILE='${POLICY}' GATEWAY_HTTP_PORT=9080 GATEWAY_HTTPS_PORT=9443 GATEWAY_ADMIN_PORT=9081 GATEWAY_ENVOY_ADMIN_PORT=9901 docker compose -p '${PROJECT}' \${env_args} -f gateways/${GATEWAY_BASE}/docker-compose.yaml -f '${OVERRIDE}' up -d > '${REMOTE_OUT}/compose-up.log' 2>&1; up_rc=\$?; if [ \"\${up_rc}\" -ne 0 ]; then exit 4; fi; for i in \$(seq 1 90); do @@ -195,7 +202,7 @@ if [[ "${up_rc}" != "0" ]]; then fi PHASE="gateway-setup" -ssh_gateway "cd /opt/gateway-benchmarks && BENCH_CONTAINER_PREFIX='${PREFIX}' DATA_URL='http://localhost:9080' ADMIN_URL='http://localhost:9081' BACKEND_URL='http://backend:8080' FEATURE_MISSING_REASON_FILE='${REMOTE_OUT}/setup-feature-missing.txt' bash gateways/${GATEWAY}/${POLICY}/setup.sh" \ +ssh_gateway "cd /opt/gateway-benchmarks && BENCH_CONTAINER_PREFIX='${PREFIX}' DATA_URL='http://localhost:9080' ADMIN_URL='http://localhost:9081' BACKEND_URL='http://backend:8080' FEATURE_MISSING_REASON_FILE='${REMOTE_OUT}/setup-feature-missing.txt' bash gateways/${GATEWAY_BASE}/${POLICY}/setup.sh" \ > "${LOGS_DIR}/setup.log" 2>&1 || setup_rc=$? setup_rc="${setup_rc:-0}" if [[ "${setup_rc}" == "42" ]]; then diff --git a/scripts/load-gateway.sh b/scripts/load-gateway.sh index 0531b73..cd61095 100755 --- a/scripts/load-gateway.sh +++ b/scripts/load-gateway.sh @@ -110,6 +110,15 @@ done [[ -n "${SCENARIO}" ]] || { printf '%s\n' "--scenario is required" >&2; exit 2; } [[ -n "${LOAD}" ]] || { printf '%s\n' "--load is required" >&2; exit 2; } +# Multi-variant wallarm comparisons (e.g. WALLARM_IMAGE=img-a,img-b) +# expose each build as a distinct gateway column "wallarm@". +# The variant suffix is purely a column label — the compose file, +# setup.sh, FEATURE-MISSING marker, and policy YAMLs all live under +# gateways/wallarm/. Strip the suffix to find them; keep the full +# name for the output directory + container prefix so the runs stay +# distinguishable on disk. +GATEWAY_BASE="${GATEWAY%@*}" + # Accepted load profiles — closed-loop (p1/p2/p3/p4-*) plus paced- # arrivals twins (p1c/p2c/p3c/p4c-paced). The `-paced` suffix is the # gate for the `constant-arrival-rate` executors; see @@ -126,8 +135,8 @@ scenario_file="k6/scenarios/${SCENARIO}.js" [[ -f "${scenario_file}" ]] \ || { printf 'scenario script not found: %s\n' "${scenario_file}" >&2; exit 2; } -compose_file="gateways/${GATEWAY}/docker-compose.yaml" -profile_dir="gateways/${GATEWAY}/${POLICY}" +compose_file="gateways/${GATEWAY_BASE}/docker-compose.yaml" +profile_dir="gateways/${GATEWAY_BASE}/${POLICY}" setup_script="${profile_dir}/setup.sh" feature_missing="${profile_dir}/FEATURE-MISSING" @@ -147,8 +156,11 @@ GATEWAY_HTTP_PORT="${GATEWAY_HTTP_PORT:-9080}" GATEWAY_HTTPS_PORT="${GATEWAY_HTTPS_PORT:-9443}" GATEWAY_ADMIN_PORT="${GATEWAY_ADMIN_PORT:-9081}" GATEWAY_ENVOY_ADMIN_PORT="${GATEWAY_ENVOY_ADMIN_PORT:-9901}" -BENCH_COMPOSE_PROJECT="${BENCH_COMPOSE_PROJECT:-gateway-benchmarks-${GATEWAY}}" -BENCH_CONTAINER_PREFIX="${BENCH_CONTAINER_PREFIX:-gwb-${GATEWAY}}" +# Sanitise the gateway slug for docker (compose project + container +# names disallow '@'). Lower-case anything outside [a-z0-9] → '-'. +gateway_docker_slug="$(printf '%s' "${GATEWAY}" | tr 'A-Z' 'a-z' | sed 's/[^a-z0-9]\+/-/g; s/^-//; s/-$//')" +BENCH_COMPOSE_PROJECT="${BENCH_COMPOSE_PROJECT:-gateway-benchmarks-${gateway_docker_slug}}" +BENCH_CONTAINER_PREFIX="${BENCH_CONTAINER_PREFIX:-gwb-${gateway_docker_slug}}" GATEWAY_TARGET="${GATEWAY_TARGET:-http://localhost:${GATEWAY_HTTP_PORT}}" export GATEWAY_HTTP_PORT GATEWAY_HTTPS_PORT GATEWAY_ADMIN_PORT GATEWAY_ENVOY_ADMIN_PORT export BENCH_COMPOSE_PROJECT BENCH_CONTAINER_PREFIX @@ -199,7 +211,7 @@ fi # Compose driver (handles per-profile .env override identically to # parity-gateway.sh — keeps the two lifecycles 100% consistent). # ----------------------------------------------------------------------------- -profile_env="gateways/${GATEWAY}/${POLICY}/.env" +profile_env="gateways/${GATEWAY_BASE}/${POLICY}/.env" compose_cmd=(docker compose -p "${BENCH_COMPOSE_PROJECT}") if [[ -f "${profile_env}" ]]; then compose_cmd+=(--env-file "${profile_env}") @@ -213,7 +225,7 @@ compose_cmd+=(-f "${compose_file}") # project name dynamically to stay agnostic of any future rename. project_name="$("${compose_cmd[@]}" config --format json 2>/dev/null \ | jq -r '.name' 2>/dev/null || true)" -project_name="${project_name:-gateway-benchmarks-${GATEWAY}}" +project_name="${project_name:-gateway-benchmarks-${gateway_docker_slug}}" bench_network="${project_name}_bench-net" # -----------------------------------------------------------------------------