Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/rulesets/branches/default-branch.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@
"parameters": {
"required_approving_review_count": 1,
"dismiss_stale_reviews_on_push": true,
"required_reviewers": [],
"require_code_owner_review": false,
"require_last_push_approval": false,
"required_review_thread_resolution": true,
"automatic_copilot_code_review_enabled": false,
"allowed_merge_methods": [
"merge"
]
Expand All @@ -41,7 +41,8 @@
"do_not_enforce_on_create": false,
"required_status_checks": [
{
"context": "call-core-crucible-ci / core-crucible-ci-complete"
"context": "crucible-ci-complete",
"integration_id": 15368
}
]
}
Expand Down
55 changes: 45 additions & 10 deletions .github/workflows/crucible-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,58 @@ name: crucible-ci
on:
pull_request:
branches: [ master ]
paths-ignore:
- LICENSE
- '**.md'
- '.github/rulesets/**'
- .github/workflows/run-crucible-tracking.yaml
- .github/workflows/faux-crucible-ci.yaml
- .github/workflows/faux-unittest.yaml
- 'docs/**'
workflow_dispatch:

concurrency:
group: ${{ github.ref }}/crucible-ci
cancel-in-progress: true

jobs:
call-core-crucible-ci:
changes:
runs-on: ubuntu-latest
outputs:
only-docs: ${{ steps.filter.outputs.only_changed }}
steps:
- uses: actions/checkout@v4
- id: filter
uses: tj-actions/changed-files@v47
with:
files: |
LICENSE
*.md
**/*.md
.github/rulesets/**
.github/workflows/run-crucible-tracking.yaml
.github/workflows/crucible-ci.yaml
.github/workflows/unittest.yaml
docs/**

call-real-core-crucible-ci:
needs: changes
if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.only-docs != 'true' }}
uses: perftool-incubator/crucible-ci/.github/workflows/core-crucible-ci.yaml@main
with:
ci_target: "multiplex"
ci_target_branch: "${{ github.ref }}"
github_workspace: "$GITHUB_WORKSPACE"
userenv_filter: "minimal"
secrets:
registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }}
ci_registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }}
quay_oauth_token: ${{ secrets.CRUCIBLE_QUAYIO_OAUTH_TOKEN }}

call-faux-core-crucible-ci:
needs: changes
if: ${{ github.event_name != 'workflow_dispatch' && needs.changes.outputs.only-docs == 'true' }}
uses: perftool-incubator/crucible-ci/.github/workflows/faux-core-crucible-ci.yaml@main

crucible-ci-complete:
needs: [ call-real-core-crucible-ci, call-faux-core-crucible-ci ]
if: always()
runs-on: ubuntu-latest
steps:
- name: Check Results
if: >-
contains(needs.*.result, 'failure') ||
contains(needs.*.result, 'cancelled')
run: exit 1
- run: echo "crucible-ci complete"
17 changes: 0 additions & 17 deletions .github/workflows/faux-crucible-ci.yaml

This file was deleted.

19 changes: 0 additions & 19 deletions .github/workflows/faux-unittest.yaml

This file was deleted.

67 changes: 44 additions & 23 deletions .github/workflows/unittest.yaml
Original file line number Diff line number Diff line change
@@ -1,37 +1,39 @@
# Basic ci workflow using github actions
name: unittest

concurrency:
group: ${{ github.ref }}/unittest
cancel-in-progress: true

# Controls when the action will run.
on:
# Triggers the workflow on push or pull request events but only for the master branch
pull_request:
branches: [ master ]
paths-ignore:
- LICENSE
- '**.md'
- '.github/rulesets/**'
- .github/workflows/run-crucible-tracking.yaml
- .github/workflows/faux-crucible-ci.yaml
- .github/workflows/faux-unittest.yaml
- 'docs/**'

# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
concurrency:
group: ${{ github.ref }}/unittest
cancel-in-progress: true

jobs:
# Jobs to test schema, input json file, and requirements
multiplex-unittests:
# Job will run on github-hosted runner
changes:
runs-on: ubuntu-latest
outputs:
only-docs: ${{ steps.filter.outputs.only_changed }}
steps:
- uses: actions/checkout@v4
- id: filter
uses: tj-actions/changed-files@v47
with:
files: |
LICENSE
*.md
**/*.md
.github/rulesets/**
.github/workflows/run-crucible-tracking.yaml
.github/workflows/crucible-ci.yaml
.github/workflows/unittest.yaml
docs/**

# Steps represent a sequence of tasks that will be executed as part of the job
multiplex-unittests:
runs-on: ubuntu-latest
needs: changes
if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.only-docs != 'true' }}
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v4

- name: Install dependencies
Expand All @@ -50,3 +52,22 @@ jobs:
with:
name: report
path: report.html

faux-unittest:
runs-on: ubuntu-latest
needs: changes
if: ${{ github.event_name != 'workflow_dispatch' && needs.changes.outputs.only-docs == 'true' }}
steps:
- run: echo "faux-unittest-complete"

unittest-complete:
runs-on: ubuntu-latest
needs: [ multiplex-unittests, faux-unittest ]
if: always()
steps:
- name: Check Results
if: >-
contains(needs.*.result, 'failure') ||
contains(needs.*.result, 'cancelled')
run: exit 1
- run: echo "unittest-complete"
53 changes: 53 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
# Multiplex - Parameter Expansion Engine

## Purpose
Multiplex translates multi-value benchmark parameters into single-value test matrices. Given a set of parameters with multiple possible values, it produces all valid combinations (the Cartesian product), respecting parameter roles and constraints. Output feeds into rickshaw-run as `bench-params.json`.

## Language
Python 3 — single main file `multiplex.py` (~510 lines), no dependencies beyond `jsonschema`.

## Key Files
| File | Purpose |
|------|---------|
| `multiplex.py` | Core parameter expansion logic and CLI entry point |
| `params` | Default CLI arguments file (read by argparse `fromfile_prefix_chars`) |
| `JSON/schema.json` | JSON schema for multi-value input validation |
| `JSON/req-schema.json` | JSON schema for requirements file validation |

## Key Functions
| Function | Purpose |
|----------|---------|
| `multiplex_sets()` | Main driver: expands all parameter sets |
| `multiplex_set()` | Expands one multi-value set into multiple single-value sets (Cartesian product via `itertools.product`) |
| `load_param_sets()` | Resolves `include` references from global-options into sets |
| `override_presets()` | Applies defaults, essentials, and named presets from requirements |
| `transform_param_val()` | Applies validation regex, unit conversion, and regex transformation |
| `validate_schema()` | Validates input JSON against schema |

## Exit Codes
| Code | Constant | Meaning |
|------|----------|---------|
| 0 | `EC_SUCCESS` | Success |
| 1 | `EC_SCHEMA_FAIL` | Input JSON schema validation failed |
| 2 | `EC_JSON_FAIL` | JSON file loading failed |
| 3 | `EC_REQUIREMENTS_FAIL` | Requirements file loading failed |
| 4 | `EC_VALIDATIONS_FAIL` | Parameter validation failed |
| 5 | `EC_REQ_SCHEMA_FAIL` | Requirements JSON schema validation failed |
| 6 | `EC_EMPTY_SET_FAIL` | Empty param set after preset override |

## Input/Output
- **Input**: JSON with `global-options` and `sets` arrays; params use `arg` and `vals` (multi-value list) keys, optional `role` (client/server/all), `id`, `enabled`
- **Output**: JSON array of arrays; each inner array is one test iteration with single-value params using `arg`, `val`, and `role` keys
- **Requirements** (optional): Defines `presets` (defaults, essentials, named), `validations` (regex), and `units` (conversion factors)

## Tests
Run with `cd` to project root, then `pytest tests/`:
- `tests/test-json.py` — Parameter loading, multiplexing, Cartesian expansion, value conversion
- `tests/test-requirements.py` — Validation regex, presets, unit conversion, preset precedence
- `tests/test-schema.py` — Schema validation for valid and invalid inputs
- 60+ JSON fixtures in `tests/JSON/`

## Conventions
- 4-space indentation, PEP 8 naming
- Module-level dicts for state: `validation_dict`, `convert_dict`, `transform_dict`, `presets_dict`
- Uses Python `logging` module for debug/info/warning/error output
15 changes: 14 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@ When running a benchmark, it is often desirable to run it multiple ways, changin

## Usage
```
./multiplex.py [--requirements JSON/requirements.json] --input JSON/mv-params-input.json [--output /path/to/bench-params.json]
./multiplex.py [--requirements JSON/requirements.json] --input JSON/mv-params-input.json [--output /path/to/bench-params.json] [--debug]
```

Default CLI arguments can be placed in a `params` file in the same directory, one argument per line (read via argparse `fromfile_prefix_chars`).

## Input multi-value JSON
Multiplex requires a JSON file with the following format:
```
Expand Down Expand Up @@ -274,3 +276,14 @@ parameters to STDOUT. A sample is available in `JSON/bench-params-output.json`:
This JSON can then optionally be modified by the user, and then provided
(typically as "bench-params.json") to a benchmark orchestrator like
rickshaw-run.

## Exit codes
| Code | Meaning |
|------|---------|
| 0 | Success |
| 1 | Input JSON schema validation failed |
| 2 | JSON file loading failed |
| 3 | Requirements file loading failed |
| 4 | Parameter validation failed |
| 5 | Requirements JSON schema validation failed |
| 6 | Empty param set after preset override (missing essentials/defaults) |
Loading