diff --git a/.github/rulesets/branches/default-branch.json b/.github/rulesets/branches/default-branch.json index a3ef448..f8a373e 100644 --- a/.github/rulesets/branches/default-branch.json +++ b/.github/rulesets/branches/default-branch.json @@ -25,10 +25,10 @@ "parameters": { "required_approving_review_count": 1, "dismiss_stale_reviews_on_push": true, + "required_reviewers": [], "require_code_owner_review": false, "require_last_push_approval": false, "required_review_thread_resolution": true, - "automatic_copilot_code_review_enabled": false, "allowed_merge_methods": [ "merge" ] @@ -41,7 +41,8 @@ "do_not_enforce_on_create": false, "required_status_checks": [ { - "context": "call-core-crucible-ci / core-crucible-ci-complete" + "context": "crucible-ci-complete", + "integration_id": 15368 } ] } diff --git a/.github/workflows/crucible-ci.yaml b/.github/workflows/crucible-ci.yaml index 50f9bac..1d37ca0 100644 --- a/.github/workflows/crucible-ci.yaml +++ b/.github/workflows/crucible-ci.yaml @@ -3,18 +3,35 @@ name: crucible-ci on: pull_request: branches: [ master ] - paths-ignore: - - LICENSE - - '**.md' - - '.github/rulesets/**' - - .github/workflows/run-crucible-tracking.yaml - - .github/workflows/faux-crucible-ci.yaml - - .github/workflows/faux-unittest.yaml - - 'docs/**' workflow_dispatch: +concurrency: + group: ${{ github.ref }}/crucible-ci + cancel-in-progress: true + jobs: - call-core-crucible-ci: + changes: + runs-on: ubuntu-latest + outputs: + only-docs: ${{ steps.filter.outputs.only_changed }} + steps: + - uses: actions/checkout@v4 + - id: filter + uses: tj-actions/changed-files@v47 + with: + files: | + LICENSE + *.md + **/*.md + .github/rulesets/** + .github/workflows/run-crucible-tracking.yaml + .github/workflows/crucible-ci.yaml + .github/workflows/unittest.yaml + docs/** + + call-real-core-crucible-ci: + needs: changes + if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.only-docs != 'true' }} uses: perftool-incubator/crucible-ci/.github/workflows/core-crucible-ci.yaml@main with: ci_target: "multiplex" @@ -22,4 +39,22 @@ jobs: github_workspace: "$GITHUB_WORKSPACE" userenv_filter: "minimal" secrets: - registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }} + ci_registry_auth: ${{ secrets.CRUCIBLE_CI_ENGINES_REGISTRY_AUTH }} + quay_oauth_token: ${{ secrets.CRUCIBLE_QUAYIO_OAUTH_TOKEN }} + + call-faux-core-crucible-ci: + needs: changes + if: ${{ github.event_name != 'workflow_dispatch' && needs.changes.outputs.only-docs == 'true' }} + uses: perftool-incubator/crucible-ci/.github/workflows/faux-core-crucible-ci.yaml@main + + crucible-ci-complete: + needs: [ call-real-core-crucible-ci, call-faux-core-crucible-ci ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check Results + if: >- + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') + run: exit 1 + - run: echo "crucible-ci complete" diff --git a/.github/workflows/faux-crucible-ci.yaml b/.github/workflows/faux-crucible-ci.yaml deleted file mode 100644 index aaf545b..0000000 --- a/.github/workflows/faux-crucible-ci.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: faux-crucible-ci - -on: - pull_request: - branches: [ master ] - paths: - - LICENSE - - '**.md' - - '.github/rulesets/**' - - .github/workflows/run-crucible-tracking.yaml - - .github/workflows/faux-crucible-ci.yaml - - .github/workflows/faux-unittest.yaml - - 'docs/**' - -jobs: - call-core-crucible-ci: - uses: perftool-incubator/crucible-ci/.github/workflows/faux-core-crucible-ci.yaml@main diff --git a/.github/workflows/faux-unittest.yaml b/.github/workflows/faux-unittest.yaml deleted file mode 100644 index 30c8c58..0000000 --- a/.github/workflows/faux-unittest.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: faux-unittest - -on: - pull_request: - branches: [ master ] - paths: - - LICENSE - - '**.md' - - '.github/rulesets/**' - - .github/workflows/run-crucible-tracking.yaml - - .github/workflows/faux-crucible-ci.yaml - - .github/workflows/faux-unittest.yaml - - 'docs/**' - -jobs: - multiplex-unittests: - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required" ' diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index 54dc78f..00c95ca 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -1,37 +1,39 @@ -# Basic ci workflow using github actions name: unittest -concurrency: - group: ${{ github.ref }}/unittest - cancel-in-progress: true - -# Controls when the action will run. on: - # Triggers the workflow on push or pull request events but only for the master branch pull_request: branches: [ master ] - paths-ignore: - - LICENSE - - '**.md' - - '.github/rulesets/**' - - .github/workflows/run-crucible-tracking.yaml - - .github/workflows/faux-crucible-ci.yaml - - .github/workflows/faux-unittest.yaml - - 'docs/**' - - # Allows you to run this workflow manually from the Actions tab workflow_dispatch: -# A workflow run is made up of one or more jobs that can run sequentially or in parallel +concurrency: + group: ${{ github.ref }}/unittest + cancel-in-progress: true + jobs: - # Jobs to test schema, input json file, and requirements - multiplex-unittests: - # Job will run on github-hosted runner + changes: runs-on: ubuntu-latest + outputs: + only-docs: ${{ steps.filter.outputs.only_changed }} + steps: + - uses: actions/checkout@v4 + - id: filter + uses: tj-actions/changed-files@v47 + with: + files: | + LICENSE + *.md + **/*.md + .github/rulesets/** + .github/workflows/run-crucible-tracking.yaml + .github/workflows/crucible-ci.yaml + .github/workflows/unittest.yaml + docs/** - # Steps represent a sequence of tasks that will be executed as part of the job + multiplex-unittests: + runs-on: ubuntu-latest + needs: changes + if: ${{ github.event_name == 'workflow_dispatch' || needs.changes.outputs.only-docs != 'true' }} steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v4 - name: Install dependencies @@ -50,3 +52,22 @@ jobs: with: name: report path: report.html + + faux-unittest: + runs-on: ubuntu-latest + needs: changes + if: ${{ github.event_name != 'workflow_dispatch' && needs.changes.outputs.only-docs == 'true' }} + steps: + - run: echo "faux-unittest-complete" + + unittest-complete: + runs-on: ubuntu-latest + needs: [ multiplex-unittests, faux-unittest ] + if: always() + steps: + - name: Check Results + if: >- + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') + run: exit 1 + - run: echo "unittest-complete" diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..9d611ec --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,53 @@ +# Multiplex - Parameter Expansion Engine + +## Purpose +Multiplex translates multi-value benchmark parameters into single-value test matrices. Given a set of parameters with multiple possible values, it produces all valid combinations (the Cartesian product), respecting parameter roles and constraints. Output feeds into rickshaw-run as `bench-params.json`. + +## Language +Python 3 — single main file `multiplex.py` (~510 lines), no dependencies beyond `jsonschema`. + +## Key Files +| File | Purpose | +|------|---------| +| `multiplex.py` | Core parameter expansion logic and CLI entry point | +| `params` | Default CLI arguments file (read by argparse `fromfile_prefix_chars`) | +| `JSON/schema.json` | JSON schema for multi-value input validation | +| `JSON/req-schema.json` | JSON schema for requirements file validation | + +## Key Functions +| Function | Purpose | +|----------|---------| +| `multiplex_sets()` | Main driver: expands all parameter sets | +| `multiplex_set()` | Expands one multi-value set into multiple single-value sets (Cartesian product via `itertools.product`) | +| `load_param_sets()` | Resolves `include` references from global-options into sets | +| `override_presets()` | Applies defaults, essentials, and named presets from requirements | +| `transform_param_val()` | Applies validation regex, unit conversion, and regex transformation | +| `validate_schema()` | Validates input JSON against schema | + +## Exit Codes +| Code | Constant | Meaning | +|------|----------|---------| +| 0 | `EC_SUCCESS` | Success | +| 1 | `EC_SCHEMA_FAIL` | Input JSON schema validation failed | +| 2 | `EC_JSON_FAIL` | JSON file loading failed | +| 3 | `EC_REQUIREMENTS_FAIL` | Requirements file loading failed | +| 4 | `EC_VALIDATIONS_FAIL` | Parameter validation failed | +| 5 | `EC_REQ_SCHEMA_FAIL` | Requirements JSON schema validation failed | +| 6 | `EC_EMPTY_SET_FAIL` | Empty param set after preset override | + +## Input/Output +- **Input**: JSON with `global-options` and `sets` arrays; params use `arg` and `vals` (multi-value list) keys, optional `role` (client/server/all), `id`, `enabled` +- **Output**: JSON array of arrays; each inner array is one test iteration with single-value params using `arg`, `val`, and `role` keys +- **Requirements** (optional): Defines `presets` (defaults, essentials, named), `validations` (regex), and `units` (conversion factors) + +## Tests +Run with `cd` to project root, then `pytest tests/`: +- `tests/test-json.py` — Parameter loading, multiplexing, Cartesian expansion, value conversion +- `tests/test-requirements.py` — Validation regex, presets, unit conversion, preset precedence +- `tests/test-schema.py` — Schema validation for valid and invalid inputs +- 60+ JSON fixtures in `tests/JSON/` + +## Conventions +- 4-space indentation, PEP 8 naming +- Module-level dicts for state: `validation_dict`, `convert_dict`, `transform_dict`, `presets_dict` +- Uses Python `logging` module for debug/info/warning/error output diff --git a/README.md b/README.md index 1d8806b..6cf677d 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,11 @@ When running a benchmark, it is often desirable to run it multiple ways, changin ## Usage ``` -./multiplex.py [--requirements JSON/requirements.json] --input JSON/mv-params-input.json [--output /path/to/bench-params.json] +./multiplex.py [--requirements JSON/requirements.json] --input JSON/mv-params-input.json [--output /path/to/bench-params.json] [--debug] ``` +Default CLI arguments can be placed in a `params` file in the same directory, one argument per line (read via argparse `fromfile_prefix_chars`). + ## Input multi-value JSON Multiplex requires a JSON file with the following format: ``` @@ -274,3 +276,14 @@ parameters to STDOUT. A sample is available in `JSON/bench-params-output.json`: This JSON can then optionally be modified by the user, and then provided (typically as "bench-params.json") to a benchmark orchestrator like rickshaw-run. + +## Exit codes +| Code | Meaning | +|------|---------| +| 0 | Success | +| 1 | Input JSON schema validation failed | +| 2 | JSON file loading failed | +| 3 | Requirements file loading failed | +| 4 | Parameter validation failed | +| 5 | Requirements JSON schema validation failed | +| 6 | Empty param set after preset override (missing essentials/defaults) |