diff --git a/.github/workflows/trigger-labs-tests.yml b/.github/workflows/trigger-labs-tests.yml new file mode 100644 index 0000000..91abb57 --- /dev/null +++ b/.github/workflows/trigger-labs-tests.yml @@ -0,0 +1,250 @@ +name: Trigger labs-tests + +on: + push: + branches: + - main + workflow_dispatch: + inputs: + labs_branch: + description: "Optional branch of specmatic/labs to test. Defaults to the branch this workflow runs on." + required: false + type: string + labs_tests_branch: + description: "Branch of specmatic/labs-tests whose workflows should run. Defaults to main." + required: false + default: "main" + type: string + labs: + description: "Optional space-separated list of labs to run in the main labs-tests workflow. Leave empty to run the full suite." + required: false + type: string + +permissions: + contents: read + +jobs: + dispatch-labs-tests: + runs-on: ubuntu-latest + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - workflow_id: labs-tests.yml + workflow_label: Main labs-tests workflow + artifact_name: specmatic-labs-reports + accepts_labs_input: true + - workflow_id: labs-tests-readme-pilot.yml + workflow_label: README-driven pilot workflow + artifact_name: specmatic-labs-readme-pilot-reports + accepts_labs_input: false + env: + TARGET_OWNER: specmatic + TARGET_REPO: labs-tests + EFFECTIVE_LABS_BRANCH: ${{ github.event.inputs.labs_branch || github.ref_name }} + EFFECTIVE_LABS_TESTS_BRANCH: ${{ github.event.inputs.labs_tests_branch || 'main' }} + SELECTED_LABS: ${{ github.event.inputs.labs || '' }} + steps: + - name: Validate labs-tests dispatch token + env: + TOKEN_PRESENT: ${{ secrets.SPECMATIC_GITHUB_TOKEN != '' }} + run: | + if [[ "${TOKEN_PRESENT}" != "true" ]]; then + echo "::error::Missing SPECMATIC_GITHUB_TOKEN secret in specmatic/labs." + exit 1 + fi + + - name: Dispatch downstream workflow + id: dispatch + uses: actions/github-script@v7 + env: + TARGET_WORKFLOW_ID: ${{ matrix.workflow_id }} + WORKFLOW_LABEL: ${{ matrix.workflow_label }} + ACCEPTS_LABS_INPUT: ${{ matrix.accepts_labs_input }} + with: + github-token: ${{ secrets.SPECMATIC_GITHUB_TOKEN }} + script: | + const owner = process.env.TARGET_OWNER; + const repo = process.env.TARGET_REPO; + const workflow_id = process.env.TARGET_WORKFLOW_ID; + const workflowLabel = process.env.WORKFLOW_LABEL; + const ref = process.env.EFFECTIVE_LABS_TESTS_BRANCH; + const labsBranch = process.env.EFFECTIVE_LABS_BRANCH; + const selectedLabs = process.env.SELECTED_LABS; + const acceptsLabsInput = process.env.ACCEPTS_LABS_INPUT === 'true'; + const dispatchStartedAt = new Date(); + const inputs = { labs_branch: labsBranch }; + if (acceptsLabsInput) { + inputs.labs = selectedLabs; + } + + core.info(`Dispatching ${workflowLabel}: ${owner}/${repo}:${workflow_id}`); + core.info(`labs branch: ${labsBranch}`); + core.info(`labs-tests branch: ${ref}`); + core.info(`selected labs: ${selectedLabs || '(all labs)'}`); + + await github.rest.actions.createWorkflowDispatch({ + owner, + repo, + workflow_id, + ref, + inputs, + }); + + let run = null; + for (let attempt = 1; attempt <= 30; attempt += 1) { + const response = await github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id, + branch: ref, + event: 'workflow_dispatch', + per_page: 20, + }); + run = response.data.workflow_runs.find(candidate => { + const createdAt = new Date(candidate.created_at); + return createdAt >= new Date(dispatchStartedAt.getTime() - 60_000); + }); + if (run) { + break; + } + core.info(`Waiting for ${workflowLabel} run to appear (attempt ${attempt}/30)...`); + await new Promise(resolve => setTimeout(resolve, 10000)); + } + + if (!run) { + core.setFailed(`Unable to locate the dispatched ${workflowLabel} run.`); + return; + } + + core.info(`Matched ${workflowLabel} run: ${run.html_url}`); + core.setOutput('run_id', String(run.id)); + core.setOutput('run_url', run.html_url); + + - name: Wait for downstream workflow completion + id: wait + uses: actions/github-script@v7 + env: + RUN_ID: ${{ steps.dispatch.outputs.run_id }} + WORKFLOW_LABEL: ${{ matrix.workflow_label }} + with: + github-token: ${{ secrets.SPECMATIC_GITHUB_TOKEN }} + script: | + const owner = process.env.TARGET_OWNER; + const repo = process.env.TARGET_REPO; + const run_id = Number(process.env.RUN_ID); + const workflowLabel = process.env.WORKFLOW_LABEL; + + if (!run_id) { + core.setFailed(`Missing downstream workflow run id for ${workflowLabel}.`); + return; + } + + let run = null; + for (let attempt = 1; attempt <= 180; attempt += 1) { + const response = await github.rest.actions.getWorkflowRun({ owner, repo, run_id }); + run = response.data; + core.info(`${workflowLabel} status: ${run.status}, conclusion: ${run.conclusion || 'n/a'} (attempt ${attempt}/180)`); + if (run.status === 'completed') { + break; + } + await new Promise(resolve => setTimeout(resolve, 30000)); + } + + if (!run || run.status !== 'completed') { + core.setFailed(`Timed out waiting for ${workflowLabel} to finish.`); + return; + } + + core.setOutput('conclusion', run.conclusion || 'unknown'); + core.setOutput('run_url', run.html_url); + core.setOutput('run_number', String(run.run_number)); + + - name: Download downstream reports artifact + env: + GH_TOKEN: ${{ secrets.SPECMATIC_GITHUB_TOKEN }} + RUN_ID: ${{ steps.dispatch.outputs.run_id }} + ARTIFACT_NAME: ${{ matrix.artifact_name }} + WORKFLOW_ID: ${{ matrix.workflow_id }} + run: | + set -euo pipefail + rm -rf downstream-artifact + mkdir -p downstream-artifact + gh run download "$RUN_ID" --repo "${TARGET_OWNER}/${TARGET_REPO}" --name "$ARTIFACT_NAME" --dir downstream-artifact + + - name: Build summary from downstream result + env: + RUN_URL: ${{ steps.wait.outputs.run_url }} + RUN_CONCLUSION: ${{ steps.wait.outputs.conclusion }} + RUN_NUMBER: ${{ steps.wait.outputs.run_number }} + SELECTED_LABS: ${{ env.SELECTED_LABS }} + EFFECTIVE_LABS_BRANCH: ${{ env.EFFECTIVE_LABS_BRANCH }} + EFFECTIVE_LABS_TESTS_BRANCH: ${{ env.EFFECTIVE_LABS_TESTS_BRANCH }} + WORKFLOW_LABEL: ${{ matrix.workflow_label }} + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + import os + from pathlib import Path + + root = Path('downstream-artifact') + candidates = [ + root / 'output' / 'consolidated-report' / 'consolidated-report.json', + root / 'consolidated-report' / 'consolidated-report.json', + root / 'output' / 'consolidated-report.json', + ] + report_path = next((path for path in candidates if path.exists()), None) + + workflow_label = os.environ['WORKFLOW_LABEL'] + run_url = os.environ['RUN_URL'] + conclusion = os.environ.get('RUN_CONCLUSION', 'unknown') + run_number = os.environ.get('RUN_NUMBER', 'n/a') + selected_labs = os.environ.get('SELECTED_LABS', '') or '(all labs)' + labs_branch = os.environ.get('EFFECTIVE_LABS_BRANCH', 'main') + labs_tests_branch = os.environ.get('EFFECTIVE_LABS_TESTS_BRANCH', 'main') + + print(f'## {workflow_label}') + print('') + print(f'- labs branch: `{labs_branch}`') + print(f'- labs-tests branch: `{labs_tests_branch}`') + print(f'- selected labs: `{selected_labs}`') + print(f'- labs-tests run: [#{run_number}]({run_url})') + print(f'- conclusion: **{conclusion.upper()}**') + + if report_path is None: + print('') + print('No consolidated report was found in the downloaded downstream artifact.') + raise SystemExit(0) + + payload = json.loads(report_path.read_text(encoding='utf-8')) + summary = {item['label']: item['value'] for item in payload.get('summary', [])} + environment = payload.get('environment', {}) + print('') + print(f'- Specmatic: `{environment.get("specmaticVersion", "n/a")}`') + print(f'- labs discovered: `{summary.get("Labs discovered", 0)}`') + print(f'- labs passed: `{summary.get("Labs passed", 0)}`') + print(f'- labs failed: `{summary.get("Labs failed", 0)}`') + print('') + print('| Lab | Status | Time (s) | Exit Code | Failures | Validations |') + print('| --- | --- | ---: | ---: | ---: | ---: |') + for lab in payload.get('labs', []): + failures = 'n/a' + validations = 'n/a' + for item in lab.get('summary', []): + if item.get('label') == 'Failures': + failures = item.get('value', 'n/a') + if item.get('label') == 'Validations': + validations = item.get('value', 'n/a') + print( + f"| {lab.get('name', 'unknown')} | {lab.get('status', 'unknown')} | " + f"{lab.get('durationSeconds', 'n/a')} | {lab.get('exitCode', 'n/a')} | " + f"{failures} | {validations} |" + ) + PY + + - name: Fail when downstream workflow fails + if: steps.wait.outputs.conclusion != 'success' + run: | + echo "Downstream workflow failed: ${{ steps.wait.outputs.run_url }}" + exit 1 diff --git a/LAB_AUTOMATION_COMMANDS_SUMMARY.md b/LAB_AUTOMATION_COMMANDS_SUMMARY.md new file mode 100644 index 0000000..411d6cc --- /dev/null +++ b/LAB_AUTOMATION_COMMANDS_SUMMARY.md @@ -0,0 +1,31 @@ +# Lab Learner Tasks Summary + +This summary lists the learner task attached to each lab README, excluding `order-bff`, `coding-agents`, and `arazzo-workflow-testing`. + +Command cells use HTML `
...
` blocks because raw triple-backtick fences break Markdown tables. + +| Lab | Task | Command | Status | Files Changed | Notes | +|---|---|---|---|---|---| +| `api-coverage` | Fix `specs/service.yaml`: change `GET /pets/search` to `GET /pets/find`. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#/pets/search:$#/pets/find:#' specs/service.yaml"
| PASS | `api-coverage/specs/service.yaml` | Changed `/pets/search` to `/pets/find`. | +| `api-resiliency-testing` | Fix downstream timeout mock examples, then enable full schema resiliency: add transient delays in `examples/order-service/stub_timeout_get_products.json` and `stub_timeout_post_product.json`, change `schemaResiliencyTests` to `all`, then update `stub_timeout_post_product.json` with `value:each` matchers. | Not simple enough for a safe one-liner; it needs multiple structured JSON edits. | Skipped | | No concrete command to run. | +| `api-security-schemes` | Fix `specmatic.yaml` auth values: replace invalid OAuth token, Basic Auth value, and API key with valid values. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#INVALID_OAUTH_TOKEN#OAUTH_TOKEN#g; s#dXNlcjppbnZhbGlkcGFzcw==#dXNlcjpwYXNzd29yZA==#g; s#INVALID_APIKEY1234#APIKEY1234#g' specmatic.yaml"
| PASS | `api-security-schemes/specmatic.yaml` | Replaced the invalid OAuth token env key, Basic Auth fallback token, and API key fallback. | +| `async-event-flow` | Fix async examples: add missing `before` fixture in `acceptOrder.json` and correct the `after` fixture count in `outForDeliveryOrder.json`. | Not simple enough for a safe one-liner; it needs inserting a multi-line fixture plus another JSON edit. | Skipped | | No concrete command to run. | +| `backward-compatibility-testing` | In `products.yaml`, keep the new `category` field but change `name` back from `number` to `string` to restore backward compatibility. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i '/^                  name:$/,/^                  sku:$/ s#type: number#type: string#' products.yaml"
| No-op | | Current checkout is not in the post-Part-A broken state; `name` is already `string`, so there was no diff to verify. | +| `continuous-integration` | In `contracts/order_api.yaml`, make CI pass by removing `priority` from the required list while keeping the `priority` property. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "awk 'BEGIN{removed=0} {if (removed==0 && \$0==\"                - priority\") {removed=1; next} print}' contracts/order_api.yaml > /tmp/order_api.yaml && mv /tmp/order_api.yaml contracts/order_api.yaml"
| PASS | `continuous-integration/contracts/order_api.yaml` | Removed only the first `- priority` from the `POST /orders` required list. Replaced the original sed command because it exited zero without changing the file, and removed `!` so zsh does not trigger history expansion. | +| `data-adapters` | Wire existing request/response adapter hooks in `specmatic.yaml` and ensure hook scripts are executable. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i '/^specmatic:/i\  data:\n    adapters:\n      pre_specmatic_request_processor: ./hooks/pre_specmatic_request_processor.sh\n      post_specmatic_response_processor: ./hooks/post_specmatic_response_processor.sh\n' specmatic.yaml && chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh"
| PASS | `data-adapters/specmatic.yaml` | Added `dependencies.data.adapters` with both hook paths. Hook chmod produced no tracked diff because executable bits were already set. | +| `dictionary` | Generate `specs/dictionary.yaml` from examples and configure `specmatic.yaml` to use it for dictionary-driven mock data. | 
docker run --rm -v "$PWD:/usr/src/app" specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i '/^specmatic:/i\        data:\n          dictionary:\n            path: specs/dictionary.yaml\n' specmatic.yaml"
| PASS | `dictionary/specs/dictionary.yaml`, `dictionary/specmatic.yaml` | Generated `specs/dictionary.yaml` and added `dependencies.services[0].service.data.dictionary.path`. | +| `external-examples` | Fix invalid external examples under `examples/`, then generate missing `201` examples for create APIs in Studio. | No simple command; the lab includes Studio validation/fix flow and example generation. | Skipped | | No concrete command to run. | +| `filters` | Use Studio to exclude failing scenarios and uncovered `429` scenarios, then export/persist filters into `specmatic.yaml`. | No simple command; the task depends on Studio selections and exported filter config. | Skipped | | No concrete command to run. | +| `kafka-avro` | Update Avro schemas with missing constraints, then update AsyncAPI example messages so they satisfy those constraints. | Not simple enough for a safe one-liner; it replaces larger structured JSON schema and example documents. | Skipped | | No concrete command to run. | +| `kafka-sqs-retry-dlq` | Fix `service/app.py` by enabling the retry consumer thread so retry-topic messages are consumed and reprocessed. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's@            # threading.Thread(target=self._run_retry_consumer, name=\"RetryConsumer\", daemon=True),@            threading.Thread(target=self._run_retry_consumer, name=\"RetryConsumer\", daemon=True),@' service/app.py"
| PASS | `kafka-sqs-retry-dlq/service/app.py` | Uncommented the `RetryConsumer` thread entry. | +| `mcp-auto-test` | Fix two bugs in `service/order_service.py`: shipment status lookup and return quote reason-to-fee lookup for `damaged`. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#\"damage\": 0.0#\"damaged\": 0.0#; s#order\\[\"shipment\"\\]#order[\"shipmentStatus\"]#' service/order_service.py"
| PASS | `mcp-auto-test/service/order_service.py` | Replaced `damage` with `damaged` and fixed shipment status lookup. | +| `overlays` | Update `overlays/path-prefix.overlay.yaml` to replace `/api/users/{id}` with `/api/v1/users/{id}`, then enable the overlay in `specmatic.yaml`. | Not simple enough for a safe one-liner; it needs a structured overlay action block and config change. | Skipped | | No concrete command to run. | +| `partial-examples` | Fix three examples using partial examples: two create-flow examples and one product search example. | No simple command; the lab intentionally uses partial-example conversion and Studio validation. | Skipped | | No concrete command to run. | +| `quick-start-api-testing` | Fix two response examples: loosen `decision` with a pattern, then use `dataType: date` and a `VRF-[0-9]{6}` pattern for dynamic fields. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#[$]match(exact: approved)#\$match(pattern: approved\|verified)#' examples/test_finance_user_11.json && sed -i 's#[$]match(exact: VRF-123456)#\$match(pattern: VRF-[0-9]{6})#; s#[$]match(exact: 2026-03-17)#\$match(dataType: date)#' examples/test_support_user_55.json"
| PASS | `quick-start-api-testing/examples/test_finance_user_11.json`, `quick-start-api-testing/examples/test_support_user_55.json` | Replaced exact matchers with the intended pattern/date matchers. Corrected the command to preserve literal `$match` in replacements. | +| `quick-start-async-contract-testing` | Fix `service/processor.py`: change emitted status from `STARTED` to `INITIATED`. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#\"status\": \"STARTED\"#\"status\": \"INITIATED\"#' service/processor.py"
| PASS | `quick-start-async-contract-testing/service/processor.py` | Changed emitted status from `STARTED` to `INITIATED`. | +| `quick-start-contract-testing` | Fix `service/server.py`: return response field `type` instead of `petType`. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#\"petType\": \"Golden Retriever\"#\"type\": \"Golden Retriever\"#' service/server.py"
| PASS | `quick-start-contract-testing/service/server.py` | Changed response key from `petType` to `type`. | +| `quick-start-mock` | No file-fix task. The attached learner task is operational: start consumer, observe failure, start mock, observe success, stop mock, then use Studio-generated examples. | No file edit command; this is an operational mock/consumer exercise. | Skipped | | No concrete command to run. | +| `response-templating` | Fix mock examples under `examples/mock/`: direct-substitute order response fields from request body and add lookup-driven product search responses. | Not simple enough for a safe one-liner; it needs response-template and lookup-driven JSON edits. | Skipped | | No concrete command to run. | +| `schema-design` | Update `specs/payment-api.yaml`: split `PaymentRequest` into card and bank-transfer schemas, then use `oneOf` with discriminator `paymentType`. | Not simple enough for a safe one-liner; it is a larger OpenAPI schema redesign. | Skipped | | No concrete command to run. | +| `schema-resiliency-testing` | No bug-fix task. The attached learner task is config exploration: change `schemaResiliencyTests` from `none` to `positiveOnly`, then to `all`, and compare generated test counts. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: none#schemaResiliencyTests: positiveOnly#' specmatic.yaml"
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: positiveOnly#schemaResiliencyTests: all#' specmatic.yaml"
| PASS | `schema-resiliency-testing/specmatic.yaml` | Sequentially changed `schemaResiliencyTests` from `none` to `positiveOnly`, then to `all`. Final diff shows `all`. | +| `workflow-in-same-spec` | Add workflow ID propagation mapping in `specmatic.yaml` so IDs from `POST /tasks` flow into `GET`, `PUT`, and `DELETE`. | 
docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i '/^specmatic:/i\        workflow:\n          ids:\n            \"POST /tasks -> 200\":\n              extract: \"BODY.tasks.[0].id\"\n            \"GET /tasks/(task_id:string) -> 200\":\n              use: \"PATH.task_id\"\n            \"PUT /tasks/(task_id:string) -> 200\":\n              use: \"PATH.task_id\"\n            \"DELETE /tasks/(task_id:string) -> 204\":\n              use: \"PATH.task_id\"\n' specmatic.yaml"
| PASS | `workflow-in-same-spec/specmatic.yaml` | Added `workflow.ids` mapping under `systemUnderTest.service.runOptions.openapi`. | diff --git a/api-coverage/README.md b/api-coverage/README.md index f7a9c5c..43f9cb2 100644 --- a/api-coverage/README.md +++ b/api-coverage/README.md @@ -70,7 +70,9 @@ This lab uses FastAPI, which can publish OpenAPI dynamically at `/openapi.json`. The important point for API coverage is not the specific library. What matters is that the application exposes a current OpenAPI document that Specmatic can fetch using `swaggerUrl`. -## 1. Baseline run (intentional coverage failure) +## Lab Implementation Phases + +### Baseline Phase Run: ```shell @@ -107,7 +109,6 @@ Expected gate failure highlight: Failed the following API Coverage Report success criteria: Total API coverage: 50% is less than the specified minimum threshold of 100%. Total missed operations: 1 is greater than the maximum threshold of 0. - ``` Clean up: @@ -127,7 +128,7 @@ Example baseline CTRF HTML report: ![Baseline CTRF HTML report](assets/api-coverage-html-report.png) -## 2. Fix the checked-in spec +### Task A: Fix the checked-in spec Open `specs/service.yaml`. Find this path: @@ -146,6 +147,12 @@ Change it to: Do not change anything else in the operation. +Alternative run command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's|/pets/search:|/pets/find:|' specs/service.yaml" +``` + ## 3. Re-run the tests and coverage check Run the same command again: @@ -186,7 +193,8 @@ The HTML report is useful both before and after the fix: - before the fix, it shows the coverage mismatch in a more readable UI - after the fix, it confirms both paths are covered -## Short Studio follow-up +## Studio verification + Start Studio and the provider: ```shell diff --git a/api-coverage/docker-compose.yaml b/api-coverage/docker-compose.yaml index 5365dff..5d57c8e 100644 --- a/api-coverage/docker-compose.yaml +++ b/api-coverage/docker-compose.yaml @@ -14,7 +14,7 @@ services: test: container_name: api-coverage-openapi-test - image: specmatic/enterprise:latest + image: ${SPECMATIC_ENTERPRISE_IMAGE:-specmatic/enterprise:latest} volumes: - ./:/usr/src/app - ../license.txt:/specmatic/specmatic-license.txt:ro @@ -25,7 +25,7 @@ services: studio: container_name: api-coverage-openapi-studio - image: specmatic/enterprise:latest + image: ${SPECMATIC_ENTERPRISE_IMAGE:-specmatic/enterprise:latest} profiles: ["studio"] volumes: - ./:/usr/src/app diff --git a/api-security-schemes/README.md b/api-security-schemes/README.md index 6a0e2d1..37f76dd 100644 --- a/api-security-schemes/README.md +++ b/api-security-schemes/README.md @@ -132,6 +132,12 @@ Update [`specmatic.yaml`](specmatic.yaml) to valid values again: Do not change anything else. Fix only the above values. +Alternative run command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#INVALID_OAUTH_TOKEN#OAUTH_TOKEN#g; s#dXNlcjppbnZhbGlkcGFzcw==#dXNlcjpwYXNzd29yZA==#g; s#INVALID_APIKEY1234#APIKEY1234#g' specmatic.yaml" +``` + ## Verify the fix Re-run: diff --git a/continuous-integration/README.md b/continuous-integration/README.md index 0525880..c5aa5b4 100644 --- a/continuous-integration/README.md +++ b/continuous-integration/README.md @@ -139,6 +139,12 @@ Keep: Do not change anything else. +Alternative run command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "awk 'BEGIN{done=0} !done && \$0==\" - priority\" {done=1; next} {print}' contracts/order_api.yaml > /tmp/order_api.yaml && mv /tmp/order_api.yaml contracts/order_api.yaml" +``` + ## Part C: Re-run the CI simulation Run the same command again: diff --git a/data-adapters/README.md b/data-adapters/README.md index 332e537..8613141 100644 --- a/data-adapters/README.md +++ b/data-adapters/README.md @@ -94,6 +94,12 @@ Why both hooks are needed: - `pre_specmatic_request_processor`: adapts PascalCase request fields to camelCase before sending to mock. - `post_specmatic_response_processor`: adapts camelCase mock response fields back to PascalCase before assertion. +Alternative run command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n adapters:\n pre_specmatic_request_processor: ./hooks/pre_specmatic_request_processor.sh\n post_specmatic_response_processor: ./hooks/post_specmatic_response_processor.sh\n' specmatic.yaml && chmod +x hooks/pre_specmatic_request_processor.sh hooks/post_specmatic_response_processor.sh" +``` + ## 5. Ensure hook scripts are executable Run: diff --git a/dictionary/README.md b/dictionary/README.md index 86bd0fd..6935d4b 100644 --- a/dictionary/README.md +++ b/dictionary/README.md @@ -59,6 +59,13 @@ data: path: specs/dictionary.yaml ``` +Alternative run command: + +```shell +docker run --rm -v "$PWD:/usr/src/app" specmatic/enterprise examples dictionary --examples-dir examples --spec-file specs/simple-openapi-spec.yaml --out specs/dictionary.yaml +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i '/^specmatic:/i\ data:\n dictionary:\n path: specs/dictionary.yaml\n' specmatic.yaml" +``` + ## 3. Re-run the suite after configuring dictionary ```shell diff --git a/kafka-sqs-retry-dlq/README.md b/kafka-sqs-retry-dlq/README.md index 10db3cb..ca79576 100644 --- a/kafka-sqs-retry-dlq/README.md +++ b/kafka-sqs-retry-dlq/README.md @@ -98,6 +98,12 @@ What to look for: Do not change the contract, examples, or Compose wiring. +Alternative run command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's@ # threading.Thread(target=self._run_retry_consumer, name=\"RetryConsumer\", daemon=True),@ threading.Thread(target=self._run_retry_consumer, name=\"RetryConsumer\", daemon=True),@' service/app.py" +``` + ## Pass criteria Re-run: diff --git a/mcp-auto-test/README.md b/mcp-auto-test/README.md index 6482440..e5aefcb 100644 --- a/mcp-auto-test/README.md +++ b/mcp-auto-test/README.md @@ -98,6 +98,12 @@ Make these two fixes: Do not change anything else. +Alternative run command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#\"damage\": 0.0#\"damaged\": 0.0#; s#order\\[\"shipment\"\\]#order[\"shipmentStatus\"]#' service/order_service.py" +``` + ## Part C: Re-run tests (expected to pass) Run: diff --git a/quick-start-api-testing/README.md b/quick-start-api-testing/README.md index 6694d33..cc49b36 100644 --- a/quick-start-api-testing/README.md +++ b/quick-start-api-testing/README.md @@ -53,7 +53,7 @@ This lab shows how to express each of those expectations with the right matcher ## Lab Implementation Phases -### Baseline Phase +## Baseline Phase Your verification service returns: - `handledBy`, which is always `verification-service` - `decision`, which may be `approved` or `verified` @@ -91,7 +91,7 @@ docker compose down -v - `test_finance_user_11.json` expects `decision` to be exactly `approved`, but the service may return `approved` or `verified` for that request. - `test_support_user_55.json` expects one hardcoded date and one exact reference code, but the service generates fresh valid values every time. -### Intermediate Phase: Task A +## Task A: Use pattern for flexible decision values Edit `examples/test_finance_user_11.json`. In `http-response.body`, change: @@ -99,6 +99,12 @@ In `http-response.body`, change: Do not change any other fields. +Alternative run command for Task A: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#[$]match(exact: approved)#\$match(pattern: approved\|verified)#' examples/test_finance_user_11.json" +``` + Re-run: ```shell @@ -117,7 +123,8 @@ Clean up: docker compose down -v ``` -### Final Phase +## Task B: Use dataType and pattern for dynamic values + Use `dataType` and `pattern` for dynamic values Edit `examples/test_support_user_55.json`. @@ -128,6 +135,12 @@ In `http-response.body`, change: Keep `handledBy` and `decision` as exact matches. +Alternative run command for Final Phase: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#[$]match(exact: VRF-123456)#\$match(pattern: VRF-[0-9]{6})#; s#[$]match(exact: 2026-03-17)#\$match(dataType: date)#' examples/test_support_user_55.json" +``` + Run: ```shell diff --git a/quick-start-api-testing/docker-compose.yaml b/quick-start-api-testing/docker-compose.yaml index 7b1dbaa..0c772a0 100644 --- a/quick-start-api-testing/docker-compose.yaml +++ b/quick-start-api-testing/docker-compose.yaml @@ -12,7 +12,7 @@ services: api-test: container_name: api-test - image: specmatic/enterprise:latest + image: ${SPECMATIC_ENTERPRISE_IMAGE:-specmatic/enterprise:latest} volumes: - ./:/usr/src/app - ../license.txt:/specmatic/specmatic-license.txt:ro diff --git a/quick-start-async-contract-testing/README.md b/quick-start-async-contract-testing/README.md index 5950bbb..1ad506b 100644 --- a/quick-start-async-contract-testing/README.md +++ b/quick-start-async-contract-testing/README.md @@ -73,6 +73,12 @@ To: "status": "INITIATED", ``` +Alternative run command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#\"status\": \"STARTED\"#\"status\": \"INITIATED\"#' service/processor.py" +``` + ## Pass criteria Re-run: diff --git a/quick-start-contract-testing/README.md b/quick-start-contract-testing/README.md index 90edf76..62cdc7b 100644 --- a/quick-start-contract-testing/README.md +++ b/quick-start-contract-testing/README.md @@ -108,6 +108,12 @@ with: Do not change anything else. +Alternative run command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#\"petType\": \"Golden Retriever\"#\"type\": \"Golden Retriever\"#' service/server.py" +``` + ## Part C: Re-run tests (expected to pass) Run: diff --git a/schema-resiliency-testing/README.md b/schema-resiliency-testing/README.md index df1e1b2..ae6a732 100644 --- a/schema-resiliency-testing/README.md +++ b/schema-resiliency-testing/README.md @@ -69,6 +69,12 @@ The goal of this lab is to try different schema resiliency testing levels and se ### Positive Only Tests In `specmatic.yaml` change `schemaResiliencyTests: none` to `schemaResiliencyTests: positiveOnly` +Alternative run command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: none#schemaResiliencyTests: positiveOnly#' specmatic.yaml" +``` + #### Run Positive only Tests Start docker containers @@ -92,6 +98,12 @@ docker compose down -v ### Positive and Negative Tests (ALL) In `specmatic.yaml` change `schemaResiliencyTests: positiveOnly` to `schemaResiliencyTests: all` +Alternative run command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i 's#schemaResiliencyTests: positiveOnly#schemaResiliencyTests: all#' specmatic.yaml" +``` + #### Run all Tests Start docker containers diff --git a/workflow-in-same-spec/README.md b/workflow-in-same-spec/README.md index 2f47f9f..1e001aa 100644 --- a/workflow-in-same-spec/README.md +++ b/workflow-in-same-spec/README.md @@ -105,6 +105,12 @@ workflow: use: "PATH.task_id" ``` +Alternative run command: + +```shell +docker run --rm --entrypoint sh -v "$PWD:/work" -w /work specmatic/enterprise -lc "sed -i '/^specmatic:/i\ workflow:\n ids:\n \"POST /tasks -> 200\":\n extract: \"BODY.tasks.[0].id\"\n \"GET /tasks/(task_id:string) -> 200\":\n use: \"PATH.task_id\"\n \"PUT /tasks/(task_id:string) -> 200\":\n use: \"PATH.task_id\"\n \"DELETE /tasks/(task_id:string) -> 204\":\n use: \"PATH.task_id\"\n' specmatic.yaml" +``` + Re-run: ```shell