Harden GitHub reporting and Action CI

.github/workflows/test.yml

@@ -5,12 +5,85 @@ on:
   push:
     branches:
       - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: test-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  shell:
+  harness:
+    name: Harness / ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+
+      - name: Validate syntax
+        run: |
+          bash -n scripts/*.sh
+          node --check scripts/test-preflight-server.cjs
+
+      - name: Run ShellCheck
+        if: runner.os == 'Linux'
+        run: shellcheck scripts/*.sh
+
+      - name: Run actionlint
+        if: runner.os == 'Linux'
+        run: >-
+          docker run --rm -v "$PWD:/repo" -w /repo
+          rhysd/actionlint@sha256:887a259a5a534f3c4f36cb02dca341673c6089431057242cdc931e9f133147e9
+
+      - name: Run command behavior tests
+        run: ./scripts/test.sh
+
+      - name: Run integration tests
+        run: ./scripts/test-integration.sh
+
+  composite-action:
+    name: Composite action contract
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Invoke the local Action
+        uses: ./
+        env:
+          CHECKLY_ACTION_DRY_RUN: "1"
+          CHECKLY_ACTION_GITHUB_REPORT_AVAILABLE: "true"
+        with:
+          command: test
+          cli-version: 8.15.0
+          reporting: github-check
+          github-check-name: Checkly action contract
+          tags: smoke
+
+  ci:
+    name: CI
+    if: always()
+    needs:
+      - harness
+      - composite-action
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - run: bash -n scripts/run.sh
-      - run: bash -n scripts/test.sh
-      - run: ./scripts/test.sh
+      - name: Verify required jobs
+        env:
+          HARNESS_RESULT: ${{ needs.harness.result }}
+          COMPOSITE_ACTION_RESULT: ${{ needs.composite-action.result }}
+        run: |
+          if [[ "$HARNESS_RESULT" != "success" || "$COMPOSITE_ACTION_RESULT" != "success" ]]; then
+            echo "Required CI failed: harness=$HARNESS_RESULT composite-action=$COMPOSITE_ACTION_RESULT"
+            exit 1
+          fi

AGENTS.md

@@ -0,0 +1,71 @@
+# Checkly GitHub Action
+
+This repository publishes a composite GitHub Action. There is no build step or
+generated distribution bundle: consumers execute `action.yml` and
+`scripts/run.sh` directly from the selected Git ref.
+
+## Source Of Truth
+
+- `action.yml` is the public input/output contract.
+- `scripts/run.sh` validates inputs, selects the reporting mode, performs the
+  preflight request, invokes the Checkly CLI, and writes Action outputs.
+- `README.md` documents the public contract and must stay aligned with
+  `action.yml`.
+- `scripts/test.sh` covers validation and command construction through the
+  script's public environment-variable interface.
+- `scripts/test-integration.sh` covers the real preflight, CLI invocation,
+  working directory, install command, outputs, summaries, and exit codes using
+  local fakes. It must never require Checkly or GitHub credentials.
+
+## Product Invariants
+
+- Only `checkly test` and `checkly trigger` are supported. Do not turn `command`
+  into an arbitrary shell or CLI escape hatch.
+- CLI options remain separate Action inputs so workflows do not need to build
+  command strings.
+- `reporting: auto` detaches only after the backend preflight confirms GitHub
+  Check writeback is available. Otherwise it waits and reports through GitHub
+  Actions.
+- `reporting: github-check` fails before scheduling checks when writeback is not
+  available. `reporting: github-actions` never detaches.
+- The Action requires Checkly CLI `8.15.0` or newer. A pinned older stable
+  version must fail before running or performing a preflight request.
+- Exact stable semver pins are compared against the minimum version. Dist-tags,
+  ranges, canaries, and prereleases are allowed because they may identify a
+  compatible build before the next stable release.
+- Repository and SHA values sent by the Action are hints, not authorization.
+  The Checkly backend must continue verifying account-scoped GitHub App access.
+- Never print API keys or other secrets. Sanitize backend-provided values before
+  placing them in GitHub workflow commands or summaries.
+
+## Verification
+
+Run all of these before pushing:
+
+```bash
+bash -n scripts/*.sh
+node --check scripts/test-preflight-server.cjs
+shellcheck scripts/*.sh
+docker run --rm -v "$PWD:/repo" -w /repo rhysd/actionlint@sha256:887a259a5a534f3c4f36cb02dca341673c6089431057242cdc931e9f133147e9
+./scripts/test.sh
+./scripts/test-integration.sh
+```
+
+CI runs the harness on Linux and macOS, invokes the composite Action itself to
+catch drift between `action.yml` and `scripts/run.sh`, and exposes the stable
+`CI` aggregator check for branch protection.
+
+When adding behavior, add a failing public-behavior test first, make the minimum
+implementation change, then run the complete verification set. Keep tests
+deterministic and local; production API calls belong in a separate canary, not
+the PR gate.
+
+## Releases
+
+- Do not publish or move release tags from a PR workflow.
+- Use immutable full-version tags for releases.
+- Only move a major convenience tag such as `v1` after the corresponding full
+  release has been verified.
+- Coordinate Action features that depend on new CLI metadata with the CLI
+  release. The Action must not advertise a pinned CLI version that silently
+  drops a requested public input.

README.md

@@ -29,6 +29,7 @@ jobs:
           install-command: npm ci
           tags: production,webapp
           grep: checkout
+          github-check-name: Checkly PR code checks
         env:
           CHECKLY_API_KEY: ${{ secrets.CHECKLY_API_KEY }}
           CHECKLY_ACCOUNT_ID: ${{ secrets.CHECKLY_ACCOUNT_ID }}
@@ -81,7 +82,7 @@ target URL explicitly through `env` or the workflow `env` block.
 | Input | Description |
 | --- | --- |
 | `command` | `test` for local constructs or `trigger` for deployed checks. Defaults to `test`. |
-| `cli-version` | Checkly CLI npm version. Defaults to `latest`. GitHub Check reporting needs `8.12.0` or newer. Dist-tags, canaries, and prereleases are assumed compatible. |
+| `cli-version` | Checkly CLI npm version. Requires `8.15.0` or newer and defaults to `latest`. Dist-tags, ranges, canaries, and prereleases are assumed compatible. |
 | `working-directory` | Directory where the CLI command should run. Defaults to `.`. |
 | `install-command` | Optional command to run before the Checkly CLI command, inside `working-directory`. |
 | `tags` | One `--tags` filter per line. Each line can contain comma-separated tags. |
@@ -102,6 +103,7 @@ target URL explicitly through `env` or the workflow `env` block.
 | `fail-on-no-matching` | `trigger` only. Set to `false` to pass `--no-fail-on-no-matching`. |
 | `verbose` | Set to `true` or `false` to pass `--verbose` or `--no-verbose`. |
 | `reporting` | Where to report the Checkly result: `auto`, `github-check`, or `github-actions`. Defaults to `auto`. |
+| `github-check-name` | GitHub Check name used when reporting through the Checkly GitHub App. Defaults to `Checkly`. |
 
 ## Outputs
 
@@ -110,8 +112,16 @@ target URL explicitly through `env` or the workflow `env` block.
 | `test-session-id` | Checkly test session ID, when detected from CLI output. |
 | `test-session-url` | Checkly test session URL, when detected from CLI output. |
 
-## Local test
+## Development
+
+The test harness does not need Checkly or GitHub credentials. It uses a local
+preflight server and a fake CLI process to exercise the complete Action flow.
 
 ```sh
+shellcheck scripts/*.sh
 ./scripts/test.sh
+./scripts/test-integration.sh
 ```
+
+See `AGENTS.md` for the repository invariants, complete verification commands,
+and release guidance.

action.yml

@@ -12,7 +12,7 @@ inputs:
     required: false
     default: test
   cli-version:
-    description: Checkly CLI npm version to run. GitHub Check reporting needs 8.12.0 or newer.
+    description: Checkly CLI npm version to run. Requires 8.15.0 or newer.
     required: false
     default: latest
   working-directory:
@@ -79,6 +79,9 @@ inputs:
     description: Where to report the Checkly result. Use "auto", "github-check", or "github-actions".
     required: false
     default: auto
+  github-check-name:
+    description: GitHub Check name to create when reporting through the Checkly GitHub App. Defaults to "Checkly".
+    required: false
 
 outputs:
   test-session-id:
@@ -117,3 +120,4 @@ runs:
         INPUT_FAIL_ON_NO_MATCHING: ${{ inputs.fail-on-no-matching }}
         INPUT_VERBOSE: ${{ inputs.verbose }}
         INPUT_REPORTING: ${{ inputs.reporting }}
+        INPUT_GITHUB_CHECK_NAME: ${{ inputs.github-check-name }}

scripts/run.sh

@@ -81,18 +81,18 @@ add_positional_from_lines() {
   done <<< "$values"
 }
 
-cli_version_supports_github_report() {
+cli_version_is_supported() {
   local version
   version="$(trim "${1:-}")"
 
-  # Only exact pinned stable semver is comparable against the 8.12.0 floor.
+  # Only exact pinned stable semver is comparable against the 8.15.0 floor.
   # Dist-tags, ranges, canaries, and prereleases pass because they may point
   # at compatible builds before a stable release exists.
   if [[ "$version" =~ ^v?([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then
     local major="${BASH_REMATCH[1]}"
     local minor="${BASH_REMATCH[2]}"
 
-    if (( major < 8 || (major == 8 && minor < 12) )); then
+    if (( major < 8 || (major == 8 && minor < 15) )); then
       return 1
     fi
   fi
@@ -150,6 +150,8 @@ try {
   }
   if (typeof value === 'string' && value.trim() !== '') {
     process.stdout.write(value)
+  } else if (typeof value === 'number' && Number.isFinite(value)) {
+    process.stdout.write(String(value))
   }
 } catch (_) {
   process.exit(0)
@@ -182,6 +184,10 @@ resolve_github_sha() {
 
 clear_github_report_env() {
   unset CHECKLY_GITHUB_REPORT
+  unset CHECKLY_GITHUB_SOURCE
+  unset CHECKLY_GITHUB_CHECK_NAME
+  unset CHECKLY_GITHUB_PULL_REQUEST_NUMBER
+  unset CHECKLY_GITHUB_ENVIRONMENT_URL
   unset CHECKLY_GITHUB_REPOSITORY
   unset CHECKLY_GITHUB_SHA
   unset CHECKLY_GITHUB_RUN_ID
@@ -224,8 +230,26 @@ configure_deployment_environment_url() {
 configure_github_report() {
   local repository="$1"
   local sha="$2"
+  local github_check_name="$3"
 
   export CHECKLY_GITHUB_REPORT=true
+  export CHECKLY_GITHUB_SOURCE=checkly-action
+
+  if [[ -n "$github_check_name" ]]; then
+    export CHECKLY_GITHUB_CHECK_NAME="$github_check_name"
+  fi
+
+  if is_pull_request_event; then
+    local pull_request_number
+    pull_request_number="$(github_event_value "pull_request.number")"
+    if [[ -n "$pull_request_number" ]]; then
+      export CHECKLY_GITHUB_PULL_REQUEST_NUMBER="$pull_request_number"
+    fi
+  fi
+
+  if [[ -n "${ENVIRONMENT_URL:-}" ]]; then
+    export CHECKLY_GITHUB_ENVIRONMENT_URL="$ENVIRONMENT_URL"
+  fi
 
   if [[ -n "$repository" ]]; then
     export CHECKLY_GITHUB_REPOSITORY="$repository"
@@ -300,6 +324,10 @@ const accountId = process.env.CHECKLY_ACCOUNT_ID
 const apiKey = process.env.CHECKLY_API_KEY
 
 const payload = {
+  source: process.env.CHECKLY_GITHUB_SOURCE,
+  githubCheckName: process.env.CHECKLY_GITHUB_CHECK_NAME,
+  pullRequestNumber: process.env.CHECKLY_GITHUB_PULL_REQUEST_NUMBER,
+  environmentUrl: process.env.CHECKLY_GITHUB_ENVIRONMENT_URL,
   repository: process.env.CHECKLY_GITHUB_REPOSITORY,
   sha: process.env.CHECKLY_GITHUB_SHA,
   runId: process.env.CHECKLY_GITHUB_RUN_ID,
@@ -353,6 +381,7 @@ cli_version="$(trim "${INPUT_CLI_VERSION:-latest}")"
 working_directory="$(trim "${INPUT_WORKING_DIRECTORY:-.}")"
 install_command="$(trim "${INPUT_INSTALL_COMMAND:-}")"
 reporting="$(trim "${INPUT_REPORTING:-auto}")"
+github_check_name="$(trim "${INPUT_GITHUB_CHECK_NAME:-}")"
 
 case "$command_name" in
   test|trigger) ;;
@@ -370,6 +399,11 @@ case "$reporting" in
     ;;
 esac
 
+if ! cli_version_is_supported "$cli_version"; then
+  echo "::error::The Checkly Action needs Checkly CLI 8.15.0 or newer (cli-version is '${cli_version}'). Use cli-version: latest or >= 8.15.0." >&2
+  exit 1
+fi
+
 if [[ "$command_name" == "trigger" && -n "$(trim "${INPUT_GREP:-}")" ]]; then
   echo "::error::Input 'grep' is only supported with command=test." >&2
   exit 1
@@ -418,31 +452,20 @@ if [[ "$reporting" == "github-actions" ]]; then
   clear_github_report_env
 else
   github_check_requested=true
-  if ! cli_version_supports_github_report "$cli_version"; then
-    github_report_reason="cli_version_too_old"
+  configure_github_report "$github_repository" "$github_sha" "$github_check_name"
+  preflight_result="$(github_report_preflight)" || preflight_result=$'false\tpreflight_crashed'
+  IFS=$'\t' read -r github_report_available github_report_reason <<< "$preflight_result"
+  github_report_reason="$(sanitize_reason "$github_report_reason")"
+  if [[ "$github_report_available" == "true" ]]; then
+    detach_run=true
+    github_reporter_run=false
+  else
     clear_github_report_env
     if [[ "$reporting" == "github-check" ]]; then
-      echo "::error::GitHub Check reporting needs Checkly CLI 8.12.0 or newer (cli-version is '${cli_version}'). Use cli-version: latest or >= 8.12.0, or set reporting: github-actions to wait for the result in this workflow." >&2
+      echo "::error::Checkly GitHub Check reporting is unavailable (${github_report_reason}). Install the Checkly GitHub App on this repository to run detached and receive a Checkly GitHub Check: ${CHECKLY_GITHUB_APP_URL}" >&2
       exit 1
     elif [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
-      echo "::warning::GitHub Check reporting needs Checkly CLI 8.12.0 or newer (cli-version is '${cli_version}'). Reporting through GitHub Actions instead. Use cli-version: latest or >= 8.12.0 to enable detached GitHub Check reporting."
-    fi
-  else
-    configure_github_report "$github_repository" "$github_sha"
-    preflight_result="$(github_report_preflight)" || preflight_result=$'false\tpreflight_crashed'
-    IFS=$'\t' read -r github_report_available github_report_reason <<< "$preflight_result"
-    github_report_reason="$(sanitize_reason "$github_report_reason")"
-    if [[ "$github_report_available" == "true" ]]; then
-      detach_run=true
-      github_reporter_run=false
-    else
-      clear_github_report_env
-      if [[ "$reporting" == "github-check" ]]; then
-        echo "::error::Checkly GitHub Check reporting is unavailable (${github_report_reason}). Install the Checkly GitHub App on this repository to run detached and receive a Checkly GitHub Check: ${CHECKLY_GITHUB_APP_URL}" >&2
-        exit 1
-      elif [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
-        echo "::warning::Checkly GitHub Check reporting is unavailable (${github_report_reason}). Reporting through GitHub Actions instead, so this job waits for the Checkly test session result. Install the Checkly GitHub App on this repository to run detached and receive a Checkly GitHub Check: ${CHECKLY_GITHUB_APP_URL}"
-      fi
+      echo "::warning::Checkly GitHub Check reporting is unavailable (${github_report_reason}). Reporting through GitHub Actions instead, so this job waits for the Checkly test session result. Install the Checkly GitHub App on this repository to run detached and receive a Checkly GitHub Check: ${CHECKLY_GITHUB_APP_URL}"
     fi
   fi
 fi
@@ -490,10 +513,21 @@ if [[ "${CHECKLY_ACTION_DRY_RUN:-}" == "1" || "${CHECKLY_ACTION_DRY_RUN:-}" == "
     printf 'Reporting: GitHub Actions\n'
   elif [[ "$github_check_requested" == "true" && "$github_report_available" == "true" ]]; then
     printf 'Reporting: GitHub Check'
+    if [[ -n "${CHECKLY_GITHUB_CHECK_NAME:-}" ]]; then
+      printf ' "%s"' "$CHECKLY_GITHUB_CHECK_NAME"
+    fi
     if [[ -n "${CHECKLY_GITHUB_REPOSITORY:-}" && -n "${CHECKLY_GITHUB_SHA:-}" ]]; then
       printf ' for %s@%s' "$CHECKLY_GITHUB_REPOSITORY" "$CHECKLY_GITHUB_SHA"
     fi
     printf '\n'
+    printf 'GitHub metadata: source=%s' "${CHECKLY_GITHUB_SOURCE:-}"
+    if [[ -n "${CHECKLY_GITHUB_PULL_REQUEST_NUMBER:-}" ]]; then
+      printf ' pullRequestNumber=%s' "$CHECKLY_GITHUB_PULL_REQUEST_NUMBER"
+    fi
+    if [[ -n "${CHECKLY_GITHUB_ENVIRONMENT_URL:-}" ]]; then
+      printf ' environmentUrl=%s' "$CHECKLY_GITHUB_ENVIRONMENT_URL"
+    fi
+    printf '\n'
   elif [[ "$github_check_requested" == "true" ]]; then
     printf 'Reporting: GitHub Actions (GitHub Check unavailable: %s)\n' "$github_report_reason"
   else