diff --git a/README.md b/README.md index 792ea21..44c2c0a 100644 --- a/README.md +++ b/README.md @@ -57,17 +57,19 @@ with: production,backend ``` -When `github-report` is enabled, the action asks Checkly whether the Checkly -GitHub App is connected to the repository and can report GitHub Checks. If it is, -the action runs the CLI with `--detach`, passes GitHub Actions metadata to the -CLI, and Checkly posts a GitHub Check that updates when the session finishes. - -If GitHub Check reporting is unavailable, the action runs without `--detach` and -waits for the test session to finish so the GitHub Actions job reports the -result. It also uses the CLI GitHub reporter and writes the Checkly summary to -the GitHub Actions step summary. Install the -[Checkly GitHub App](https://github.com/apps/checkly) on the repository to run -detached and receive a Checkly GitHub Check instead. +The `reporting` input controls where the Checkly result is reported: + +- `auto` (default): use a detached run with GitHub Check reporting when the + Checkly GitHub App can report on the repository. Otherwise, wait in the + GitHub Actions job and report there. +- `github-check`: require detached GitHub Check reporting. The action fails + before running checks if the Checkly GitHub App cannot report on the + repository. +- `github-actions`: always wait in the GitHub Actions job and report through the + CLI GitHub reporter and step summary. + +Install the [Checkly GitHub App](https://github.com/apps/checkly) on the +repository to use detached GitHub Check reporting. For `deployment_status` workflows, the action exposes `github.event.deployment_status.environment_url` as `ENVIRONMENT_URL` when that @@ -79,7 +81,7 @@ target URL explicitly through `env` or the workflow `env` block. | Input | Description | | --- | --- | | `command` | `test` for local constructs or `trigger` for deployed checks. Defaults to `test`. | -| `cli-version` | Checkly CLI npm version. Defaults to `latest`. GitHub Check writeback needs `8.12.0` or newer; older pinned versions fall back to waiting in the Action. Dist-tags, canaries, and prereleases are assumed compatible. | +| `cli-version` | Checkly CLI npm version. Defaults to `latest`. GitHub Check reporting needs `8.12.0` or newer. Dist-tags, canaries, and prereleases are assumed compatible. | | `working-directory` | Directory where the CLI command should run. Defaults to `.`. | | `install-command` | Optional command to run before the Checkly CLI command, inside `working-directory`. | | `tags` | One `--tags` filter per line. Each line can contain comma-separated tags. | @@ -99,14 +101,14 @@ target URL explicitly through `env` or the workflow `env` block. | `verify-runtime-dependencies` | `test` only. Set to `false` to pass `--no-verify-runtime-dependencies`. | | `fail-on-no-matching` | `trigger` only. Set to `false` to pass `--no-fail-on-no-matching`. | | `verbose` | Set to `true` or `false` to pass `--verbose` or `--no-verbose`. | -| `github-report` | Use detached Checkly runs with GitHub Check writeback when the Checkly GitHub App is connected. Falls back to waiting in the Action when unavailable. Defaults to `true`. | +| `reporting` | Where to report the Checkly result: `auto`, `github-check`, or `github-actions`. Defaults to `auto`. | ## Outputs | Output | Description | | --- | --- | -| `test-session-id` | Detached Checkly test session ID, when detected from CLI output. | -| `test-session-url` | Detached Checkly test session URL, when detected from CLI output. | +| `test-session-id` | Checkly test session ID, when detected from CLI output. | +| `test-session-url` | Checkly test session URL, when detected from CLI output. | ## Local test diff --git a/action.yml b/action.yml index 0ba7fd0..0152635 100644 --- a/action.yml +++ b/action.yml @@ -12,7 +12,7 @@ inputs: required: false default: test cli-version: - description: Checkly CLI npm version to run. GitHub Check writeback needs 8.12.0 or newer; older pinned versions fall back to waiting in the Action. + description: Checkly CLI npm version to run. GitHub Check reporting needs 8.12.0 or newer. required: false default: latest working-directory: @@ -53,7 +53,7 @@ inputs: description: Name to use when storing results as a Checkly test session. required: false timeout: - description: Timeout in seconds to wait for checks to complete. Mostly useful when detach is disabled in future versions. + description: Timeout in seconds to wait for checks to complete. Mostly useful when reporting through GitHub Actions. required: false retries: description: Number of times to retry a failing run. @@ -75,10 +75,10 @@ inputs: verbose: description: Set to true or false to pass --verbose or --no-verbose. required: false - github-report: - description: Use detached Checkly runs with GitHub Check writeback when the Checkly GitHub App is connected. Falls back to waiting in the Action when unavailable. + reporting: + description: Where to report the Checkly result. Use "auto", "github-check", or "github-actions". required: false - default: "true" + default: auto outputs: test-session-id: @@ -116,4 +116,4 @@ runs: INPUT_VERIFY_RUNTIME_DEPENDENCIES: ${{ inputs.verify-runtime-dependencies }} INPUT_FAIL_ON_NO_MATCHING: ${{ inputs.fail-on-no-matching }} INPUT_VERBOSE: ${{ inputs.verbose }} - INPUT_GITHUB_REPORT: ${{ inputs.github-report }} + INPUT_REPORTING: ${{ inputs.reporting }} diff --git a/scripts/run.sh b/scripts/run.sh index 3a4b2a0..3b55a6f 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -352,6 +352,7 @@ command_name="$(trim "${INPUT_COMMAND:-test}")" cli_version="$(trim "${INPUT_CLI_VERSION:-latest}")" working_directory="$(trim "${INPUT_WORKING_DIRECTORY:-.}")" install_command="$(trim "${INPUT_INSTALL_COMMAND:-}")" +reporting="$(trim "${INPUT_REPORTING:-auto}")" case "$command_name" in test|trigger) ;; @@ -361,6 +362,14 @@ case "$command_name" in ;; esac +case "$reporting" in + auto|github-check|github-actions) ;; + *) + echo "::error::Unsupported reporting '${reporting}'. Expected 'auto', 'github-check', or 'github-actions'." >&2 + exit 1 + ;; +esac + if [[ "$command_name" == "trigger" && -n "$(trim "${INPUT_GREP:-}")" ]]; then echo "::error::Input 'grep' is only supported with command=test." >&2 exit 1 @@ -399,23 +408,24 @@ github_sha="$(resolve_github_sha)" configure_generic_repo_env "$github_repository" "$github_sha" configure_deployment_environment_url -github_report_requested=false +github_check_requested=false github_report_available=false github_report_reason="disabled" detach_run=false github_reporter_run=true -if falsey "${INPUT_GITHUB_REPORT:-true}"; then +if [[ "$reporting" == "github-actions" ]]; then clear_github_report_env -elif truthy "${INPUT_GITHUB_REPORT:-true}"; then - github_report_requested=true +else + github_check_requested=true if ! cli_version_supports_github_report "$cli_version"; then - # Writeback needs CLI >= 8.12.0, but an older pinned CLI is not an error: - # fall back to the non-detached mode like any other unavailable reason. github_report_reason="cli_version_too_old" clear_github_report_env - if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then - echo "::warning::GitHub Check writeback needs Checkly CLI 8.12.0 or newer (cli-version is '${cli_version}'). Running without --detach so this GitHub Actions job waits for the Checkly test session result. Use cli-version: latest or >= 8.12.0 to enable detached GitHub Check reporting." + if [[ "$reporting" == "github-check" ]]; then + echo "::error::GitHub Check reporting needs Checkly CLI 8.12.0 or newer (cli-version is '${cli_version}'). Use cli-version: latest or >= 8.12.0, or set reporting: github-actions to wait for the result in this workflow." >&2 + exit 1 + elif [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + echo "::warning::GitHub Check reporting needs Checkly CLI 8.12.0 or newer (cli-version is '${cli_version}'). Reporting through GitHub Actions instead. Use cli-version: latest or >= 8.12.0 to enable detached GitHub Check reporting." fi else configure_github_report "$github_repository" "$github_sha" @@ -427,14 +437,14 @@ elif truthy "${INPUT_GITHUB_REPORT:-true}"; then github_reporter_run=false else clear_github_report_env - if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then - echo "::warning::Checkly GitHub App reporting is unavailable (${github_report_reason}). Running without --detach so this GitHub Actions job waits for the Checkly test session result. Install the Checkly GitHub App on this repository to run detached and receive a Checkly GitHub Check instead: ${CHECKLY_GITHUB_APP_URL}" + if [[ "$reporting" == "github-check" ]]; then + echo "::error::Checkly GitHub Check reporting is unavailable (${github_report_reason}). Install the Checkly GitHub App on this repository to run detached and receive a Checkly GitHub Check: ${CHECKLY_GITHUB_APP_URL}" >&2 + exit 1 + elif [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + echo "::warning::Checkly GitHub Check reporting is unavailable (${github_report_reason}). Reporting through GitHub Actions instead, so this job waits for the Checkly test session result. Install the Checkly GitHub App on this repository to run detached and receive a Checkly GitHub Check: ${CHECKLY_GITHUB_APP_URL}" fi fi fi -else - echo "::error::Expected boolean input for github-report, got '${INPUT_GITHUB_REPORT}'." >&2 - exit 1 fi checkly_command=(npx --yes "checkly@${cli_version}" "$command_name") @@ -476,16 +486,18 @@ if [[ "${CHECKLY_ACTION_DRY_RUN:-}" == "1" || "${CHECKLY_ACTION_DRY_RUN:-}" == " printf 'Command: ' printf '%q ' "${checkly_command[@]}" printf '\n' - if [[ "$github_report_requested" == "true" && "$github_report_available" == "true" ]]; then - printf 'GitHub report: detached writeback enabled' + if [[ "$reporting" == "github-actions" ]]; then + printf 'Reporting: GitHub Actions\n' + elif [[ "$github_check_requested" == "true" && "$github_report_available" == "true" ]]; then + printf 'Reporting: GitHub Check' if [[ -n "${CHECKLY_GITHUB_REPOSITORY:-}" && -n "${CHECKLY_GITHUB_SHA:-}" ]]; then printf ' for %s@%s' "$CHECKLY_GITHUB_REPOSITORY" "$CHECKLY_GITHUB_SHA" fi printf '\n' - elif [[ "$github_report_requested" == "true" ]]; then - printf 'GitHub report: unavailable (%s), waiting for CLI result\n' "$github_report_reason" + elif [[ "$github_check_requested" == "true" ]]; then + printf 'Reporting: GitHub Actions (GitHub Check unavailable: %s)\n' "$github_report_reason" else - printf 'GitHub report: disabled\n' + printf 'Reporting: GitHub Actions\n' fi exit 0 fi @@ -533,12 +545,12 @@ if [[ -z "$test_session_id" && -z "$test_session_url" ]]; then append_summary "- The Checkly CLI did not print a detached test session reference." fi -if [[ "$github_report_requested" == "true" && "$github_report_available" == "true" ]]; then +if [[ "$github_check_requested" == "true" && "$github_report_available" == "true" ]]; then append_summary "" append_summary "GitHub Check reporting is enabled for this run." -elif [[ "$github_report_requested" == "true" ]]; then +elif [[ "$github_check_requested" == "true" ]]; then append_summary "" - append_summary "GitHub Check reporting was unavailable (${github_report_reason}). This job waited for the Checkly run to finish. [Install the Checkly GitHub App](${CHECKLY_GITHUB_APP_URL}) on this repository to run detached and receive a Checkly GitHub Check." + append_summary "GitHub Check reporting was unavailable (${github_report_reason}). This job reported through GitHub Actions and waited for the Checkly run to finish. [Install the Checkly GitHub App](${CHECKLY_GITHUB_APP_URL}) on this repository to run detached and receive a Checkly GitHub Check." fi rm -f "$output_file" diff --git a/scripts/test.sh b/scripts/test.sh index 887661e..1fa52fa 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -55,7 +55,7 @@ assert_contains "$test_command_output" "--tags production\\,webapp --tags produc assert_contains "$test_command_output" "--grep checkout" assert_contains "$test_command_output" "--update-snapshots" assert_contains "$test_command_output" "checks/\\*\\*/\\*.check.ts smoke.check.ts" -assert_contains "$test_command_output" "GitHub report: detached writeback enabled" +assert_contains "$test_command_output" "Reporting: GitHub Check" trigger_command_output="$( INPUT_COMMAND=trigger \ @@ -72,6 +72,11 @@ assert_contains "$trigger_command_output" "--tags production" assert_contains "$trigger_command_output" "--check-id abc\\,def" assert_contains "$trigger_command_output" "--no-fail-on-no-matching" +assert_fails_with "Unsupported command 'deploy'" env \ + INPUT_COMMAND=deploy \ + CHECKLY_ACTION_DRY_RUN=1 \ + "$ROOT_DIR/scripts/run.sh" + fallback_command_output="$( INPUT_COMMAND=test \ INPUT_CLI_VERSION=8.12.0 \ @@ -82,8 +87,8 @@ fallback_command_output="$( )" assert_contains "$fallback_command_output" "checkly@8.12.0 test --reporter=github" -assert_contains "$fallback_command_output" "GitHub report: unavailable (github_app_not_connected), waiting for CLI result" -assert_contains "$fallback_command_output" "Install the Checkly GitHub App on this repository to run detached and receive a Checkly GitHub Check instead: https://github.com/apps/checkly" +assert_contains "$fallback_command_output" "Reporting: GitHub Actions (GitHub Check unavailable: github_app_not_connected)" +assert_contains "$fallback_command_output" "Install the Checkly GitHub App on this repository to run detached and receive a Checkly GitHub Check: https://github.com/apps/checkly" # An old pinned CLI must fall back (skipping the preflight entirely), not fail # the job: CHECKLY_ACTION_GITHUB_REPORT_AVAILABLE=true would force the detached @@ -91,24 +96,37 @@ assert_contains "$fallback_command_output" "Install the Checkly GitHub App on th old_cli_fallback_output="$( INPUT_COMMAND=test \ INPUT_CLI_VERSION=8.11.9 \ - INPUT_GITHUB_REPORT=true \ CHECKLY_ACTION_GITHUB_REPORT_AVAILABLE=true \ run_dry )" assert_contains "$old_cli_fallback_output" "checkly@8.11.9 test --reporter=github" -assert_contains "$old_cli_fallback_output" "GitHub report: unavailable (cli_version_too_old), waiting for CLI result" +assert_contains "$old_cli_fallback_output" "Reporting: GitHub Actions (GitHub Check unavailable: cli_version_too_old)" + +assert_fails_with "Unsupported reporting 'banana'" env \ + INPUT_COMMAND=test \ + INPUT_REPORTING=banana \ + CHECKLY_ACTION_DRY_RUN=1 \ + "$ROOT_DIR/scripts/run.sh" -assert_fails_with "Expected boolean input for github-report" env \ +assert_fails_with "GitHub Check reporting needs Checkly CLI 8.12.0 or newer" env \ INPUT_COMMAND=test \ - INPUT_GITHUB_REPORT=banana \ + INPUT_CLI_VERSION=8.11.9 \ + INPUT_REPORTING=github-check \ + CHECKLY_ACTION_DRY_RUN=1 \ + "$ROOT_DIR/scripts/run.sh" + +assert_fails_with "Checkly GitHub Check reporting is unavailable (github_app_not_connected)" env \ + INPUT_COMMAND=test \ + INPUT_REPORTING=github-check \ + CHECKLY_ACTION_GITHUB_REPORT_AVAILABLE=false \ + CHECKLY_ACTION_GITHUB_REPORT_REASON=github_app_not_connected \ CHECKLY_ACTION_DRY_RUN=1 \ "$ROOT_DIR/scripts/run.sh" prerelease_version_output="$( INPUT_COMMAND=test \ INPUT_CLI_VERSION=0.0.0-canary.58c867e \ - INPUT_GITHUB_REPORT=true \ CHECKLY_ACTION_GITHUB_REPORT_AVAILABLE=true \ run_dry )" @@ -132,7 +150,7 @@ JSON github_report_output="$( INPUT_COMMAND=test \ - INPUT_GITHUB_REPORT=true \ + INPUT_REPORTING=github-check \ CHECKLY_ACTION_GITHUB_REPORT_AVAILABLE=true \ GITHUB_REPOSITORY=checkly/playwright-reporter-demo \ GITHUB_SHA=merge123def456 \ @@ -150,12 +168,12 @@ github_report_output="$( run_dry )" -assert_contains "$github_report_output" "GitHub report: detached writeback enabled for checkly/playwright-reporter-demo@head123def456" +assert_contains "$github_report_output" "Reporting: GitHub Check for checkly/playwright-reporter-demo@head123def456" -github_report_disabled_output="$( +github_actions_output="$( INPUT_COMMAND=test \ INPUT_CLI_VERSION=8.11.9 \ - INPUT_GITHUB_REPORT=false \ + INPUT_REPORTING=github-actions \ CHECKLY_GITHUB_REPORT=true \ CHECKLY_GITHUB_REPOSITORY=spoofed/repository \ CHECKLY_GITHUB_SHA=spoofed-sha \ @@ -164,8 +182,8 @@ github_report_disabled_output="$( run_dry )" -assert_contains "$github_report_disabled_output" "GitHub report: disabled" -assert_contains "$github_report_disabled_output" "checkly@8.11.9 test --reporter=github" +assert_contains "$github_actions_output" "Reporting: GitHub Actions" +assert_contains "$github_actions_output" "checkly@8.11.9 test --reporter=github" deployment_event_path="$(mktemp)" trap 'rm -f "$github_event_path" "$deployment_event_path"' EXIT @@ -179,7 +197,7 @@ JSON deployment_url_output="$( INPUT_COMMAND=test \ - INPUT_GITHUB_REPORT=false \ + INPUT_REPORTING=github-actions \ GITHUB_EVENT_NAME=deployment_status \ GITHUB_EVENT_PATH="$deployment_event_path" \ run_dry