api-test-runner.yml

name: API test runner

on:
  workflow_dispatch:
  pull_request:
  push:
    branches: [main]

permissions:
  contents: read

jobs:
  smoke-local-rest-graphql:
    name: Smoke suite (local REST + GraphQL via httpbin fixture)
    runs-on: ${{ matrix.os }}
    timeout-minutes: 10
    strategy:
      matrix:
        os: [macos-latest, ubuntu-latest]
    env:
      AGENT_HOME: ${{ github.workspace }}
      SUITE_NAME: smoke-local-rest-graphql
      SUITE_DIR: out/api-test-runner/smoke-local-rest-graphql

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Homebrew + nils-cli
        run: |
          scripts/install-homebrew-nils-cli.sh

      - name: Install jq
        run: |
          if ! command -v jq >/dev/null 2>&1; then
            brew install jq
          fi

      - name: Bootstrap public suite (gitignored setup/)
        run: |
          rm -rf setup
          cp -R skills/tools/testing/api-test-runner/assets/scaffold/setup ./setup

      - name: Run suite
        env:
          AGENT_HOME: ${{ github.workspace }}
        run: |
          set -euo pipefail

          port=43127
          mkdir -p "$SUITE_DIR"
          python3 -u setup/fixtures/httpbin/server.py --port "$port" >"$SUITE_DIR/httpbin-fixture.log" 2>&1 &
          pid=$!
          trap 'kill "$pid" >/dev/null 2>&1 || true' EXIT

          for i in $(seq 1 50); do
            if curl -fsS "http://127.0.0.1:${port}/health" >/dev/null 2>&1; then
              break
            fi
            sleep 0.1
          done
          curl -fsS "http://127.0.0.1:${port}/health" >/dev/null

          api-test run \
            --suite smoke-demo \
            --out "$SUITE_DIR/results.json" \
            --junit "$SUITE_DIR/junit.xml"

      - name: Summarize results
        if: always()
        run: |
          set -euo pipefail

          results_file="$SUITE_DIR/results.json"
          summary_file="$SUITE_DIR/summary.md"
          mkdir -p "$SUITE_DIR"

          if ! command -v api-test >/dev/null 2>&1; then
            printf '%s\n' '_Summary skipped: api-test not found._' >"$summary_file"
          elif [[ ! -f "$results_file" ]]; then
            printf '%s\n' "_Summary skipped: missing ${results_file}._" >"$summary_file"
          elif ! api-test summary --in "$results_file" --out "$summary_file" --slow 5; then
            printf '%s\n' "_Summary generation failed for ${results_file}._" >"$summary_file"
          fi

          if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
            {
              echo "## API test summary: ${SUITE_NAME} (${RUNNER_OS})"
              echo ""
              cat "$summary_file"
              echo ""
              echo "Artifacts: \`${SUITE_DIR}/\`"
              echo ""
            } >>"$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-test-runner-${{ env.SUITE_NAME }}-${{ matrix.os }}
          if-no-files-found: warn
          path: |
            ${{ env.SUITE_DIR }}/

  cleanup-fixture-rest-graphql:
    name: Cleanup fixture suite (REST + GraphQL writes + cleanup)
    runs-on: ${{ matrix.os }}
    timeout-minutes: 10
    strategy:
      matrix:
        os: [macos-latest, ubuntu-latest]
    env:
      AGENT_HOME: ${{ github.workspace }}
      SUITE_NAME: cleanup-fixture-rest-graphql
      SUITE_DIR: out/api-test-runner/cleanup-fixture-rest-graphql

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Homebrew + nils-cli
        run: |
          scripts/install-homebrew-nils-cli.sh

      - name: Install jq
        run: |
          if ! command -v jq >/dev/null 2>&1; then
            brew install jq
          fi

      - name: Run cleanup fixture suite
        env:
          AGENT_HOME: ${{ github.workspace }}
          API_TEST_ALLOW_WRITES_ENABLED: "true"
          API_TEST_OUTPUT_DIR: ${{ env.SUITE_DIR }}
        run: |
          set -euo pipefail

          port=43127
          mkdir -p "$SUITE_DIR"

          python3 -u tests/fixtures/api-test-runner/cleanup/server.py --port "$port" >"$SUITE_DIR/server.log" 2>&1 &
          pid=$!
          trap 'kill "$pid" >/dev/null 2>&1 || true' EXIT

          for i in $(seq 1 50); do
            if curl -fsS "http://127.0.0.1:${port}/health" >/dev/null 2>&1; then
              break
            fi
            sleep 0.1
          done
          curl -fsS "http://127.0.0.1:${port}/health" >/dev/null

          api-test run \
            --suite-file tests/fixtures/api-test-runner/cleanup/cleanup.suite.json \
            --out "$SUITE_DIR/results.json" \
            --junit "$SUITE_DIR/junit.xml"

      - name: Summarize results
        if: always()
        run: |
          set -euo pipefail

          results_file="$SUITE_DIR/results.json"
          summary_file="$SUITE_DIR/summary.md"
          mkdir -p "$SUITE_DIR"

          if ! command -v api-test >/dev/null 2>&1; then
            printf '%s\n' '_Summary skipped: api-test not found._' >"$summary_file"
          elif [[ ! -f "$results_file" ]]; then
            printf '%s\n' "_Summary skipped: missing ${results_file}._" >"$summary_file"
          elif ! api-test summary --in "$results_file" --out "$summary_file" --slow 5; then
            printf '%s\n' "_Summary generation failed for ${results_file}._" >"$summary_file"
          fi

          if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
            {
              echo "## API test summary: ${SUITE_NAME} (${RUNNER_OS})"
              echo ""
              cat "$summary_file"
              echo ""
              echo "Artifacts: \`${SUITE_DIR}/\`"
              echo ""
            } >>"$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-test-runner-${{ env.SUITE_NAME }}-${{ matrix.os }}
          if-no-files-found: warn
          path: |
            ${{ env.SUITE_DIR }}/

  public-auth-rest-graphql-demo:
    name: Public auth demo (DummyJSON REST + Escuelajs GraphQL expected fail)
    runs-on: ${{ matrix.os }}
    timeout-minutes: 10
    strategy:
      matrix:
        os: [macos-latest, ubuntu-latest]
    env:
      AGENT_HOME: ${{ github.workspace }}
      SUITE_NAME: public-auth-rest-graphql-demo
      SUITE_DIR: out/api-test-runner/public-auth-rest-graphql-demo

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Homebrew + nils-cli
        run: |
          scripts/install-homebrew-nils-cli.sh

      - name: Install jq
        run: |
          if ! command -v jq >/dev/null 2>&1; then
            brew install jq
          fi

      - name: Bootstrap public suite (gitignored setup/)
        run: |
          rm -rf setup
          cp -R skills/tools/testing/api-test-runner/assets/scaffold/setup ./setup

      - name: Run suite (expected GraphQL failure demo)
        continue-on-error: true
        env:
          AGENT_HOME: ${{ github.workspace }}
        run: |
          set -euo pipefail

          mkdir -p "$SUITE_DIR"
          api-test run \
            --suite public-auth-demo \
            --out "$SUITE_DIR/results.json" \
            --junit "$SUITE_DIR/junit.xml"

      - name: Summarize results
        if: always()
        run: |
          set -euo pipefail

          results_file="$SUITE_DIR/results.json"
          summary_file="$SUITE_DIR/summary.md"
          mkdir -p "$SUITE_DIR"

          if ! command -v api-test >/dev/null 2>&1; then
            printf '%s\n' '_Summary skipped: api-test not found._' >"$summary_file"
          elif [[ ! -f "$results_file" ]]; then
            printf '%s\n' "_Summary skipped: missing ${results_file}._" >"$summary_file"
          elif ! api-test summary --in "$results_file" --out "$summary_file" --slow 5; then
            printf '%s\n' "_Summary generation failed for ${results_file}._" >"$summary_file"
          fi

          if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
            {
              echo "## API test summary: ${SUITE_NAME} (${RUNNER_OS})"
              echo ""
              cat "$summary_file"
              echo ""
              echo "Artifacts: \`${SUITE_DIR}/\`"
              echo ""
            } >>"$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-test-runner-${{ env.SUITE_NAME }}-${{ matrix.os }}
          if-no-files-found: warn
          path: |
            ${{ env.SUITE_DIR }}/

  auth-secrets-rest-graphql:
    # Requires a repo secret named API_TEST_AUTH_JSON (single JSON string).
    # Note: `jobs.<job_id>.if` cannot use `secrets` context, so we gate steps via env instead.
    name: Auth via API_TEST_AUTH_JSON (DummyJSON REST + Countries GraphQL)
    runs-on: ${{ matrix.os }}
    timeout-minutes: 10
    strategy:
      matrix:
        os: [macos-latest, ubuntu-latest]
    env:
      API_TEST_AUTH_JSON: ${{ secrets.API_TEST_AUTH_JSON }}
      AGENT_HOME: ${{ github.workspace }}
      SUITE_NAME: auth-secrets-rest-graphql
      SUITE_DIR: out/api-test-runner/auth-secrets-rest-graphql

    steps:
      - name: Skip (missing API_TEST_AUTH_JSON)
        if: ${{ env.API_TEST_AUTH_JSON == '' }}
        run: |
          if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
            {
              echo "## API test summary: ${SUITE_NAME} (${RUNNER_OS})"
              echo ""
              echo "_Skipped: missing repo secret \`API_TEST_AUTH_JSON\`._"
              echo ""
              echo "Artifacts: \`${SUITE_DIR}/\` (not generated)"
              echo ""
            } >>"$GITHUB_STEP_SUMMARY"
          fi
          echo "Skipping ${SUITE_NAME}: missing repo secret API_TEST_AUTH_JSON."

      - name: Checkout
        if: ${{ env.API_TEST_AUTH_JSON != '' }}
        uses: actions/checkout@v4

      - name: Install Homebrew + nils-cli
        if: ${{ env.API_TEST_AUTH_JSON != '' }}
        run: |
          scripts/install-homebrew-nils-cli.sh

      - name: Install jq
        if: ${{ env.API_TEST_AUTH_JSON != '' }}
        run: |
          if ! command -v jq >/dev/null 2>&1; then
            brew install jq
          fi

      - name: Bootstrap public suite (gitignored setup/)
        if: ${{ env.API_TEST_AUTH_JSON != '' }}
        run: |
          rm -rf setup
          cp -R skills/tools/testing/api-test-runner/assets/scaffold/setup ./setup

      - name: Run suite (auth via GitHub Secrets)
        if: ${{ env.API_TEST_AUTH_JSON != '' }}
        env:
          AGENT_HOME: ${{ github.workspace }}
        run: |
          set -euo pipefail

          mkdir -p "$SUITE_DIR"
          api-test run \
            --suite auth-secrets-demo \
            --out "$SUITE_DIR/results.json" \
            --junit "$SUITE_DIR/junit.xml"

      - name: Summarize results
        if: ${{ always() && env.API_TEST_AUTH_JSON != '' }}
        run: |
          set -euo pipefail

          results_file="$SUITE_DIR/results.json"
          summary_file="$SUITE_DIR/summary.md"
          mkdir -p "$SUITE_DIR"

          if ! command -v api-test >/dev/null 2>&1; then
            printf '%s\n' '_Summary skipped: api-test not found._' >"$summary_file"
          elif [[ ! -f "$results_file" ]]; then
            printf '%s\n' "_Summary skipped: missing ${results_file}._" >"$summary_file"
          elif ! api-test summary --in "$results_file" --out "$summary_file" --slow 5; then
            printf '%s\n' "_Summary generation failed for ${results_file}._" >"$summary_file"
          fi

          if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
            {
              echo "## API test summary: ${SUITE_NAME} (${RUNNER_OS})"
              echo ""
              cat "$summary_file"
              echo ""
              echo "Artifacts: \`${SUITE_DIR}/\`"
              echo ""
            } >>"$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload artifacts
        if: ${{ always() && env.API_TEST_AUTH_JSON != '' }}
        uses: actions/upload-artifact@v4
        with:
          name: api-test-runner-${{ env.SUITE_NAME }}-${{ matrix.os }}
          if-no-files-found: warn
          path: |
            ${{ env.SUITE_DIR }}/