diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 21a7a26c..253fba70 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,474 +1,322 @@ -name: Deploy - -on: - push: - branches: [master] - paths-ignore: - - '**.md' - - 'LICENSE' - - '.gitignore' - - 'docs/**' - pull_request: - branches: [master] - -concurrency: - group: deploy-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - -# Pipeline: -# node-test (frontend tests) ──┐ -# go-test ├──→ build → deploy → publish -# └─ (both wait) -# -# Proto validation flow: -# 1. go-test job: verify .proto files compile (syntax check) -# 2. deploy job: capture fresh fixtures from prod, validate protos match actual API responses - -jobs: - # ─────────────────────────────────────────────────────────────── - # 1. Go Build & Test — compiles + tests Go modules, coverage badges - # ─────────────────────────────────────────────────────────────── - go-test: - name: "✅ Go Build & Test" - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v5 - with: - fetch-depth: 0 - - - name: Skip if docs-only change - id: docs-check - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - CHANGED=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) - NON_DOCS=$(echo "$CHANGED" | grep -cvE '\.(md)$|^LICENSE$|^\.gitignore$|^docs/' || true) - if [ "$NON_DOCS" -eq 0 ]; then - echo "docs_only=true" >> $GITHUB_OUTPUT - echo "📄 Docs-only PR — skipping heavy CI" - fi - fi - - - name: Set up Go 1.22 - if: steps.docs-check.outputs.docs_only != 'true' - uses: actions/setup-go@v6 - with: - go-version: '1.22' - cache-dependency-path: | - cmd/server/go.sum - cmd/ingestor/go.sum - - - name: Build and test Go server (with coverage) - if: steps.docs-check.outputs.docs_only != 'true' - run: | - set -e -o pipefail - cd cmd/server - go build . - go test -coverprofile=server-coverage.out ./... 2>&1 | tee server-test.log - echo "--- Go Server Coverage ---" - go tool cover -func=server-coverage.out | tail -1 - - - name: Build and test Go ingestor (with coverage) - if: steps.docs-check.outputs.docs_only != 'true' - run: | - set -e -o pipefail - cd cmd/ingestor - go build . - go test -coverprofile=ingestor-coverage.out ./... 2>&1 | tee ingestor-test.log - echo "--- Go Ingestor Coverage ---" - go tool cover -func=ingestor-coverage.out | tail -1 - - - name: Verify proto syntax (all .proto files compile) - if: steps.docs-check.outputs.docs_only != 'true' - run: | - set -e - echo "Installing protoc..." - sudo apt-get update -qq - sudo apt-get install -y protobuf-compiler - - echo "Checking proto syntax..." - for proto in proto/*.proto; do - echo " ✓ $(basename "$proto")" - protoc --proto_path=proto --descriptor_set_out=/dev/null "$proto" - done - echo "✅ All .proto files are syntactically valid" - - - name: Generate Go coverage badges - if: always() && steps.docs-check.outputs.docs_only != 'true' - run: | - mkdir -p .badges - - # Parse server coverage - SERVER_COV="0" - if [ -f cmd/server/server-coverage.out ]; then - SERVER_COV=$(cd cmd/server && go tool cover -func=server-coverage.out | tail -1 | grep -oP '[\d.]+(?=%)') - fi - SERVER_COLOR="red" - if [ "$(echo "$SERVER_COV >= 80" | bc -l 2>/dev/null)" = "1" ]; then - SERVER_COLOR="green" - elif [ "$(echo "$SERVER_COV >= 60" | bc -l 2>/dev/null)" = "1" ]; then - SERVER_COLOR="yellow" - fi - echo "{\"schemaVersion\":1,\"label\":\"go server coverage\",\"message\":\"${SERVER_COV}%\",\"color\":\"${SERVER_COLOR}\"}" > .badges/go-server-coverage.json - echo "Go server coverage: ${SERVER_COV}% (${SERVER_COLOR})" - - # Parse ingestor coverage - INGESTOR_COV="0" - if [ -f cmd/ingestor/ingestor-coverage.out ]; then - INGESTOR_COV=$(cd cmd/ingestor && go tool cover -func=ingestor-coverage.out | tail -1 | grep -oP '[\d.]+(?=%)') - fi - INGESTOR_COLOR="red" - if [ "$(echo "$INGESTOR_COV >= 80" | bc -l 2>/dev/null)" = "1" ]; then - INGESTOR_COLOR="green" - elif [ "$(echo "$INGESTOR_COV >= 60" | bc -l 2>/dev/null)" = "1" ]; then - INGESTOR_COLOR="yellow" - fi - echo "{\"schemaVersion\":1,\"label\":\"go ingestor coverage\",\"message\":\"${INGESTOR_COV}%\",\"color\":\"${INGESTOR_COLOR}\"}" > .badges/go-ingestor-coverage.json - echo "Go ingestor coverage: ${INGESTOR_COV}% (${INGESTOR_COLOR})" - - echo "## Go Coverage" >> $GITHUB_STEP_SUMMARY - echo "| Module | Coverage |" >> $GITHUB_STEP_SUMMARY - echo "|--------|----------|" >> $GITHUB_STEP_SUMMARY - echo "| Server | ${SERVER_COV}% |" >> $GITHUB_STEP_SUMMARY - echo "| Ingestor | ${INGESTOR_COV}% |" >> $GITHUB_STEP_SUMMARY - - - name: Cancel workflow on failure - if: failure() - run: | - curl -s -X POST \ - -H "Authorization: Bearer ${{ github.token }}" \ - "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/cancel" - - - name: Upload Go coverage badges - if: always() - uses: actions/upload-artifact@v6 - with: - name: go-badges - path: .badges/go-*.json - retention-days: 1 - if-no-files-found: ignore - - # ─────────────────────────────────────────────────────────────── - # 2. Node.js Tests — backend unit tests + Playwright E2E, coverage - # ─────────────────────────────────────────────────────────────── - node-test: - name: "🧪 Node.js Tests" - if: false # TEMPORARILY DISABLED — CI restructure in progress - runs-on: [self-hosted, Linux] - defaults: - run: - shell: bash - steps: - - name: Checkout code - uses: actions/checkout@v5 - with: - fetch-depth: 0 - - - name: Skip if docs-only change - id: docs-check - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - CHANGED=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) - NON_DOCS=$(echo "$CHANGED" | grep -cvE '\.(md)$|^LICENSE$|^\.gitignore$|^docs/' || true) - if [ "$NON_DOCS" -eq 0 ]; then - echo "docs_only=true" >> $GITHUB_OUTPUT - echo "📄 Docs-only PR — skipping heavy CI" - fi - fi - - - name: Set up Node.js 22 - if: steps.docs-check.outputs.docs_only != 'true' - uses: actions/setup-node@v5 - with: - node-version: '22' - - - name: Set up Go 1.22 - if: steps.docs-check.outputs.docs_only != 'true' - uses: actions/setup-go@v6 - with: - go-version: '1.22' - cache-dependency-path: cmd/server/go.sum - - - name: Build Go server for E2E tests - if: steps.docs-check.outputs.docs_only != 'true' - run: | - cd cmd/server - go build -o ../../corescope-server . - echo "Go server built successfully" - - - name: Install npm dependencies - if: steps.docs-check.outputs.docs_only != 'true' - run: npm ci --production=false - - - name: Detect changed files - if: steps.docs-check.outputs.docs_only != 'true' - id: changes - run: | - BACKEND=$(git diff --name-only HEAD~1 | grep -cE '^(server|db|decoder|packet-store|server-helpers|iata-coords)\.js$' || true) - FRONTEND=$(git diff --name-only HEAD~1 | grep -cE '^public/' || true) - TESTS=$(git diff --name-only HEAD~1 | grep -cE '^test-|^tools/' || true) - CI=$(git diff --name-only HEAD~1 | grep -cE '\.github/|package\.json|test-all\.sh|scripts/' || true) - # If CI/test infra changed, run everything - if [ "$CI" -gt 0 ]; then BACKEND=1; FRONTEND=1; fi - # If test files changed, run everything - if [ "$TESTS" -gt 0 ]; then BACKEND=1; FRONTEND=1; fi - echo "backend=$([[ $BACKEND -gt 0 ]] && echo true || echo false)" >> $GITHUB_OUTPUT - echo "frontend=$([[ $FRONTEND -gt 0 ]] && echo true || echo false)" >> $GITHUB_OUTPUT - echo "Changes: backend=$BACKEND frontend=$FRONTEND tests=$TESTS ci=$CI" - - - name: Run backend tests with coverage - if: steps.docs-check.outputs.docs_only != 'true' && steps.changes.outputs.backend == 'true' - run: | - npx c8 --reporter=text-summary --reporter=text sh test-all.sh 2>&1 | tee test-output.txt - - TOTAL_PASS=$(grep -oP '\d+(?= passed)' test-output.txt | awk '{s+=$1} END {print s}') - TOTAL_FAIL=$(grep -oP '\d+(?= failed)' test-output.txt | awk '{s+=$1} END {print s}') - BE_COVERAGE=$(grep 'Statements' test-output.txt | tail -1 | grep -oP '[\d.]+(?=%)') - - mkdir -p .badges - BE_COLOR="red" - [ "$(echo "$BE_COVERAGE > 60" | bc -l 2>/dev/null)" = "1" ] && BE_COLOR="yellow" - [ "$(echo "$BE_COVERAGE > 80" | bc -l 2>/dev/null)" = "1" ] && BE_COLOR="brightgreen" - echo "{\"schemaVersion\":1,\"label\":\"backend tests\",\"message\":\"${TOTAL_PASS} passed\",\"color\":\"brightgreen\"}" > .badges/backend-tests.json - echo "{\"schemaVersion\":1,\"label\":\"backend coverage\",\"message\":\"${BE_COVERAGE}%\",\"color\":\"${BE_COLOR}\"}" > .badges/backend-coverage.json - - echo "## Backend: ${TOTAL_PASS} tests, ${BE_COVERAGE}% coverage" >> $GITHUB_STEP_SUMMARY - - - name: Run backend tests (quick, no coverage) - if: steps.docs-check.outputs.docs_only != 'true' && steps.changes.outputs.backend == 'false' - run: npm run test:unit - - - name: Install Playwright browser - if: steps.docs-check.outputs.docs_only != 'true' && steps.changes.outputs.frontend == 'true' - run: | - # Install chromium (skips download if already cached on self-hosted runner) - npx playwright install chromium 2>/dev/null || true - # Install system deps only if missing (apt-get is slow) - npx playwright install-deps chromium 2>/dev/null || true - - - name: Instrument frontend JS for coverage - if: steps.docs-check.outputs.docs_only != 'true' && steps.changes.outputs.frontend == 'true' - run: sh scripts/instrument-frontend.sh - - - name: Start instrumented test server on port 13581 - if: steps.docs-check.outputs.docs_only != 'true' && steps.changes.outputs.frontend == 'true' - run: | - # Kill any stale server on 13581 - fuser -k 13581/tcp 2>/dev/null || true - sleep 2 - ./corescope-server -port 13581 -public public-instrumented & - echo $! > .server.pid - echo "Server PID: $(cat .server.pid)" - # Health-check poll loop (up to 30s) - for i in $(seq 1 30); do - if curl -sf http://localhost:13581/api/stats > /dev/null 2>&1; then - echo "Server ready after ${i}s" - break - fi - if [ "$i" -eq 30 ]; then - echo "Server failed to start within 30s" - echo "Last few lines from server logs:" - ps aux | grep "corescope-server" || echo "No server process found" - exit 1 - fi - sleep 1 - done - - - name: Run Playwright E2E + coverage collection concurrently - if: steps.docs-check.outputs.docs_only != 'true' && steps.changes.outputs.frontend == 'true' - run: | - # Run E2E tests and coverage collection in parallel — both use the same server - BASE_URL=http://localhost:13581 node test-e2e-playwright.js 2>&1 | tee e2e-output.txt & - E2E_PID=$! - BASE_URL=http://localhost:13581 node scripts/collect-frontend-coverage.js 2>&1 | tee fe-coverage-output.txt & - COV_PID=$! - - # Wait for both — E2E must pass, coverage is best-effort - E2E_EXIT=0 - wait $E2E_PID || E2E_EXIT=$? - wait $COV_PID || true - - # Fail if E2E failed - [ $E2E_EXIT -ne 0 ] && exit $E2E_EXIT - true - - - name: Generate frontend coverage badges - if: always() && steps.docs-check.outputs.docs_only != 'true' && steps.changes.outputs.frontend == 'true' - run: | - E2E_PASS=$(grep -oP '[0-9]+(?=/)' e2e-output.txt | tail -1) - - mkdir -p .badges - # Merge E2E + coverage collector data if both exist - if [ -f .nyc_output/frontend-coverage.json ] || [ -f .nyc_output/e2e-coverage.json ]; then - npx nyc report --reporter=text-summary --reporter=text 2>&1 | tee fe-report.txt - FE_COVERAGE=$(grep 'Statements' fe-report.txt | head -1 | grep -oP '[\d.]+(?=%)' || echo "0") - FE_COVERAGE=${FE_COVERAGE:-0} - FE_COLOR="red" - [ "$(echo "$FE_COVERAGE > 50" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="yellow" - [ "$(echo "$FE_COVERAGE > 80" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="brightgreen" - echo "{\"schemaVersion\":1,\"label\":\"frontend coverage\",\"message\":\"${FE_COVERAGE}%\",\"color\":\"${FE_COLOR}\"}" > .badges/frontend-coverage.json - echo "## Frontend: ${FE_COVERAGE}% coverage" >> $GITHUB_STEP_SUMMARY - fi - echo "{\"schemaVersion\":1,\"label\":\"frontend tests\",\"message\":\"${E2E_PASS:-0} E2E passed\",\"color\":\"brightgreen\"}" > .badges/frontend-tests.json - - - name: Stop test server - if: always() && steps.docs-check.outputs.docs_only != 'true' && steps.changes.outputs.frontend == 'true' - run: | - if [ -f .server.pid ]; then - kill $(cat .server.pid) 2>/dev/null || true - rm -f .server.pid - echo "Server stopped" - fi - - - name: Run frontend E2E (quick, no coverage) - if: steps.docs-check.outputs.docs_only != 'true' && steps.changes.outputs.frontend == 'false' - run: | - fuser -k 13581/tcp 2>/dev/null || true - ./corescope-server -port 13581 -public public & - SERVER_PID=$! - # Wait for server to be ready (up to 15s) - for i in $(seq 1 15); do - curl -sf http://localhost:13581/api/stats > /dev/null 2>&1 && break - sleep 1 - done - BASE_URL=http://localhost:13581 node test-e2e-playwright.js || true - kill $SERVER_PID 2>/dev/null || true - - - name: Cancel workflow on failure - if: failure() - run: | - curl -s -X POST \ - -H "Authorization: Bearer ${{ github.token }}" \ - "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/cancel" - - - name: Upload Node.js test badges - if: always() - uses: actions/upload-artifact@v6 - with: - name: node-badges - path: .badges/ - retention-days: 1 - if-no-files-found: ignore - - # ─────────────────────────────────────────────────────────────── - # 3. Build Docker Image - # ─────────────────────────────────────────────────────────────── - build: - name: "🏗️ Build Docker Image" - if: github.event_name == 'push' - needs: [go-test, node-test] - runs-on: [self-hosted, Linux] - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Set up Node.js 22 - uses: actions/setup-node@v5 - with: - node-version: '22' - - - name: Build Go Docker image - run: | - echo "${GITHUB_SHA::7}" > .git-commit - APP_VERSION=$(node -p "require('./package.json').version") \ - GIT_COMMIT="${GITHUB_SHA::7}" \ - docker compose --profile staging-go build staging-go - echo "Built Go staging image" - - # ─────────────────────────────────────────────────────────────── - # 4. Deploy Staging — start on port 82, healthcheck, smoke test - # ─────────────────────────────────────────────────────────────── - deploy: - name: "🚀 Deploy Staging" - if: github.event_name == 'push' - needs: [build] - runs-on: [self-hosted, Linux] - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Start staging on port 82 - run: | - # Force remove stale containers - docker rm -f corescope-staging-go 2>/dev/null || true - # Clean up stale ports - fuser -k 82/tcp 2>/dev/null || true - docker compose --profile staging-go up -d staging-go - - - name: Healthcheck staging container - run: | - for i in $(seq 1 120); do - HEALTH=$(docker inspect corescope-staging-go --format '{{.State.Health.Status}}' 2>/dev/null || echo "starting") - if [ "$HEALTH" = "healthy" ]; then - echo "Staging healthy after ${i}s" - break - fi - if [ "$i" -eq 120 ]; then - echo "Staging failed health check after 120s" - docker logs corescope-staging-go --tail 50 - exit 1 - fi - sleep 1 - done - - - name: Smoke test staging API - run: | - if curl -sf http://localhost:82/api/stats | grep -q engine; then - echo "Staging verified — engine field present ✅" - else - echo "Staging /api/stats did not return engine field" - exit 1 - fi - - # ─────────────────────────────────────────────────────────────── - # 5. Publish Badges & Summary - # ─────────────────────────────────────────────────────────────── - publish: - name: "📝 Publish Badges & Summary" - if: github.event_name == 'push' - needs: [deploy] - runs-on: [self-hosted, Linux] - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Download Go coverage badges - continue-on-error: true - uses: actions/download-artifact@v6 - with: - name: go-badges - path: .badges/ - - - name: Download Node.js test badges - continue-on-error: true - uses: actions/download-artifact@v6 - with: - name: node-badges - path: .badges/ - - - name: Publish coverage badges to repo - continue-on-error: true - run: | - git config user.name "github-actions" - git config user.email "actions@github.com" - git remote set-url origin https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git - git add .badges/ -f - git diff --cached --quiet || (git commit -m "ci: update test badges [skip ci]" && git push) || echo "Badge push failed" - - - name: Post deployment summary - run: | - echo "## Staging Deployed ✓" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Commit:** \`$(git rev-parse --short HEAD)\` — $(git log -1 --format=%s)" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Staging:** http://:82" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "To promote to production:" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY - echo "ssh deploy@\$VM_HOST" >> $GITHUB_STEP_SUMMARY - echo "cd /opt/corescope-deploy" >> $GITHUB_STEP_SUMMARY - echo "./manage.sh promote" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY +name: CI/CD Pipeline + +on: + push: + branches: [master] + pull_request: + branches: [master] + +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +# Pipeline (sequential, fail-fast): +# go-test → e2e-test → build → deploy → publish +# PRs stop after build. Master continues to deploy + publish. + +jobs: + # ─────────────────────────────────────────────────────────────── + # 1. Go Build & Test + # ─────────────────────────────────────────────────────────────── + go-test: + name: "✅ Go Build & Test" + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Go 1.22 + uses: actions/setup-go@v6 + with: + go-version: '1.22' + cache-dependency-path: | + cmd/server/go.sum + cmd/ingestor/go.sum + + - name: Build and test Go server (with coverage) + run: | + set -e -o pipefail + cd cmd/server + go build . + go test -coverprofile=server-coverage.out ./... 2>&1 | tee server-test.log + echo "--- Go Server Coverage ---" + go tool cover -func=server-coverage.out | tail -1 + + - name: Build and test Go ingestor (with coverage) + run: | + set -e -o pipefail + cd cmd/ingestor + go build . + go test -coverprofile=ingestor-coverage.out ./... 2>&1 | tee ingestor-test.log + echo "--- Go Ingestor Coverage ---" + go tool cover -func=ingestor-coverage.out | tail -1 + + - name: Verify proto syntax + run: | + set -e + sudo apt-get update -qq + sudo apt-get install -y protobuf-compiler + for proto in proto/*.proto; do + echo " ✓ $(basename "$proto")" + protoc --proto_path=proto --descriptor_set_out=/dev/null "$proto" + done + echo "✅ All .proto files are syntactically valid" + + - name: Generate Go coverage badges + if: success() + run: | + mkdir -p .badges + + SERVER_COV="0" + if [ -f cmd/server/server-coverage.out ]; then + SERVER_COV=$(cd cmd/server && go tool cover -func=server-coverage.out | tail -1 | grep -oP '[\d.]+(?=%)') + fi + SERVER_COLOR="red" + if [ "$(echo "$SERVER_COV >= 80" | bc -l 2>/dev/null)" = "1" ]; then SERVER_COLOR="green" + elif [ "$(echo "$SERVER_COV >= 60" | bc -l 2>/dev/null)" = "1" ]; then SERVER_COLOR="yellow"; fi + echo "{\"schemaVersion\":1,\"label\":\"go server coverage\",\"message\":\"${SERVER_COV}%\",\"color\":\"${SERVER_COLOR}\"}" > .badges/go-server-coverage.json + + INGESTOR_COV="0" + if [ -f cmd/ingestor/ingestor-coverage.out ]; then + INGESTOR_COV=$(cd cmd/ingestor && go tool cover -func=ingestor-coverage.out | tail -1 | grep -oP '[\d.]+(?=%)') + fi + INGESTOR_COLOR="red" + if [ "$(echo "$INGESTOR_COV >= 80" | bc -l 2>/dev/null)" = "1" ]; then INGESTOR_COLOR="green" + elif [ "$(echo "$INGESTOR_COV >= 60" | bc -l 2>/dev/null)" = "1" ]; then INGESTOR_COLOR="yellow"; fi + echo "{\"schemaVersion\":1,\"label\":\"go ingestor coverage\",\"message\":\"${INGESTOR_COV}%\",\"color\":\"${INGESTOR_COLOR}\"}" > .badges/go-ingestor-coverage.json + + echo "## Go Coverage" >> $GITHUB_STEP_SUMMARY + echo "| Module | Coverage |" >> $GITHUB_STEP_SUMMARY + echo "|--------|----------|" >> $GITHUB_STEP_SUMMARY + echo "| Server | ${SERVER_COV}% |" >> $GITHUB_STEP_SUMMARY + echo "| Ingestor | ${INGESTOR_COV}% |" >> $GITHUB_STEP_SUMMARY + + - name: Upload Go coverage badges + if: success() + uses: actions/upload-artifact@v6 + with: + name: go-badges + path: .badges/go-*.json + retention-days: 1 + if-no-files-found: ignore + + # ─────────────────────────────────────────────────────────────── + # 2. Playwright E2E Tests (against Go server with fixture DB) + # ─────────────────────────────────────────────────────────────── + e2e-test: + name: "🎭 Playwright E2E Tests" + needs: [go-test] + runs-on: [self-hosted, Linux] + defaults: + run: + shell: bash + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Node.js 22 + uses: actions/setup-node@v5 + with: + node-version: '22' + + - name: Set up Go 1.22 + uses: actions/setup-go@v6 + with: + go-version: '1.22' + cache-dependency-path: cmd/server/go.sum + + - name: Build Go server + run: | + cd cmd/server + go build -o ../../corescope-server . + echo "Go server built successfully" + + - name: Install npm dependencies + run: npm ci --production=false + + - name: Install Playwright browser + run: | + npx playwright install chromium 2>/dev/null || true + npx playwright install-deps chromium 2>/dev/null || true + + - name: Instrument frontend JS for coverage + run: sh scripts/instrument-frontend.sh + + - name: Start Go server with fixture DB + run: | + fuser -k 13581/tcp 2>/dev/null || true + sleep 1 + ./corescope-server -port 13581 -db test-fixtures/e2e-fixture.db -public public-instrumented & + echo $! > .server.pid + for i in $(seq 1 30); do + if curl -sf http://localhost:13581/api/stats > /dev/null 2>&1; then + echo "Server ready after ${i}s" + break + fi + if [ "$i" -eq 30 ]; then + echo "Server failed to start within 30s" + exit 1 + fi + sleep 1 + done + + - name: Run Playwright E2E tests (fail-fast) + run: | + BASE_URL=http://localhost:13581 node test-e2e-playwright.js 2>&1 | tee e2e-output.txt + + # DISABLED: Coverage collector takes 8+ min. E2E tests extract window.__coverage__ directly. + # Re-enable when collector is optimized or if E2E coverage numbers are insufficient. + # - name: Collect frontend coverage + # if: success() + # run: | + # BASE_URL=http://localhost:13581 node scripts/collect-frontend-coverage.js 2>&1 | tee fe-coverage-output.txt || true + + - name: Generate frontend coverage badges + if: success() + run: | + E2E_PASS=$(grep -oP '[0-9]+(?=/)' e2e-output.txt | tail -1 || echo "0") + + mkdir -p .badges + if [ -f .nyc_output/frontend-coverage.json ] || [ -f .nyc_output/e2e-coverage.json ]; then + npx nyc report --reporter=text-summary --reporter=text 2>&1 | tee fe-report.txt + FE_COVERAGE=$(grep 'Statements' fe-report.txt | head -1 | grep -oP '[\d.]+(?=%)' || echo "0") + FE_COVERAGE=${FE_COVERAGE:-0} + FE_COLOR="red" + [ "$(echo "$FE_COVERAGE > 50" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="yellow" + [ "$(echo "$FE_COVERAGE > 80" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="brightgreen" + echo "{\"schemaVersion\":1,\"label\":\"frontend coverage\",\"message\":\"${FE_COVERAGE}%\",\"color\":\"${FE_COLOR}\"}" > .badges/frontend-coverage.json + echo "## Frontend: ${FE_COVERAGE}% coverage" >> $GITHUB_STEP_SUMMARY + fi + echo "{\"schemaVersion\":1,\"label\":\"e2e tests\",\"message\":\"${E2E_PASS:-0} passed\",\"color\":\"brightgreen\"}" > .badges/e2e-tests.json + + - name: Stop test server + if: success() + run: | + if [ -f .server.pid ]; then + kill $(cat .server.pid) 2>/dev/null || true + rm -f .server.pid + fi + + - name: Upload E2E badges + if: success() + uses: actions/upload-artifact@v6 + with: + name: e2e-badges + path: .badges/ + retention-days: 1 + if-no-files-found: ignore + + # ─────────────────────────────────────────────────────────────── + # 3. Build Docker Image + # ─────────────────────────────────────────────────────────────── + build: + name: "🏗️ Build Docker Image" + needs: [e2e-test] + runs-on: [self-hosted, Linux] + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Node.js 22 + uses: actions/setup-node@v5 + with: + node-version: '22' + + - name: Build Go Docker image + run: | + echo "${GITHUB_SHA::7}" > .git-commit + APP_VERSION=$(node -p "require('./package.json').version") \ + GIT_COMMIT="${GITHUB_SHA::7}" \ + docker compose --profile staging-go build staging-go + echo "Built Go staging image ✅" + + # ─────────────────────────────────────────────────────────────── + # 4. Deploy Staging (master only) + # ─────────────────────────────────────────────────────────────── + deploy: + name: "🚀 Deploy Staging" + if: github.event_name == 'push' + needs: [build] + runs-on: [self-hosted, Linux] + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Start staging on port 82 + run: | + docker rm -f corescope-staging-go 2>/dev/null || true + fuser -k 82/tcp 2>/dev/null || true + docker compose --profile staging-go up -d staging-go + + - name: Healthcheck staging container + run: | + for i in $(seq 1 120); do + HEALTH=$(docker inspect corescope-staging-go --format '{{.State.Health.Status}}' 2>/dev/null || echo "starting") + if [ "$HEALTH" = "healthy" ]; then + echo "Staging healthy after ${i}s" + break + fi + if [ "$i" -eq 120 ]; then + echo "Staging failed health check after 120s" + docker logs corescope-staging-go --tail 50 + exit 1 + fi + sleep 1 + done + + - name: Smoke test staging API + run: | + if curl -sf http://localhost:82/api/stats | grep -q engine; then + echo "Staging verified — engine field present ✅" + else + echo "Staging /api/stats did not return engine field" + exit 1 + fi + + # ─────────────────────────────────────────────────────────────── + # 5. Publish Badges & Summary (master only) + # ─────────────────────────────────────────────────────────────── + publish: + name: "📝 Publish Badges & Summary" + if: github.event_name == 'push' + needs: [deploy] + runs-on: [self-hosted, Linux] + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Download Go coverage badges + continue-on-error: true + uses: actions/download-artifact@v6 + with: + name: go-badges + path: .badges/ + + - name: Download E2E badges + continue-on-error: true + uses: actions/download-artifact@v6 + with: + name: e2e-badges + path: .badges/ + + - name: Publish coverage badges to repo + continue-on-error: true + run: | + git config user.name "github-actions" + git config user.email "actions@github.com" + git remote set-url origin https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git + git add .badges/ -f + git diff --cached --quiet || (git commit -m "ci: update test badges [skip ci]" && git push) || echo "Badge push failed" + + - name: Post deployment summary + run: | + echo "## Staging Deployed ✓" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Commit:** \`$(git rev-parse --short HEAD)\` — $(git log -1 --format=%s)" >> $GITHUB_STEP_SUMMARY diff --git a/.gitignore b/.gitignore index 66261f71..2a035606 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ reps.txt cmd/server/server.exe cmd/ingestor/ingestor.exe # CI trigger +!test-fixtures/e2e-fixture.db diff --git a/scripts/capture-fixture.sh b/scripts/capture-fixture.sh new file mode 100755 index 00000000..c491d248 --- /dev/null +++ b/scripts/capture-fixture.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# Capture a fixture DB from staging for E2E tests +# Usage: ./scripts/capture-fixture.sh [source_url] +# +# Downloads nodes, observers, and recent packets from the staging API +# and creates a SQLite database suitable for E2E testing. + +set -e + +SOURCE_URL="${1:-https://analyzer.00id.net}" +DB_PATH="test-fixtures/e2e-fixture.db" + +echo "Capturing fixture from $SOURCE_URL..." + +mkdir -p test-fixtures +rm -f "$DB_PATH" + +# Create schema +sqlite3 "$DB_PATH" <<'SQL' +CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat REAL, + lon REAL, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c REAL +); + +CREATE TABLE observers ( + id TEXT PRIMARY KEY, + name TEXT, + iata TEXT, + last_seen TEXT, + first_seen TEXT, + packet_count INTEGER DEFAULT 0, + model TEXT, + firmware TEXT, + client_version TEXT, + radio TEXT, + battery_mv INTEGER, + uptime_secs INTEGER, + noise_floor REAL +); + +CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + created_at TEXT DEFAULT (datetime('now')) +); + +CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL +); +SQL + +# Fetch nodes +echo "Fetching nodes..." +curl -sf "$SOURCE_URL/api/nodes?limit=200" | python3 -c " +import json, sys, sqlite3 +data = json.load(sys.stdin) +nodes = data.get('nodes', data) if isinstance(data, dict) else data +db = sqlite3.connect('$DB_PATH') +for n in nodes[:200]: + db.execute('INSERT OR IGNORE INTO nodes VALUES (?,?,?,?,?,?,?,?,?,?)', + (n.get('public_key',''), n.get('name',''), n.get('role',''), + n.get('lat'), n.get('lon'), n.get('last_seen',''), n.get('first_seen',''), + n.get('advert_count',0), n.get('battery_mv'), n.get('temperature_c'))) +db.commit() +print(f' Inserted {min(len(nodes), 200)} nodes') +db.close() +" + +# Fetch observers +echo "Fetching observers..." +curl -sf "$SOURCE_URL/api/observers" | python3 -c " +import json, sys, sqlite3 +data = json.load(sys.stdin) +observers = data.get('observers', data) if isinstance(data, dict) else data +db = sqlite3.connect('$DB_PATH') +for o in observers: + db.execute('INSERT OR IGNORE INTO observers VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)', + (o.get('id',''), o.get('name',''), o.get('iata',''), + o.get('last_seen',''), o.get('first_seen',''), + o.get('packet_count',0), o.get('model',''), o.get('firmware',''), + o.get('client_version',''), o.get('radio',''), + o.get('battery_mv'), o.get('uptime_secs'), o.get('noise_floor'))) +db.commit() +print(f' Inserted {len(observers)} observers') +db.close() +" + +# Fetch recent packets +echo "Fetching recent packets..." +curl -sf "$SOURCE_URL/api/packets?limit=500" | python3 -c " +import json, sys, sqlite3 +data = json.load(sys.stdin) +packets = data.get('packets', data) if isinstance(data, dict) else data +db = sqlite3.connect('$DB_PATH') +for p in packets: + try: + cur = db.execute('INSERT OR IGNORE INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?,?,?,?,?,?,?)', + (p.get('raw_hex',''), p.get('hash',''), p.get('first_seen',''), + p.get('route_type'), p.get('payload_type'), p.get('payload_version'), + p.get('decoded_json'))) + tid = cur.lastrowid + if tid and p.get('observer_id'): + db.execute('INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) VALUES (?,?,?,?,?,?,?,?)', + (tid, p.get('observer_id'), p.get('direction'), + p.get('snr'), p.get('rssi'), None, + p.get('path_json'), + int(p.get('timestamp','0')) if p.get('timestamp','').isdigit() else 0)) + except Exception as e: + pass # Skip duplicates +db.commit() +print(f' Inserted {len(packets)} transmissions') +db.close() +" + +SIZE=$(du -h "$DB_PATH" | cut -f1) +echo "✅ Fixture DB created: $DB_PATH ($SIZE)" +echo " Nodes: $(sqlite3 "$DB_PATH" 'SELECT COUNT(*) FROM nodes')" +echo " Observers: $(sqlite3 "$DB_PATH" 'SELECT COUNT(*) FROM observers')" +echo " Transmissions: $(sqlite3 "$DB_PATH" 'SELECT COUNT(*) FROM transmissions')" +echo " Observations: $(sqlite3 "$DB_PATH" 'SELECT COUNT(*) FROM observations')" diff --git a/test-e2e-playwright.js b/test-e2e-playwright.js index 6aaf6003..ee2412bc 100644 --- a/test-e2e-playwright.js +++ b/test-e2e-playwright.js @@ -17,6 +17,8 @@ async function test(name, fn) { } catch (err) { results.push({ name, pass: false, error: err.message }); console.log(` \u274c ${name}: ${err.message}`); + console.log(`\nFail-fast: stopping after first failure.`); + process.exit(1); } } diff --git a/test-fixtures/e2e-fixture.db b/test-fixtures/e2e-fixture.db new file mode 100644 index 00000000..3a8ad83e Binary files /dev/null and b/test-fixtures/e2e-fixture.db differ