diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b93ab03b0e..2c5d976ff7 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -386,6 +386,13 @@ jobs: run: | ./hack/gh-workflow-ci.sh run_e2e_tests + - name: Generate GitHub Step Summary + if: ${{ always() }} + env: + TEST_PROVIDER: ${{ matrix.provider }} + run: | + ./hack/gh-workflow-ci.sh generate_github_summary + - name: Collect logs if: ${{ always() }} env: diff --git a/hack/generate_github_summary.py b/hack/generate_github_summary.py new file mode 100755 index 0000000000..a85fde6667 --- /dev/null +++ b/hack/generate_github_summary.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Parse gotestsum JSON output and generate a GitHub Step Summary in markdown.""" + +import collections +import json +import sys + + +def main(): + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + json_file = sys.argv[1] + target = sys.argv[2] + + results = {} # test_name -> {action, elapsed} + test_output = collections.defaultdict(list) + + with open(json_file) as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + test = entry.get("Test") + if not test: + continue + action = entry.get("Action", "") + if action == "output": + test_output[test].append(entry.get("Output", "")) + elif action in ("pass", "fail", "skip"): + results[test] = { + "action": action, + "elapsed": entry.get("Elapsed", 0), + } + + passed = sorted([t for t, r in results.items() if r["action"] == "pass"]) + failed = sorted([t for t, r in results.items() if r["action"] == "fail"]) + skipped = sorted([t for t, r in results.items() if r["action"] == "skip"]) + + # Filter to top-level tests only (no subtests) for counts + top_passed = [t for t in passed if "/" not in t] + top_failed = [t for t in failed if "/" not in t] + top_skipped = [t for t in skipped if "/" not in t] + + status = ":x: FAILED" if top_failed else ":white_check_mark: PASSED" + + lines = [] + lines.append(f"## E2E Tests: {target} {status}") + lines.append("") + lines.append( + f":white_check_mark: **{len(top_passed)}** passed" + f" | :x: **{len(top_failed)}** failed" + f" | :fast_forward: **{len(top_skipped)}** skipped" + ) + lines.append("") + + if top_failed: + lines.append("### Failed Tests") + lines.append("") + for t in top_failed: + elapsed = results[t]["elapsed"] + lines.append(f"
:x: {t} ({elapsed:.1f}s)") + lines.append("") + lines.append("```") + output = "".join(test_output.get(t, [])) + # Include subtest output too + for sub, r in sorted(results.items()): + if sub.startswith(t + "/") and r["action"] == "fail": + output += "".join(test_output.get(sub, [])) + lines.append(output.rstrip()) + lines.append("```") + lines.append("") + lines.append("
") + lines.append("") + + if top_passed: + lines.append("
Passed Tests") + lines.append("") + for t in top_passed: + elapsed = results[t]["elapsed"] + lines.append(f"- :white_check_mark: {t} ({elapsed:.1f}s)") + lines.append("") + lines.append("
") + lines.append("") + + if top_skipped: + lines.append("
Skipped Tests") + lines.append("") + for t in top_skipped: + lines.append(f"- :fast_forward: {t}") + lines.append("") + lines.append("
") + + print("\n".join(lines)) + + +if __name__ == "__main__": + main() diff --git a/hack/gh-workflow-ci.sh b/hack/gh-workflow-ci.sh index b9432d9b3b..abbfc862df 100755 --- a/hack/gh-workflow-ci.sh +++ b/hack/gh-workflow-ci.sh @@ -115,8 +115,8 @@ get_tests() { # which runs the workflow YAML from main (old target names). local github_chunk_size github_remainder if [[ ${#github_tests[@]} -gt 0 ]]; then - github_chunk_size=$(( ${#github_tests[@]} / 2 )) - github_remainder=$(( ${#github_tests[@]} % 2 )) + github_chunk_size=$((${#github_tests[@]} / 2)) + github_remainder=$((${#github_tests[@]} % 2)) fi case "${target}" in @@ -287,6 +287,23 @@ collect_logs() { detect_panic } +generate_github_summary() { + local raw_output="/tmp/logs/e2e-test-output.json" + local target="${TEST_PROVIDER:-unknown}" + + if [[ -z "${GITHUB_STEP_SUMMARY:-}" ]]; then + echo "GITHUB_STEP_SUMMARY not set, skipping summary generation" + return 0 + fi + + if [[ ! -f "${raw_output}" ]]; then + echo "No test output found at ${raw_output}, skipping summary generation" + return 0 + fi + + python3 ./hack/generate_github_summary.py "${raw_output}" "${target}" >>"${GITHUB_STEP_SUMMARY}" +} + detect_panic() { # shellcheck disable=SC2016 (find /tmp/logs/ -type f -regex '.*/pipelines-as-code.*/[0-9]\.log$' | xargs -r sed -n '/stderr F panic:.*/,$p' | head -n 80) >/tmp/panic.log @@ -328,6 +345,10 @@ help() { Will output logs using snazzy formatting when available or otherwise through a simple python formatter. This makes debugging easier from the GitHub Actions interface. + generate_github_summary + Parse gotestsum JSON output and write a markdown summary to GITHUB_STEP_SUMMARY. + Required env vars: TEST_PROVIDER, GITHUB_STEP_SUMMARY + print_tests Print the list of tests that would be run for each provider target. @@ -350,6 +371,9 @@ collect_logs) output_logs) output_logs ;; +generate_github_summary) + generate_github_summary + ;; print_tests) set +x for target in github_public github_ghe_1 github_ghe_2 github_ghe_3 gitlab_bitbucket gitea_1 gitea_2 gitea_3 concurrency flaky; do