|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +set -euo pipefail |
| 4 | + |
| 5 | +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 6 | +MATCH_FILTER="$SCRIPT_DIR/match-vulnerability-issue.jq" |
| 7 | +CLASSIFY_FILTER="$SCRIPT_DIR/classify-vulnerability-issues.jq" |
| 8 | +LINEAR_REQUEST="$SCRIPT_DIR/linear-graphql-request.sh" |
| 9 | +WORKFLOW_FILE="$SCRIPT_DIR/../workflows/vulnerability-triage.yml" |
| 10 | +PREFIX="[sourcebot-dev/example]" |
| 11 | + |
| 12 | +assert_json() { |
| 13 | + local description="$1" |
| 14 | + local actual="$2" |
| 15 | + local expected="$3" |
| 16 | + |
| 17 | + if ! jq -e --argjson expected "$expected" '. == $expected' <<<"$actual" >/dev/null; then |
| 18 | + echo "FAIL: $description" |
| 19 | + echo "Expected: $expected" |
| 20 | + echo "Actual: $actual" |
| 21 | + exit 1 |
| 22 | + fi |
| 23 | +} |
| 24 | + |
| 25 | +assert_workflow_contains() { |
| 26 | + local description="$1" |
| 27 | + local expected="$2" |
| 28 | + |
| 29 | + if ! grep -Fq -- "$expected" "$WORKFLOW_FILE"; then |
| 30 | + echo "FAIL: $description" |
| 31 | + echo "Expected workflow to contain: $expected" |
| 32 | + exit 1 |
| 33 | + fi |
| 34 | +} |
| 35 | + |
| 36 | +assert_workflow_contains "checks out assets from the called workflow repository" 'repository: ${{ job.workflow_repository }}' |
| 37 | +assert_workflow_contains "pins assets to the called workflow revision" 'ref: ${{ job.workflow_sha }}' |
| 38 | +assert_workflow_contains "uses the shared match filter" '-f .vulnerability-triage-workflow/.github/scripts/match-vulnerability-issue.jq' |
| 39 | +assert_workflow_contains "uses the shared classification filter" '-f .vulnerability-triage-workflow/.github/scripts/classify-vulnerability-issues.jq' |
| 40 | +assert_workflow_contains "uses the retrying Linear GraphQL client" '.vulnerability-triage-workflow/.github/scripts/linear-graphql-request.sh' |
| 41 | + |
| 42 | +FAKE_CURL_DIR=$(mktemp -d) |
| 43 | +FAKE_CURL_COUNT=$(mktemp) |
| 44 | +trap 'rm -rf "$FAKE_CURL_DIR"; rm -f "$FAKE_CURL_COUNT"' EXIT |
| 45 | +printf '0\n' > "$FAKE_CURL_COUNT" |
| 46 | + |
| 47 | +cat > "$FAKE_CURL_DIR/curl" <<'EOF' |
| 48 | +#!/usr/bin/env bash |
| 49 | +set -euo pipefail |
| 50 | +
|
| 51 | +output_file="" |
| 52 | +while (($# > 0)); do |
| 53 | + case "$1" in |
| 54 | + --output) |
| 55 | + output_file="$2" |
| 56 | + shift 2 |
| 57 | + ;; |
| 58 | + *) |
| 59 | + shift |
| 60 | + ;; |
| 61 | + esac |
| 62 | +done |
| 63 | +
|
| 64 | +count=$(( $(<"$FAKE_CURL_COUNT") + 1 )) |
| 65 | +printf '%s\n' "$count" > "$FAKE_CURL_COUNT" |
| 66 | +if ((count < 3)); then |
| 67 | + printf 'Bad Gateway' > "$output_file" |
| 68 | + printf '502' |
| 69 | +else |
| 70 | + printf '{"data":{"ok":true}}' > "$output_file" |
| 71 | + printf '200' |
| 72 | +fi |
| 73 | +EOF |
| 74 | +chmod +x "$FAKE_CURL_DIR/curl" |
| 75 | + |
| 76 | +LINEAR_RESPONSE=$( |
| 77 | + PATH="$FAKE_CURL_DIR:$PATH" \ |
| 78 | + FAKE_CURL_COUNT="$FAKE_CURL_COUNT" \ |
| 79 | + LINEAR_API_KEY="test-key" \ |
| 80 | + LINEAR_GRAPHQL_ATTEMPTS=3 \ |
| 81 | + LINEAR_GRAPHQL_RETRY_DELAY_SECONDS=0 \ |
| 82 | + "$LINEAR_REQUEST" <<<'{"query":"query { viewer { id } }"}' |
| 83 | +) |
| 84 | +assert_json "retries non-JSON Linear responses" "$LINEAR_RESPONSE" '{"data":{"ok":true}}' |
| 85 | +if [[ "$(<"$FAKE_CURL_COUNT")" != "3" ]]; then |
| 86 | + echo "FAIL: expected Linear request helper to retry twice" |
| 87 | + exit 1 |
| 88 | +fi |
| 89 | + |
| 90 | +printf '0\n' > "$FAKE_CURL_COUNT" |
| 91 | +if PATH="$FAKE_CURL_DIR:$PATH" \ |
| 92 | + FAKE_CURL_COUNT="$FAKE_CURL_COUNT" \ |
| 93 | + LINEAR_API_KEY="test-key" \ |
| 94 | + LINEAR_GRAPHQL_ATTEMPTS=3 \ |
| 95 | + LINEAR_GRAPHQL_RETRY_DELAY_SECONDS=0 \ |
| 96 | + "$LINEAR_REQUEST" <<<'{"query":"mutation { issueCreate(input: {}) { success } }"}' >/dev/null 2>&1; then |
| 97 | + echo "FAIL: ambiguous Linear mutations must not be retried" |
| 98 | + exit 1 |
| 99 | +fi |
| 100 | +if [[ "$(<"$FAKE_CURL_COUNT")" != "1" ]]; then |
| 101 | + echo "FAIL: expected exactly one Linear mutation attempt" |
| 102 | + exit 1 |
| 103 | +fi |
| 104 | + |
| 105 | +match() { |
| 106 | + local finding_id="$1" |
| 107 | + jq -c --arg prefix "$PREFIX" --arg findingId "$finding_id" -f "$MATCH_FILTER" |
| 108 | +} |
| 109 | + |
| 110 | +EMPTY_MATCH='{"linearIssueExists":false,"linearIssueId":"","linearIssueIdentifier":"","linearIssueUrl":"","linearIssueClosed":false}' |
| 111 | +OPEN_MATCH='{"linearIssueExists":true,"linearIssueId":"open-id","linearIssueIdentifier":"SOU-2","linearIssueUrl":"https://linear.app/SOU-2","linearIssueClosed":false}' |
| 112 | +CLOSED_MATCH='{"linearIssueExists":true,"linearIssueId":"closed-id","linearIssueIdentifier":"SOU-1","linearIssueUrl":"https://linear.app/SOU-1","linearIssueClosed":true}' |
| 113 | + |
| 114 | +NO_ISSUES='{"data":{"issues":{"nodes":[]}}}' |
| 115 | +assert_json "creates when no Linear issue exists" "$(match CVE-2026-123 <<<"$NO_ISSUES")" "$EMPTY_MATCH" |
| 116 | + |
| 117 | +ISSUES='{ |
| 118 | + "data": { |
| 119 | + "issues": { |
| 120 | + "nodes": [ |
| 121 | + { |
| 122 | + "id": "closed-id", |
| 123 | + "identifier": "SOU-1", |
| 124 | + "url": "https://linear.app/SOU-1", |
| 125 | + "title": "[sourcebot-dev/example] CVE-2026-123: old finding", |
| 126 | + "state": {"type": "completed"} |
| 127 | + }, |
| 128 | + { |
| 129 | + "id": "open-id", |
| 130 | + "identifier": "SOU-2", |
| 131 | + "url": "https://linear.app/SOU-2", |
| 132 | + "title": "[sourcebot-dev/example] CVE-2026-123: current finding", |
| 133 | + "state": {"type": "started"} |
| 134 | + }, |
| 135 | + { |
| 136 | + "id": "prefix-collision", |
| 137 | + "identifier": "SOU-3", |
| 138 | + "url": "https://linear.app/SOU-3", |
| 139 | + "title": "[sourcebot-dev/example] CVE-2026-1234: different finding", |
| 140 | + "state": {"type": "started"} |
| 141 | + }, |
| 142 | + { |
| 143 | + "id": "other-repo", |
| 144 | + "identifier": "SOU-4", |
| 145 | + "url": "https://linear.app/SOU-4", |
| 146 | + "title": "[sourcebot-dev/other] CVE-2026-123: different repository", |
| 147 | + "state": {"type": "started"} |
| 148 | + }, |
| 149 | + { |
| 150 | + "id": "duplicate-id", |
| 151 | + "identifier": "SOU-5", |
| 152 | + "url": "https://linear.app/SOU-5", |
| 153 | + "title": "[sourcebot-dev/example] CVE-2026-123: duplicate marker", |
| 154 | + "state": {"type": "duplicate"} |
| 155 | + } |
| 156 | + ] |
| 157 | + } |
| 158 | + } |
| 159 | +}' |
| 160 | +assert_json "deduplicates by exact finding id and prefers an open issue" "$(match CVE-2026-123 <<<"$ISSUES")" "$OPEN_MATCH" |
| 161 | +assert_json "does not confuse an id with a longer id" "$(match CVE-2026-12 <<<"$ISSUES")" "$EMPTY_MATCH" |
| 162 | + |
| 163 | +CLOSED_ONLY="$(jq '.data.issues.nodes |= map(select(.id == "closed-id"))' <<<"$ISSUES")" |
| 164 | +assert_json "reopens a closed issue when the finding returns" "$(match CVE-2026-123 <<<"$CLOSED_ONLY")" "$CLOSED_MATCH" |
| 165 | + |
| 166 | +DUPLICATE_ONLY="$(jq '.data.issues.nodes |= map(select(.id == "duplicate-id"))' <<<"$ISSUES")" |
| 167 | +DUPLICATE_MATCH='{"linearIssueExists":true,"linearIssueId":"duplicate-id","linearIssueIdentifier":"SOU-5","linearIssueUrl":"https://linear.app/SOU-5","linearIssueClosed":false}' |
| 168 | +assert_json "does not reopen a duplicate marker" "$(match CVE-2026-123 <<<"$DUPLICATE_ONLY")" "$DUPLICATE_MATCH" |
| 169 | + |
| 170 | +FINDINGS='{ |
| 171 | + "cves": [ |
| 172 | + {"cveId": "CVE-2026-123"}, |
| 173 | + {"cveId": "GHSA-abcd-efgh-ijkl"}, |
| 174 | + {"cveId": "codeql:js/example-rule"} |
| 175 | + ] |
| 176 | +}' |
| 177 | +OPEN_ISSUES='[ |
| 178 | + { |
| 179 | + "id": "keep-cve", |
| 180 | + "title": "[sourcebot-dev/example] CVE-2026-123: still present" |
| 181 | + }, |
| 182 | + { |
| 183 | + "id": "keep-codeql", |
| 184 | + "title": "[sourcebot-dev/example] codeql:js/example-rule: still present" |
| 185 | + }, |
| 186 | + { |
| 187 | + "id": "close-cve", |
| 188 | + "title": "[sourcebot-dev/example] CVE-2025-999: resolved" |
| 189 | + }, |
| 190 | + { |
| 191 | + "id": "close-prefix-collision", |
| 192 | + "title": "[sourcebot-dev/example] CVE-2026-12: resolved" |
| 193 | + }, |
| 194 | + { |
| 195 | + "id": "ignore-other-repo", |
| 196 | + "title": "[sourcebot-dev/other] CVE-2025-999: unrelated" |
| 197 | + }, |
| 198 | + { |
| 199 | + "id": "ignore-unmanaged", |
| 200 | + "title": "[sourcebot-dev/example] maintenance without a finding delimiter" |
| 201 | + } |
| 202 | +]' |
| 203 | + |
| 204 | +CLASSIFIED="$(jq -c --arg prefix "$PREFIX" --argjson skipCodeql false --slurpfile findings <(printf '%s\n' "$FINDINGS") -f "$CLASSIFY_FILTER" <<<"$OPEN_ISSUES")" |
| 205 | +assert_json "keeps every exact current finding" "$(jq -c '[.[] | select(.action == "keep") | .id]' <<<"$CLASSIFIED")" '["keep-cve","keep-codeql"]' |
| 206 | +assert_json "closes only resolved managed findings" "$(jq -c '[.[] | select(.action == "close") | .id]' <<<"$CLASSIFIED")" '["close-cve","close-prefix-collision"]' |
| 207 | +assert_json "ignores unrelated titles and repositories" "$(jq -c '[.[] | select(.action == "ignore") | .id]' <<<"$CLASSIFIED")" '["ignore-other-repo","ignore-unmanaged"]' |
| 208 | + |
| 209 | +EMPTY_FINDINGS='{"cves":[]}' |
| 210 | +CLASSIFIED_EMPTY="$(jq -c --arg prefix "$PREFIX" --argjson skipCodeql false --slurpfile findings <(printf '%s\n' "$EMPTY_FINDINGS") -f "$CLASSIFY_FILTER" <<<"$OPEN_ISSUES")" |
| 211 | +assert_json "closes all managed issues when a complete scan is clean" "$(jq -c '[.[] | select(.action == "close") | .id]' <<<"$CLASSIFIED_EMPTY")" '["keep-cve","keep-codeql","close-cve","close-prefix-collision"]' |
| 212 | + |
| 213 | +CLASSIFIED_WITH_CODEQL_SKIPPED="$(jq -c --arg prefix "$PREFIX" --argjson skipCodeql true --slurpfile findings <(printf '%s\n' "$EMPTY_FINDINGS") -f "$CLASSIFY_FILTER" <<<"$OPEN_ISSUES")" |
| 214 | +assert_json "preserves CodeQL issues when CodeQL is unavailable" "$(jq -c '[.[] | select(.action == "ignore") | .id]' <<<"$CLASSIFIED_WITH_CODEQL_SKIPPED")" '["keep-codeql","ignore-other-repo","ignore-unmanaged"]' |
| 215 | +assert_json "still closes resolved non-CodeQL issues when CodeQL is unavailable" "$(jq -c '[.[] | select(.action == "close") | .id]' <<<"$CLASSIFIED_WITH_CODEQL_SKIPPED")" '["keep-cve","close-cve","close-prefix-collision"]' |
| 216 | + |
| 217 | +echo "All vulnerability triage reconciliation tests passed." |
0 commit comments