-
Notifications
You must be signed in to change notification settings - Fork 51
feat(ci): add hourly canary for smoke test #1486
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,163 @@ | ||
| /** | ||
| * Creates a GitHub issue for an automated failure. Ex. Canary failure. | ||
| * | ||
| * Usage: | ||
| * npx tsx .github/scripts/create-failure-issue.ts \ | ||
| * --title-prefix "CI Failure" \ | ||
| * --name "Build and Test" \ | ||
| * --branch main \ | ||
| * --commit <sha> \ | ||
| * --run-url <url> \ | ||
| * [--labels high-severity,ci] \ | ||
| * [--detail "Variant: Released/Preview"] | ||
| * | ||
| * Required env: | ||
| * GH_TOKEN (or GITHUB_TOKEN) — token with `issues: write` | ||
| * GITHUB_REPOSITORY — "owner/repo" (auto-set by GitHub Actions) | ||
| * | ||
| */ | ||
|
|
||
| const GITHUB_API_BASE_URL = 'https://api.github.com'; | ||
|
|
||
| interface CreateIssueArgs { | ||
| titlePrefix: string; | ||
| name: string; | ||
| branch: string; | ||
| commit: string; | ||
| runUrl: string; | ||
| labels: string[]; | ||
| detail?: string; | ||
| } | ||
|
|
||
| function parseArgs(argv: string[]): CreateIssueArgs { | ||
| const map = new Map<string, string>(); | ||
| for (let i = 0; i < argv.length; i++) { | ||
| const arg = argv[i]; | ||
| if (arg.startsWith('--')) { | ||
| const key = arg.slice(2); | ||
| const value = argv[i + 1]; | ||
| if (value === undefined || value.startsWith('--')) { | ||
| throw new Error(`Missing value for --${key}`); | ||
| } | ||
| map.set(key, value); | ||
| i++; | ||
| } | ||
| } | ||
|
|
||
| const required = ['title-prefix', 'name', 'commit', 'run-url']; | ||
| const missing = required.filter(k => !map.get(k)); | ||
| if (missing.length > 0) { | ||
| throw new Error(`Missing required arguments: ${missing.map(k => `--${k}`).join(', ')}`); | ||
| } | ||
|
|
||
| return { | ||
| titlePrefix: map.get('title-prefix')!, | ||
| name: map.get('name')!, | ||
| branch: map.get('branch') ?? 'main', | ||
| commit: map.get('commit')!, | ||
| runUrl: map.get('run-url')!, | ||
| labels: (map.get('labels') ?? 'high-severity,ci') | ||
| .split(',') | ||
| .map(l => l.trim()) | ||
| .filter(Boolean), | ||
| detail: map.get('detail'), | ||
| }; | ||
| } | ||
|
|
||
| function getRepo(): { owner: string; repo: string } { | ||
| const repository = process.env.GITHUB_REPOSITORY; | ||
| if (!repository || !repository.includes('/')) { | ||
| throw new Error('GITHUB_REPOSITORY env var must be set to "owner/repo"'); | ||
| } | ||
| const [owner, repo] = repository.split('/'); | ||
| return { owner, repo }; | ||
| } | ||
|
|
||
| function ghHeaders(token: string): Record<string, string> { | ||
| return { | ||
| Authorization: `Bearer ${token}`, | ||
| Accept: 'application/vnd.github+json', | ||
| 'X-GitHub-Api-Version': '2022-11-28', | ||
| 'User-Agent': 'agentcore-cli-failure-issue-script', | ||
| }; | ||
| } | ||
|
|
||
| async function ghFetch(url: string, token: string, init?: RequestInit): Promise<unknown> { | ||
| const res = await fetch(url, { | ||
| ...init, | ||
| headers: { ...ghHeaders(token), ...(init?.headers ?? {}) }, | ||
| }); | ||
| if (!res.ok) { | ||
| const text = await res.text(); | ||
| throw new Error(`GitHub API ${init?.method ?? 'GET'} ${url} failed: ${res.status} ${res.statusText} — ${text}`); | ||
| } | ||
| return res.json(); | ||
| } | ||
|
|
||
| interface IssueItem { | ||
| title: string; | ||
| } | ||
|
|
||
| async function issueExists(owner: string, repo: string, title: string, token: string): Promise<boolean> { | ||
| // Primary: search index (fast, but can be stale). | ||
| const q = encodeURIComponent(`repo:${owner}/${repo} is:issue is:open in:title "${title}"`); | ||
| const search = (await ghFetch(`${GITHUB_API_BASE_URL}/search/issues?q=${q}`, token)) as { items?: IssueItem[] }; | ||
| if ((search.items ?? []).some(i => i.title === title)) { | ||
| return true; | ||
| } | ||
|
|
||
| // Fallback: scan recent open issues in case the search index is stale. | ||
| const recent = (await ghFetch( | ||
| `${GITHUB_API_BASE_URL}/repos/${owner}/${repo}/issues?state=open&sort=created&direction=desc&per_page=30`, | ||
| token | ||
| )) as IssueItem[]; | ||
| return recent.some(i => i.title === title); | ||
| } | ||
|
|
||
| async function main(): Promise<void> { | ||
| const token = process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN; | ||
| if (!token) { | ||
| throw new Error('GH_TOKEN (or GITHUB_TOKEN) env var is required'); | ||
| } | ||
|
|
||
| const args = parseArgs(process.argv.slice(2)); | ||
| console.log(`args: ${JSON.stringify(args)}`); | ||
| const { owner, repo } = getRepo(); | ||
|
|
||
| const titleField = `${args.titlePrefix}: ${args.name}`; | ||
|
|
||
| if (await issueExists(owner, repo, titleField, token)) { | ||
| console.log(`Issue already exists for "${titleField}" — skipping creation.`); | ||
| return; | ||
| } | ||
|
|
||
| const bodyLines = [ | ||
| `## ${args.titlePrefix}`, | ||
| '', | ||
| `- **Name:** ${args.name}`, | ||
| `- **Branch:** ${args.branch}`, | ||
| `- **Commit:** ${args.commit}`, | ||
| `- **Run:** ${args.runUrl}`, | ||
| ]; | ||
| if (args.detail) { | ||
| bodyLines.push(`- **Detail:** ${args.detail}`); | ||
| } | ||
| bodyLines.push('', 'Please investigate this failure.'); | ||
|
|
||
| await ghFetch(`${GITHUB_API_BASE_URL}/repos/${owner}/${repo}/issues`, token, { | ||
| method: 'POST', | ||
| headers: { 'Content-Type': 'application/json' }, | ||
| body: JSON.stringify({ | ||
| title: titleField, | ||
| labels: args.labels, | ||
| body: bodyLines.join('\n'), | ||
| }), | ||
| }); | ||
|
|
||
| console.log(`Created issue "${titleField}".`); | ||
| } | ||
|
|
||
| main().catch((error: unknown) => { | ||
| console.error(`Failed to create failure issue: ${error instanceof Error ? error.message : String(error)}`); | ||
| process.exit(1); | ||
| }); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| name: Canary | ||
|
|
||
| on: | ||
| # Run once an hour | ||
| schedule: | ||
| - cron: '0 * * * *' | ||
| workflow_dispatch: | ||
| inputs: | ||
| aws_region: | ||
| description: 'AWS region for deployment' | ||
| default: 'us-east-1' | ||
|
|
||
| # Serialize canary runs so overlapping scheduled runs don't pile up. Matrix | ||
| # jobs within a single run still execute in parallel. | ||
| concurrency: | ||
| group: canary | ||
| cancel-in-progress: false | ||
|
|
||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| issues: write | ||
|
|
||
| env: | ||
| AGENTCORE_TELEMETRY_DISABLED: '1' | ||
| PRERELEASE_BASE_URL: https://github.com/aws/agentcore-cli/releases/download/prerelease | ||
|
|
||
| jobs: | ||
| canary: | ||
| runs-on: ubuntu-latest | ||
| environment: e2e-testing | ||
| timeout-minutes: 30 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| variant: [Released, Prerelease] | ||
| build: [GA, Preview] | ||
| steps: | ||
| - uses: actions/checkout@v6 | ||
| - uses: actions/setup-node@v6 | ||
| with: | ||
| node-version: '20.x' | ||
| cache: 'npm' | ||
| - name: Configure git | ||
| run: | | ||
| git config --global user.email "ci@amazon.com" | ||
| git config --global user.name "CI" | ||
| - uses: astral-sh/setup-uv@v7 | ||
| - name: Configure AWS credentials | ||
| uses: aws-actions/configure-aws-credentials@v6 | ||
| with: | ||
| role-to-assume: ${{ secrets.E2E_AWS_ROLE_ARN }} | ||
| aws-region: ${{ inputs.aws_region || 'us-east-1' }} | ||
| - name: Get AWS Account ID | ||
| id: aws | ||
| run: echo "account_id=$(aws sts get-caller-identity --query Account --output text)" >> "$GITHUB_OUTPUT" | ||
| - name: Get API keys from Secrets Manager | ||
| uses: aws-actions/aws-secretsmanager-get-secrets@v2 | ||
| with: | ||
| secret-ids: | | ||
| E2E,${{ secrets.E2E_SECRET_ARN }} | ||
| parse-json-secrets: true | ||
|
|
||
| - run: npm ci | ||
|
|
||
| # Resolve the install spec for this matrix cell, then install the | ||
| # published CLI globally (it already bundles the CDK constructs). | ||
| - name: Install agentcore CLI (${{ matrix.variant }} / ${{ matrix.build }}) | ||
| id: install | ||
| run: | | ||
| case "${{ matrix.variant }}/${{ matrix.build }}" in | ||
| "Released/GA") SPEC="@aws/agentcore" ;; | ||
| "Released/Preview") SPEC="@aws/agentcore@preview" ;; | ||
| "Prerelease/GA") SPEC="${PRERELEASE_BASE_URL}/agentcore-cli-prerelease.tgz" ;; | ||
| "Prerelease/Preview") SPEC="${PRERELEASE_BASE_URL}/agentcore-cli-prerelease-preview.tgz" ;; | ||
| *) echo "Unknown variant/build combination" >&2; exit 1 ;; | ||
| esac | ||
| echo "Installing: $SPEC" | ||
| echo "spec=$SPEC" >> "$GITHUB_OUTPUT" | ||
| npm install -g "$SPEC" | ||
| echo "agentcore version: $(agentcore --version || echo unknown)" | ||
|
|
||
| - name: Run canary smoke test | ||
| env: | ||
| AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }} | ||
| AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }} | ||
| ANTHROPIC_API_KEY: ${{ env.E2E_ANTHROPIC_API_KEY }} | ||
| OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} | ||
| GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} | ||
| # Smoke test for now, only runs strands-bedrock.test.ts | ||
| run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts --retry 3 | ||
|
|
||
| - name: Generate GitHub App Token | ||
| if: failure() | ||
| id: app-token | ||
| uses: actions/create-github-app-token@v1 | ||
| with: | ||
| app-id: ${{ vars.APP_ID }} | ||
| private-key: ${{ secrets.APP_PRIVATE_KEY }} | ||
|
|
||
| - name: Create canary failure issue | ||
| if: failure() | ||
| env: | ||
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | ||
| VARIANT_NAME: ${{ matrix.variant }}/${{ matrix.build }} | ||
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | ||
| INSTALL_SPEC: ${{ steps.install.outputs.spec }} | ||
| run: | | ||
| npx -y tsx .github/scripts/create-failure-issue.ts \ | ||
| --title-prefix "Canary Failure" \ | ||
| --name "$VARIANT_NAME" \ | ||
| --branch "${{ github.ref_name }}" \ | ||
| --commit "${{ github.sha }}" \ | ||
| --run-url "$RUN_URL" \ | ||
| --labels "high-severity,ci" \ | ||
| --detail "strands-bedrock smoke test; installed from ${INSTALL_SPEC:-unknown}" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This removes the Monday cron from
e2e-full-testwhich in my opinion shrinks our coverage pretty significantly. The new canary replaces it with onlystrands-bedrock.test.ts(1 of 30 e2e suites). For catching CDK schema drift this is fine. But a regression specific to, say, LangGraph+Gemini or container builds is now only caught on push-to-main, not on a schedule. Is this intentional?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, intentional for a few reasons: