diff --git a/.github/scripts/create-failure-issue.ts b/.github/scripts/create-failure-issue.ts new file mode 100644 index 000000000..99d3e479b --- /dev/null +++ b/.github/scripts/create-failure-issue.ts @@ -0,0 +1,163 @@ +/** + * Creates a GitHub issue for an automated failure. Ex. Canary failure. + * + * Usage: + * npx tsx .github/scripts/create-failure-issue.ts \ + * --title-prefix "CI Failure" \ + * --name "Build and Test" \ + * --branch main \ + * --commit \ + * --run-url \ + * [--labels high-severity,ci] \ + * [--detail "Variant: Released/Preview"] + * + * Required env: + * GH_TOKEN (or GITHUB_TOKEN) — token with `issues: write` + * GITHUB_REPOSITORY — "owner/repo" (auto-set by GitHub Actions) + * + */ + +const GITHUB_API_BASE_URL = 'https://api.github.com'; + +interface CreateIssueArgs { + titlePrefix: string; + name: string; + branch: string; + commit: string; + runUrl: string; + labels: string[]; + detail?: string; +} + +function parseArgs(argv: string[]): CreateIssueArgs { + const map = new Map(); + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg.startsWith('--')) { + const key = arg.slice(2); + const value = argv[i + 1]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`Missing value for --${key}`); + } + map.set(key, value); + i++; + } + } + + const required = ['title-prefix', 'name', 'commit', 'run-url']; + const missing = required.filter(k => !map.get(k)); + if (missing.length > 0) { + throw new Error(`Missing required arguments: ${missing.map(k => `--${k}`).join(', ')}`); + } + + return { + titlePrefix: map.get('title-prefix')!, + name: map.get('name')!, + branch: map.get('branch') ?? 'main', + commit: map.get('commit')!, + runUrl: map.get('run-url')!, + labels: (map.get('labels') ?? 'high-severity,ci') + .split(',') + .map(l => l.trim()) + .filter(Boolean), + detail: map.get('detail'), + }; +} + +function getRepo(): { owner: string; repo: string } { + const repository = process.env.GITHUB_REPOSITORY; + if (!repository || !repository.includes('/')) { + throw new Error('GITHUB_REPOSITORY env var must be set to "owner/repo"'); + } + const [owner, repo] = repository.split('/'); + return { owner, repo }; +} + +function ghHeaders(token: string): Record { + return { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'agentcore-cli-failure-issue-script', + }; +} + +async function ghFetch(url: string, token: string, init?: RequestInit): Promise { + const res = await fetch(url, { + ...init, + headers: { ...ghHeaders(token), ...(init?.headers ?? {}) }, + }); + if (!res.ok) { + const text = await res.text(); + throw new Error(`GitHub API ${init?.method ?? 'GET'} ${url} failed: ${res.status} ${res.statusText} — ${text}`); + } + return res.json(); +} + +interface IssueItem { + title: string; +} + +async function issueExists(owner: string, repo: string, title: string, token: string): Promise { + // Primary: search index (fast, but can be stale). + const q = encodeURIComponent(`repo:${owner}/${repo} is:issue is:open in:title "${title}"`); + const search = (await ghFetch(`${GITHUB_API_BASE_URL}/search/issues?q=${q}`, token)) as { items?: IssueItem[] }; + if ((search.items ?? []).some(i => i.title === title)) { + return true; + } + + // Fallback: scan recent open issues in case the search index is stale. + const recent = (await ghFetch( + `${GITHUB_API_BASE_URL}/repos/${owner}/${repo}/issues?state=open&sort=created&direction=desc&per_page=30`, + token + )) as IssueItem[]; + return recent.some(i => i.title === title); +} + +async function main(): Promise { + const token = process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN; + if (!token) { + throw new Error('GH_TOKEN (or GITHUB_TOKEN) env var is required'); + } + + const args = parseArgs(process.argv.slice(2)); + console.log(`args: ${JSON.stringify(args)}`); + const { owner, repo } = getRepo(); + + const titleField = `${args.titlePrefix}: ${args.name}`; + + if (await issueExists(owner, repo, titleField, token)) { + console.log(`Issue already exists for "${titleField}" — skipping creation.`); + return; + } + + const bodyLines = [ + `## ${args.titlePrefix}`, + '', + `- **Name:** ${args.name}`, + `- **Branch:** ${args.branch}`, + `- **Commit:** ${args.commit}`, + `- **Run:** ${args.runUrl}`, + ]; + if (args.detail) { + bodyLines.push(`- **Detail:** ${args.detail}`); + } + bodyLines.push('', 'Please investigate this failure.'); + + await ghFetch(`${GITHUB_API_BASE_URL}/repos/${owner}/${repo}/issues`, token, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + title: titleField, + labels: args.labels, + body: bodyLines.join('\n'), + }), + }); + + console.log(`Created issue "${titleField}".`); +} + +main().catch((error: unknown) => { + console.error(`Failed to create failure issue: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); +}); diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml new file mode 100644 index 000000000..0e3ec303e --- /dev/null +++ b/.github/workflows/canary.yml @@ -0,0 +1,116 @@ +name: Canary + +on: + # Run once an hour + schedule: + - cron: '0 * * * *' + workflow_dispatch: + inputs: + aws_region: + description: 'AWS region for deployment' + default: 'us-east-1' + +# Serialize canary runs so overlapping scheduled runs don't pile up. Matrix +# jobs within a single run still execute in parallel. +concurrency: + group: canary + cancel-in-progress: false + +permissions: + id-token: write + contents: read + issues: write + +env: + AGENTCORE_TELEMETRY_DISABLED: '1' + PRERELEASE_BASE_URL: https://github.com/aws/agentcore-cli/releases/download/prerelease + +jobs: + canary: + runs-on: ubuntu-latest + environment: e2e-testing + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + variant: [Released, Prerelease] + build: [GA, Preview] + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: '20.x' + cache: 'npm' + - name: Configure git + run: | + git config --global user.email "ci@amazon.com" + git config --global user.name "CI" + - uses: astral-sh/setup-uv@v7 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ secrets.E2E_AWS_ROLE_ARN }} + aws-region: ${{ inputs.aws_region || 'us-east-1' }} + - name: Get AWS Account ID + id: aws + run: echo "account_id=$(aws sts get-caller-identity --query Account --output text)" >> "$GITHUB_OUTPUT" + - name: Get API keys from Secrets Manager + uses: aws-actions/aws-secretsmanager-get-secrets@v2 + with: + secret-ids: | + E2E,${{ secrets.E2E_SECRET_ARN }} + parse-json-secrets: true + + - run: npm ci + + # Resolve the install spec for this matrix cell, then install the + # published CLI globally (it already bundles the CDK constructs). + - name: Install agentcore CLI (${{ matrix.variant }} / ${{ matrix.build }}) + id: install + run: | + case "${{ matrix.variant }}/${{ matrix.build }}" in + "Released/GA") SPEC="@aws/agentcore" ;; + "Released/Preview") SPEC="@aws/agentcore@preview" ;; + "Prerelease/GA") SPEC="${PRERELEASE_BASE_URL}/agentcore-cli-prerelease.tgz" ;; + "Prerelease/Preview") SPEC="${PRERELEASE_BASE_URL}/agentcore-cli-prerelease-preview.tgz" ;; + *) echo "Unknown variant/build combination" >&2; exit 1 ;; + esac + echo "Installing: $SPEC" + echo "spec=$SPEC" >> "$GITHUB_OUTPUT" + npm install -g "$SPEC" + echo "agentcore version: $(agentcore --version || echo unknown)" + + - name: Run canary smoke test + env: + AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }} + AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }} + ANTHROPIC_API_KEY: ${{ env.E2E_ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} + # Smoke test for now, only runs strands-bedrock.test.ts + run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts --retry 3 + + - name: Generate GitHub App Token + if: failure() + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + - name: Create canary failure issue + if: failure() + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + VARIANT_NAME: ${{ matrix.variant }}/${{ matrix.build }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + INSTALL_SPEC: ${{ steps.install.outputs.spec }} + run: | + npx -y tsx .github/scripts/create-failure-issue.ts \ + --title-prefix "Canary Failure" \ + --name "$VARIANT_NAME" \ + --branch "${{ github.ref_name }}" \ + --commit "${{ github.sha }}" \ + --run-url "$RUN_URL" \ + --labels "high-severity,ci" \ + --detail "strands-bedrock smoke test; installed from ${INSTALL_SPEC:-unknown}" diff --git a/.github/workflows/ci-failure-issue.yml b/.github/workflows/ci-failure-issue.yml index 0cbb430a1..443fc61a5 100644 --- a/.github/workflows/ci-failure-issue.yml +++ b/.github/workflows/ci-failure-issue.yml @@ -19,6 +19,11 @@ jobs: permissions: issues: write steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: '20.x' + - name: Generate GitHub App Token id: app-token uses: actions/create-github-app-token@v1 @@ -26,54 +31,18 @@ jobs: app-id: ${{ vars.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} - - uses: actions/github-script@v9 - with: - github-token: ${{ steps.app-token.outputs.token }} - script: | - try { - const workflowName = context.payload.workflow_run.name; - - if (!/^[A-Za-z0-9 _()\-]+$/.test(workflowName)) { - core.setFailed(`Unexpected characters in workflow name: ${workflowName}`); - return; - } - - const sha = context.payload.workflow_run.head_sha; - const runUrl = context.payload.workflow_run.html_url; - const branch = context.payload.workflow_run.head_branch; - const title = `CI Failure: ${workflowName}`; - - const { data } = await github.rest.search.issuesAndPullRequests({ - q: `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"` - }); - - if (data.items.some(i => i.title === title)) { - core.info(`Issue already exists for "${title}"`); - return; - } - - // Fallback: check recent issues in case search index is stale - const { data: recent } = await github.rest.issues.listForRepo({ - owner: context.repo.owner, - repo: context.repo.repo, - state: 'open', - sort: 'created', - direction: 'desc', - per_page: 30 - }); - - if (recent.some(i => i.title === title)) { - core.info(`Issue already exists for "${title}" (found via fallback)`); - return; - } - - await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title, - labels: ['high-severity', 'ci'], - body: `## CI Failure\n\n- **Workflow:** ${workflowName}\n- **Branch:** ${branch}\n- **Commit:** ${sha}\n- **Run:** ${runUrl}\n\nPlease investigate this failure.` - }); - } catch (error) { - core.setFailed(`Failed to create CI failure issue: ${error.message || error}`); - } + - name: Create CI failure issue + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + WORKFLOW_NAME: ${{ github.event.workflow_run.name }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + run: | + npx -y tsx .github/scripts/create-failure-issue.ts \ + --title-prefix "CI Failure" \ + --name "$WORKFLOW_NAME" \ + --branch "$HEAD_BRANCH" \ + --commit "$HEAD_SHA" \ + --run-url "$RUN_URL" \ + --labels "high-severity,ci" diff --git a/.github/workflows/e2e-tests-full.yml b/.github/workflows/e2e-tests-full.yml index 3e3143e91..3490e871e 100644 --- a/.github/workflows/e2e-tests-full.yml +++ b/.github/workflows/e2e-tests-full.yml @@ -5,8 +5,6 @@ on: aws_region: description: 'AWS region for deployment' default: 'us-east-1' - schedule: - - cron: '0 14 * * 1' # Every Monday at 9 AM EST (14:00 UTC) push: branches: [main] diff --git a/e2e-tests/e2e-helper.ts b/e2e-tests/e2e-helper.ts index f5fbce546..68bdd2b28 100644 --- a/e2e-tests/e2e-helper.ts +++ b/e2e-tests/e2e-helper.ts @@ -76,7 +76,7 @@ export function createE2ESuite(cfg: E2EConfig) { testDir = join(tmpdir(), `agentcore-e2e-${randomUUID()}`); await mkdir(testDir, { recursive: true }); - agentName = `E2e${cfg.framework.slice(0, 4)}${cfg.modelProvider.slice(0, 4)}${String(Date.now()).slice(-8)}`; + agentName = `E2e${cfg.framework.slice(0, 4)}${cfg.modelProvider.slice(0, 4)}${randomUUID().replace(/-/g, '').slice(0, 8)}`; const createArgs = [ 'create', '--name', diff --git a/eslint.config.mjs b/eslint.config.mjs index 5111978cb..f689048b5 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -164,6 +164,7 @@ export default tseslint.config( ignores: [ 'dist', 'node_modules', + '.github', 'src/assets', 'src/schema/llm-compacted', '.agentcore', diff --git a/tsconfig.json b/tsconfig.json index eedb71730..0cf16f25b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -33,5 +33,5 @@ "vitest.integ.config.ts", "vitest.unit.config.ts" ], - "exclude": ["node_modules", "dist", "packages", "assets", "src/assets"] + "exclude": ["node_modules", "dist", "packages", "assets", "src/assets", ".github"] }