Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 85 additions & 10 deletions .github/workflows/oc-connectivity.yml
Original file line number Diff line number Diff line change
@@ -1,33 +1,108 @@
name: E2E Test

# Per-PR smoke test: build four images locally on the runner, stand up the
# aime/zerostack/gpt-5.4 compose stack, assert result.json + agent duration
# + gen_ai spans. Runs on the ubuntu-latest docker daemon — no OC cluster.

on:
pull_request:
branches: [main]
workflow_dispatch:

permissions:
contents: read
packages: read # pull GHCR-hosted base images without rate-limiting

defaults:
run:
shell: bash

jobs:
e2e:
name: E2E pipeline test (aime / codex / bifrost)
# Same-repo only: fork PRs receive empty secrets (oc login fails noisily) and
# this job runs untrusted PR code (deploy/oc/test.sh). Manual dispatch allowed.
name: E2E pipeline test (aime / zerostack / gpt-5.4)
# Same-repo only: fork PRs receive empty secrets and this job builds images
# from PR code. Manual dispatch allowed.
if: >-
github.event_name == 'workflow_dispatch' ||
github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
timeout-minutes: 30
env:
# EVAL_GATEWAY_IMAGE and EVAL_GATEWAY_LABEL are not CLI flags — they are
# compose interpolations that the gateway/runner services read directly.
# Set here so they are inherited by every step (cargo → docker compose).
EVAL_GATEWAY_IMAGE: gpt-5.4
EVAL_GATEWAY_LABEL: gpt-5.4
steps:
- uses: actions/checkout@v6

- name: Install oc CLI
uses: redhat-actions/openshift-tools-installer@144527c7d98999f2652264c048c7a9bd103f8a82 # v1
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable

- name: Cache cargo
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2

# NB: deliberately NO docker/setup-buildx-action — its docker-container
# builder can't see locally-baked bases, so eval images' FROM
# ghcr.io/exgentic/core/<base> would miss. Use the runner's default driver.
- name: Log in to GHCR
uses: docker/login-action@v4
with:
oc: latest
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Build images
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: |
cargo run -- build bench aime
cargo run -- build agent zerostack
cargo run -- build model gpt-5.4
cargo run -- build eval aime --agent zerostack

- name: Run e2e and assert
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
run: |
OUTPUT_DIR=$(mktemp -d)

# Pre-create the compose named volume (aime_output) as a host bind-mount
# so result files are readable after compose exits. Compose reuses an
# existing volume rather than creating a new one.
docker volume create --driver local \
--opt type=none --opt device="$OUTPUT_DIR" --opt o=bind \
aime_output

cargo run -- run aime \
--agent zerostack \
--model gpt-5.4 \
--task-id 0 \
--local \
--mode compose

echo "=== assertions ==="

TASK_JSON=$(cat "$OUTPUT_DIR/task/result.json")
[[ -n "$TASK_JSON" ]] || { echo "FAIL: task/result.json missing"; exit 1; }
echo "PASS: result written: $TASK_JSON"

AGENT_JSON=$(cat "$OUTPUT_DIR/agent/result.json")
A_START=$(echo "$AGENT_JSON" | sed -n 's/.*"started_at":"\([^"]*\)".*/\1/p')
A_END=$(echo "$AGENT_JSON" | sed -n 's/.*"ended_at":"\([^"]*\)".*/\1/p')
if [[ -n "$A_END" && "$A_END" != "$A_START" ]]; then
echo "PASS: agent ran ($A_START → $A_END)"
else
echo "FAIL: agent did not run (0-duration): ${AGENT_JSON:-<missing>}"
exit 1
fi

- name: Login to OpenShift
run: oc login "${{ secrets.OC_SERVER }}" --token="${{ secrets.OC_TOKEN }}" --insecure-skip-tls-verify
TRACES=$(cat "$OUTPUT_DIR/traces.jsonl" 2>/dev/null || true)
if echo "$TRACES" | grep -q '"gen_ai'; then
echo "PASS: OTel gen_ai spans present"
else
echo "WARN: no gen_ai spans (LLM call still confirmed by clean agent exit)"
fi

- name: Run e2e test
run: bash deploy/oc/test.sh --benchmark aime --agent codex --model bifrost --task 0
echo "=== ALL CHECKS PASSED ==="
Loading