diff --git a/.env.example b/.env.example index 34a183c..cdf34fe 100644 --- a/.env.example +++ b/.env.example @@ -13,7 +13,7 @@ INFERENCE_BACKEND=fireworks # (see docs/amd_proof/ — serve it on the AMD MI300X notebook first) MODEL_ID=accounts/your-account/deployments/your-deployment-id -# Fireworks (AMD-hosted) — used when INFERENCE_BACKEND=fireworks +# Fireworks — used when INFERENCE_BACKEND=fireworks FIREWORKS_API_KEY= FIREWORKS_BASE_URL=https://api.fireworks.ai/inference/v1 diff --git a/AGENTS.md b/AGENTS.md index cbcb21f..9ecd6f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,7 +69,7 @@ open-weight family, **self-hosted on AMD** so eval data never leaves your hardwa judge isn't a metered closed API. Do NOT claim "nobody does adversarial generation" — Promptfoo already does; our angle is the self-hosted single-family closed loop on AMD. -- **P0 failure mode: hallucination only.** Jailbreak = P1, bias = P2. Do not add P0 scope. +- **Failure modes supported:** hallucination, jailbreak, and bias are all already implemented. - **Demo target (system-under-test):** a deliberately weak model (e.g. Gemma 3 4B) so failures are dramatic. Must be verified on Day 1 that it actually hallucinates on our seeded attacks (the "Day-1 spike"). @@ -101,7 +101,7 @@ and tested — reuse, don't redefine. 1. **Response time < 30s per request** (all-tracks rule). → async fan-out; never run a huge batch on the live path. Instrument wall-clock per run. Batch of ~20 is the demo size. 2. **AMD compute usage is REQUIRED or the project is disqualified.** Gemma must run on AMD: - self-hosted on MI300X (vLLM+ROCm) and/or Gemma on Fireworks' AMD-hosted infra. Proof + self-hosted on MI300X (vLLM+ROCm). Proof (config + logs + screenshot) must be committed under `docs/amd_proof/` by end of Day 3. 3. **No secrets in git.** `.env` is gitignored; read all keys from env. We supply our OWN Fireworks keys — the `FIREWORKS_*`/`ALLOWED_MODELS` harness injection is a **Track 1** @@ -113,7 +113,7 @@ and tested — reuse, don't redefine. ## 6. Inference backends (env-selected) `INFERENCE_BACKEND` picks where Attacker+Judge Gemma runs: -- `fireworks` → Gemma on Fireworks (AMD-hosted). Powers the **live URL** (managed uptime). +- `fireworks` → Gemma on Fireworks. Powers the **live URL** (managed uptime). Exact available Gemma model IDs must be confirmed from Fireworks (launch day was 6 Jul — confirm and set `MODEL_ID`). - `mi300x` → Gemma self-hosted on AMD Dev Cloud MI300X via vLLM+ROCm. Powers the @@ -156,7 +156,7 @@ Do these in order. Each should be small, typed, and testable. it; it imports **only** `run_eval` + schemas. - Tests: 68 passing, `ruff` clean, CI green (`pythonpath=["."]` so bare `pytest` resolves). -**P1 (only if ahead):** jailbreak mode (`prompts/*_jailbreak.md` + AdvBench seed), full +**Future plans:** full self-consistency (every case ×3), leaderboard data, SQLite run history, PDF export data. ## 8. Stack & conventions diff --git a/Dockerfile b/Dockerfile index de6c7c4..a59386d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # Not required for Track 3, but useful for a reproducible one-command run. # # Builds the Streamlit demo (app.py). The engine talks to Gemma over an -# OpenAI-compatible endpoint - Fireworks (AMD-hosted) or self-hosted vLLM on AMD + +# OpenAI-compatible endpoint - Fireworks or self-hosted vLLM on AMD + # ROCm - configured entirely via environment variables (see .env.example). With no # engine configured, the app still renders the committed real runs and the clearly # labelled simulated demo, so the container is always demonstrable. diff --git a/README.md b/README.md index acfea7f..6af3aa8 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,7 @@ GemmaJudge uses one open-weight model family (Gemma) in two adversarial roles — an **Attacker** that generates targeted adversarial test cases for a chosen failure mode, and a **Judge** that scores a target model's responses against a rubric — running the -whole closed loop on AMD (self-hosted on AMD GPUs via vLLM + ROCm, or Gemma on Fireworks' -AMD-hosted infrastructure). +whole closed loop on AMD (self-hosted on AMD GPUs via vLLM + ROCm, or Gemma on Fireworks). Built for the **AMD Developer Hackathon: ACT II — Track 3 (Unicorn)**. @@ -112,7 +111,7 @@ All configuration is via environment variables — nothing is hardcoded. See [`.env.example`](.env.example) for the full list. Two inference backends, selected by `INFERENCE_BACKEND`: -- `fireworks` — Gemma on Fireworks' AMD-hosted infra (powers the live demo URL). +- `fireworks` — Gemma on Fireworks (powers the live demo URL). - `mi300x` — Gemma self-hosted on an AMD GPU (AMD Developer Cloud) via vLLM + ROCm. The committed proof used an AMD Radeon PRO W7900 (gfx1100); the same backend serves an AMD Instinct MI300X unchanged. diff --git a/app.py b/app.py index e624fbd..75d0e59 100644 --- a/app.py +++ b/app.py @@ -726,7 +726,7 @@ def _render_amd_proof(settings: Any | None, config_error: str | None) -> None: st.dataframe(pd.DataFrame(rows), use_container_width=True, hide_index=True) st.caption( "Pending files are shown honestly. The live demo can still run in simulated mode, " - "but only real Fireworks/MI300X runs should be presented as Gemma/AMD evidence." + "but only real MI300X/W7900 runs should be presented as Gemma/AMD evidence." ) diff --git a/deck/build_deck.py b/deck/build_deck.py index 0e90e35..878185c 100644 --- a/deck/build_deck.py +++ b/deck/build_deck.py @@ -188,7 +188,7 @@ def slide_solution(c): # AMD band under attacker+judge c.setFillColor(AMD) c.setFont("Helvetica-Bold", 9) - c.drawString(x0, y - 0.28 * inch, "▲ both Gemma roles run on AMD (self-hosted via vLLM + ROCm, or Fireworks' AMD-hosted infra)") + c.drawString(x0, y - 0.28 * inch, "▲ both Gemma roles run on AMD (self-hosted via vLLM + ROCm)") _bullets(c, [ ("Config → generate N adversarial prompts → run the target → judge each with " "written reasoning → report. Fully automated, no human in the loop.", INK, False), @@ -257,7 +257,7 @@ def slide_amd(c): ("Proof-of-compute on AMD Radeon PRO W7900 (gfx1100 / ROCm 7.2): Gemma served via " "vLLM — a real run with committed rocm-smi + serve command + vLLM logs in the repo " "(hallucination ASR 80%, judge self-consistency stdev 0.00).", INK, False), - ("Live URL backed by Fireworks (AMD-hosted) for uptime; self-hosted AMD (Radeon W7900, " + ("Live URL backed by Fireworks for uptime; self-hosted AMD (Radeon W7900, " "ROCm) for the screenshottable hardware proof.", INK, False), ("On-screen: model id + inference backend on every run, plus a cost/throughput " "panel driven by actual measured token usage.", INK, False), diff --git a/gemmajudge/config.py b/gemmajudge/config.py index 27c9dfd..2702bad 100644 --- a/gemmajudge/config.py +++ b/gemmajudge/config.py @@ -47,7 +47,7 @@ class ConfigError(RuntimeError): class InferenceBackend(StrEnum): """Where the Attacker + Judge Gemma runs (env ``INFERENCE_BACKEND``).""" - FIREWORKS = "fireworks" # Gemma on Fireworks' AMD-hosted infra → live URL + FIREWORKS = "fireworks" # Gemma on Fireworks → live URL MI300X = "mi300x" # Gemma self-hosted on AMD Dev Cloud MI300X → AMD proof diff --git a/gemmajudge/orchestrator.py b/gemmajudge/orchestrator.py index 8b693e3..2288f51 100644 --- a/gemmajudge/orchestrator.py +++ b/gemmajudge/orchestrator.py @@ -214,7 +214,7 @@ def _build_cost( """Turn measured per-role usage into a CostReport with a $ figure. The $ figure prices the **engine** (Attacker+Judge) tokens, which is what runs - on the AMD-hosted Gemma; the target is a separate system whose price we don't + on the Gemma engine; the target is a separate system whose price we don't assume. If no pricing is configured, ``usd`` is a truthful ``0.0``.""" pricing = settings.pricing if settings else None if pricing is not None: