From 953cb30275ed41922866797fd4d7ac8915851277 Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Tue, 4 Aug 2026 19:21:39 +0300 Subject: [PATCH 1/3] fix(bifrost): price calls so gen_ai.usage.cost is a real number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bifrost prices every call from its bundled pricing.json, keyed on the model name. Ours never matched: the handles we route are the upstream proxy's own (`azure/gpt-5.5`, `aws/claude-opus-5`, `rits/google/gemma-4-31B`), not litellm's public catalog names. The lookup missed and bifrost wrote gen_ai.usage.cost=0 — on every one of the ~2,100 provider-call spans in the results store, which is why the dashboard's cost column was blank. The upstream does know the cost; it returns `x-litellm-response-cost` on every response. But that is an HTTP header, and bifrost's span carries the cost it computed itself, not one read off the wire — and its otel plugin has no header-capture knob. So rather than teach it to forward a header, give it the rate: a global wildcard `governance.pricing_overrides` entry makes its own arithmetic come out right. One flat rate covers every model because the upstream bills one. Probing it with gpt-5.4 / gpt-5.5 / gpt-4.1 / gpt-5.6-sol / claude-opus-5 / claude-sonnet-5 / claude-haiku-4-5 / gemini-3-flash / gemma-4-31B returns the same cost for the same token counts. Solving two samples gives $2.50/1M in and $15.00/1M out, and that predicts `x-litellm-response-cost` to within 1e-9 on every model tried. Overridable via EVAL_COST_INPUT_PER_TOKEN / EVAL_COST_OUTPUT_PER_TOKEN for when the contract changes. Verified end to end against the real upstream: span cost now equals the header exactly (0.000265, 0.0003325, 0.00093 over three calls). The dashboard needs no change — it already reads `gen_ai.usage.cost`. This fixes new runs only; existing traces keep their zeros (their token counts are intact, so history could be back-computed at the same rates if we want it). Tests, per gateways/RULES.md rule 6 (OTel emission is covered per flavor): * otel_bifrost_span_cost_is_nonzero asserts the attribute is populated. Not a cost SLO (rule 12 forbids those) — no threshold, just "not zero". * EVAL_MODEL in the fixtures was `openai/azure/gpt-5.4`, which violates the bifrost/litellm env contract (a bare handle; the wire comes from the inbound path). The prefixed string went upstream verbatim and 403'd, so all eight upstream_{bifrost,litellm}_* tests and both bifrost OTel tests were failing before this change. portkey's contract is the opposite — it requires / and strips the first segment — so the default is now per-flavor via eval_model(). The full `--ignored` suite passes (15/15); it did not before. * start_pod_with_otel named its collector `otelcol-`, so two OTel tests entering on the same clock tick collided on the container name. Added a sequence counter. Signed-off-by: Elron Bandel --- containers/gateways/bifrost/start | 34 ++++++- tests/run/gateways/test.rs | 144 ++++++++++++++++++++++++++++-- 2 files changed, 168 insertions(+), 10 deletions(-) diff --git a/containers/gateways/bifrost/start b/containers/gateways/bifrost/start index 9650eb71..0b9bced1 100644 --- a/containers/gateways/bifrost/start +++ b/containers/gateways/bifrost/start @@ -13,6 +13,9 @@ # EVAL_MODEL_API OPTIONAL wire override: anthropic | openai | gemini. # OPENAI_API_BASE upstream root; each provider appends its native path. # LOG_LEVEL bifrost log level (default info). +# EVAL_COST_INPUT_PER_TOKEN, EVAL_COST_OUTPUT_PER_TOKEN +# what the upstream bills us per token (see the pricing +# override below). Defaults match the IBM litellm proxy. # Upstream auth is the provider-native env var (OPENAI_API_KEY etc.) — no # umbrella alias (gateways/RULES.md rule 3). @@ -45,7 +48,36 @@ if [ -f "$TEMPLATE" ]; then RULES="$RULES{ \"id\": \"pin-$p\", \"name\": \"pin-$p\", \"enabled\": true, \"scope\": \"global\", \"priority\": $pr, \"cel_expression\": \"headers['x-eval-wire'] == '$p'\", \"targets\": [{ \"provider\": \"$p\"${MODEL}, \"weight\": 1.0 }] }" done fi - GOVERNANCE="\"governance\": { \"routing_rules\": [${RULES}] }," + # PRICING: make gen_ai.usage.cost on the OTel span a real number. + # + # bifrost prices every call from its bundled pricing.json, keyed on the model + # name. Ours never matches: the handles we route are the upstream proxy's own + # (`azure/gpt-5.5`, `aws/claude-opus-5`, `rits/google/gemma-4-31B`), not + # litellm's public catalog names — so the lookup misses and bifrost writes + # gen_ai.usage.cost=0. Empirically that was 0 on every one of the ~2,100 + # provider-call spans in the results store, which is why the dashboard's cost + # column was blank. + # + # The upstream DOES know the cost — it returns `x-litellm-response-cost` on + # every response — but that is an HTTP header, and bifrost's span carries the + # cost it computed itself, not one read off the wire. Rather than teach it to + # forward a header, give it the rate: a global wildcard pricing override makes + # its own arithmetic come out right. + # + # One flat rate covers every model because the upstream bills one: probing it + # with gpt-5.4 / gpt-5.5 / gpt-4.1-mini / claude-opus-5 / claude-sonnet-5 / + # gemini-3-flash / gemma-4-31B returns the SAME cost for the same token counts. + # Solving two samples gives $2.50/1M in, $15.00/1M out, and that predicts + # `x-litellm-response-cost` exactly (to 1e-9) on every model tried. Overridable + # by env for when the contract changes. + : "${EVAL_COST_INPUT_PER_TOKEN:=0.0000025}" + : "${EVAL_COST_OUTPUT_PER_TOKEN:=0.000015}" + # `pricing_patch` is JSON *inside* a JSON string, and this whole thing is a sed + # replacement — so its inner quotes need a backslash that survives sed's own + # unescaping (sed reduces `\"` to `"`). Hence the four backslashes: shell → + # `\\"`, sed → `\"`, which is what the config file must contain. + PRICING="{ \"id\": \"00000000-0000-4000-8000-000000000001\", \"name\": \"upstream-flat-rate\", \"scope_kind\": \"global\", \"match_type\": \"wildcard\", \"pattern\": \"*\", \"request_types\": [\"chat_completion\"], \"pricing_patch\": \"{\\\\\"input_cost_per_token\\\\\":${EVAL_COST_INPUT_PER_TOKEN},\\\\\"output_cost_per_token\\\\\":${EVAL_COST_OUTPUT_PER_TOKEN}}\" }" + GOVERNANCE="\"governance\": { \"routing_rules\": [${RULES}], \"pricing_overrides\": [${PRICING}] }," : "${OTEL_EXPORTER_OTLP_ENDPOINT:=http://otelcol:4318}" OTEL_COLLECTOR_URL="${OTEL_EXPORTER_OTLP_ENDPOINT%/}/v1/traces" diff --git a/tests/run/gateways/test.rs b/tests/run/gateways/test.rs index 92ec612e..b978946a 100644 --- a/tests/run/gateways/test.rs +++ b/tests/run/gateways/test.rs @@ -46,6 +46,7 @@ //! — IBM litellm, OpenAI, Azure, vLLM, etc.). use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; use reqwest::Client; @@ -78,6 +79,24 @@ fn health_path(flavor: &str) -> &'static str { } } +/// The `EVAL_MODEL` a flavor needs to reach `azure/gpt-5.4` upstream. +/// The two contracts differ and no single string satisfies both: +/// +/// * bifrost + litellm want a BARE upstream handle — the wire is chosen +/// by the inbound path, not a prefix (gateways/*/start). Prefix it and +/// the whole string goes upstream verbatim, which 403s +/// ("team not allowed to access model"). +/// * portkey requires `/` and eats the first segment +/// (`MODEL_NAME=${EVAL_MODEL#*/}` in gateways/portkey/start), so it +/// needs the extra hop to forward the same handle. +fn eval_model(flavor: &str) -> &'static str { + match flavor { + "bifrost" | "litellm" => "azure/gpt-5.4", + "portkey" => "openai/azure/gpt-5.4", + _ => panic!("unknown flavor: {flavor}"), + } +} + fn dockerfile_text(flavor: &str) -> String { std::fs::read_to_string( test_support::repo_root().join(format!("containers/gateways/{flavor}/Dockerfile")), @@ -141,7 +160,7 @@ async fn start_gateway(flavor: &str, extra_env: &[(&str, &str)]) -> ContainerAsy .with_expected_status_code(200u16), ))) // Boot defaults. extra_env overrides any of these. - .with_env_var("EVAL_MODEL", "openai/azure/gpt-5.4") + .with_env_var("EVAL_MODEL", eval_model(flavor)) .with_env_var("OPENAI_API_KEY", "sk-bogus") .with_env_var("OPENAI_API_BASE", "http://127.0.0.1:9999/v1") .with_env_var("HOST", "0.0.0.0"); @@ -168,6 +187,11 @@ fn http() -> Client { fn body_openai() -> Value { json!({ + // Prefixed, unlike EVAL_MODEL. bifrost and litellm pin EVAL_MODEL and + // ignore the client's model entirely; portkey does not pin, and eats + // the leading segment as its provider prefix — so it needs the extra + // `openai/` here to forward `azure/gpt-5.4` to the upstream. Drop it + // and portkey sends a bare `gpt-5.4`, which the upstream 403s. "model": "openai/azure/gpt-5.4", "messages": [{"role": "user", "content": "Reply with exactly: OK"}] }) @@ -675,13 +699,19 @@ async fn start_pod_with_otel( ensure_built().await; let (key, base) = upstream_creds(); - // Unique network per pod so parallel test threads don't collide - // when multiple OTel tests run side by side. + // Unique network + container name per pod so parallel test threads + // don't collide when multiple OTel tests run side by side. A plain + // nanos timestamp is NOT unique enough: two tests entering here at + // once read the same clock tick and the second one fails to create + // `otelcol-` ("name is already in use"). The counter makes + // the suffix unique regardless of clock granularity. + static POD_SEQ: AtomicU64 = AtomicU64::new(0); let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos(); - let net = format!("gw-test-{flavor}-{nanos}"); + let uniq = format!("{nanos}-{}", POD_SEQ.fetch_add(1, Ordering::Relaxed)); + let net = format!("gw-test-{flavor}-{uniq}"); // Order matters: `GenericImage` methods (with_wait_for) MUST come // before any `ImageExt` method (with_platform, with_mount, ...) @@ -696,10 +726,10 @@ async fn start_pod_with_otel( "/output", )) .with_network(&net) - // container_name=otelcol-{nanos} so the gateway can reach it - // as OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol-:4318 + // container_name=otelcol-{uniq} so the gateway can reach it + // as OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol-:4318 // — the bridge-network DNS resolves the container name. - .with_container_name(format!("otelcol-{nanos}")) + .with_container_name(format!("otelcol-{uniq}")) .start() .await .expect("start otelcol"); @@ -723,7 +753,8 @@ async fn start_pod_with_otel( "/output", )) .with_network(&net) - .with_env_var("EVAL_MODEL", "openai/azure/gpt-5.4") + // Per-flavor — the two env contracts disagree; see eval_model(). + .with_env_var("EVAL_MODEL", eval_model(flavor)) .with_env_var("OPENAI_API_KEY", key) .with_env_var("OPENAI_API_BASE", base) .with_env_var("HOST", "0.0.0.0") @@ -732,7 +763,7 @@ async fn start_pod_with_otel( // litellm via the `otel` callback in config.yaml.template. .with_env_var( "OTEL_EXPORTER_OTLP_ENDPOINT", - format!("http://otelcol-{nanos}:4318"), + format!("http://otelcol-{uniq}:4318"), ) .start() .await @@ -806,6 +837,101 @@ async fn otel_bifrost_gen_ai_attrs() { assert_gen_ai_attrs("bifrost").await } +/// The largest `gen_ai.usage.cost` over every span in an OTLP-JSON +/// trace file, or None if no span carries the attribute at all. Both +/// file shapes the collector writes are handled: one export request per +/// line, or a single pretty-printed object. +fn max_span_cost(traces: &str) -> Option { + fn walk(obj: &Value, out: &mut Option) { + for rs in obj["resourceSpans"].as_array().into_iter().flatten() { + for ss in rs["scopeSpans"].as_array().into_iter().flatten() { + for sp in ss["spans"].as_array().into_iter().flatten() { + for a in sp["attributes"].as_array().into_iter().flatten() { + if a["key"] == "gen_ai.usage.cost" { + // OTLP JSON puts a double in doubleValue; a + // whole number may arrive as an intValue + // *string*, so accept both shapes. + let v = &a["value"]; + let n = v["doubleValue"] + .as_f64() + .or_else(|| v["intValue"].as_str().and_then(|s| s.parse().ok())); + if let Some(n) = n { + *out = Some(out.map_or(n, |m: f64| m.max(n))); + } + } + } + } + } + } + } + let mut out = None; + match serde_json::from_str::(traces) { + Ok(whole) => walk(&whole, &mut out), + Err(_) => { + for line in traces.lines().filter(|l| !l.trim().is_empty()) { + if let Ok(obj) = serde_json::from_str::(line) { + walk(&obj, &mut out); + } + } + } + } + out +} + +#[tokio::test] +#[ignore] +async fn otel_bifrost_span_cost_is_nonzero() { + // bifrost prices each call from its bundled pricing.json keyed on the + // model name, and the handles we route are the upstream proxy's own + // (`azure/gpt-5.4`, `aws/claude-opus-5`, …) — never in that catalog. + // So the lookup missed and every span carried gen_ai.usage.cost=0, + // which is what left the dashboard's cost column blank across the + // whole result history. gateways/bifrost/start now installs a + // governance pricing override with the rate the upstream actually + // bills; this pins that the span comes out with a real number on it. + // + // Asserting the attribute is *populated*, not that it is under some + // threshold — a cost SLO is forbidden (RULES.md rule 12). + let tmp = tempfile::tempdir().expect("tempdir"); + let (_otel, gw) = start_pod_with_otel("bifrost", tmp.path()).await; + let port = gateway_port(&gw).await; + let resp = http() + .post(format!( + "http://127.0.0.1:{port}/openai/v1/chat/completions" + )) + .json(&body_openai()) + .send() + .await + .expect("post chat"); + assert_eq!(resp.status(), 200, "precondition: gateway returned 200"); + + // The cost rides the same leaf provider-call span as the gen_ai.* + // attrs, so once one of those has landed the cost is there too — + // but keep polling to the same 30s budget in case the exporter + // flushed a parent span first. + let mut traces = await_traces_with_gen_ai(tmp.path()); + let path = tmp.path().join("traces.jsonl"); + for _ in 0..40 { + if max_span_cost(&traces).is_some_and(|c| c > 0.0) { + return; + } + std::thread::sleep(Duration::from_millis(500)); + traces = std::fs::read_to_string(&path).unwrap_or(traces); + } + panic!( + "bifrost span cost is {} — expected a nonzero gen_ai.usage.cost. \ + Either the governance pricing override in gateways/bifrost/start no \ + longer renders (check the sed escaping of pricing_patch), or bifrost \ + changed how it applies pricing_overrides. First 600 bytes of \ + traces.jsonl: {}", + match max_span_cost(&traces) { + None => "absent".to_string(), + Some(c) => format!("{c}"), + }, + &traces[..traces.len().min(600)] + ); +} + #[tokio::test] #[ignore] async fn otel_litellm_gen_ai_attrs() { From f2914555e663869bda56ca731d549291c0c31ec3 Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Wed, 5 Aug 2026 11:12:14 +0300 Subject: [PATCH 2/3] feat(bifrost): let a mounted file set per-model prices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flat upstream rate is right for our litellm proxy, which bills every model the same, but the gateway image ships standalone — anyone pointing it at a provider with real per-model pricing had no way to say so short of rebuilding. `EVAL_PRICING_FILE` (default /opt/gateway/data/pricing.tsv, absent from the image) is read at startup and rendered into one `exact` pricing_override per model, layered alongside the wildcard rather than replacing it — so an incomplete list falls back to EVAL_COST_* instead of pricing at 0, which was the original bug. A malformed line exits 2 with the line number (rule 22). Two traps are load-bearing and documented in the file format, the comments, and the assertions: * bifrost's `exact` match sees the BARE requested model (`azure/gpt-5.5`), not the provider-prefixed key it uses to index its own pricing.json (`openai/azure/gpt-5.5`). A prefixed pattern matches nothing and silently leaves the model on the fallback. * prices are per token, not per million. Whitespace-separated rather than JSON because the alpine image has neither jq nor python, and it must run under plain `docker run` (rule 20) — hence awk. pricing.tsv.example is copied in as documentation only; `start` never reads it, so no price is baked into the image (rule 1). Tests: a static check that the example parses the way `start` reads it (3 fields, per-token magnitude, no wire prefix), plus two runtime tests that a mounted file's rate wins for a listed model and that an unlisted one stays on the fallback. Signed-off-by: Elron Bandel --- containers/gateways/bifrost/Dockerfile | 5 + .../gateways/bifrost/pricing.tsv.example | 33 +++ containers/gateways/bifrost/start | 50 ++++- tests/run/gateways/test.rs | 200 +++++++++++++++++- 4 files changed, 282 insertions(+), 6 deletions(-) create mode 100644 containers/gateways/bifrost/pricing.tsv.example diff --git a/containers/gateways/bifrost/Dockerfile b/containers/gateways/bifrost/Dockerfile index eac16fb0..1dc00480 100644 --- a/containers/gateways/bifrost/Dockerfile +++ b/containers/gateways/bifrost/Dockerfile @@ -56,6 +56,11 @@ COPY --from=bifrost-bin /app/main /opt/gateway/main COPY start /opt/gateway/start COPY health /opt/gateway/health COPY Caddyfile /opt/gateway/Caddyfile +# Documentation + starting point for a per-model price list. Shipped as +# `.example` so it is inert: `start` reads pricing.tsv (EVAL_PRICING_FILE), +# never this. Copying it in bakes no price into the image (rule 1) — a user +# mounts their own edited copy over the real path. +COPY pricing.tsv.example /opt/gateway/data/pricing.tsv.example RUN chmod +x /opt/gateway/start /opt/gateway/health /opt/gateway/main \ && mkdir -p /opt/gateway/data diff --git a/containers/gateways/bifrost/pricing.tsv.example b/containers/gateways/bifrost/pricing.tsv.example new file mode 100644 index 00000000..faac307b --- /dev/null +++ b/containers/gateways/bifrost/pricing.tsv.example @@ -0,0 +1,33 @@ +# Per-model token prices for the bifrost gateway. Copy, edit, and point the +# gateway at it: +# +# docker run -e EVAL_MODEL=azure/gpt-5.5 \ +# -e EVAL_PRICING_FILE=/etc/eval/pricing.tsv \ +# -v ./pricing.tsv:/etc/eval/pricing.tsv:ro \ +# ... ghcr.io/exgentic/models/bifrost:latest +# +# Or mount it at the default path and skip the env var entirely: +# -v ./pricing.tsv:/opt/gateway/data/pricing.tsv:ro +# +# The gateway renders these into bifrost's `governance.pricing_overrides` at +# startup; the resulting cost lands on the OTel span as `gen_ai.usage.cost`, +# which is what the dashboard's cost column reads. +# +# Format: three whitespace-separated fields. `#` comments and blank lines are +# ignored. Prices are US dollars PER TOKEN, not per million — divide the usual +# per-1M figure by 1,000,000 ($2.50/1M -> 0.0000025). +# +# The model is the BARE handle, exactly as you'd pass it in EVAL_MODEL +# (`azure/gpt-5.5`, not `openai/azure/gpt-5.5`). +# +# Any model NOT listed here falls back to EVAL_COST_INPUT_PER_TOKEN / +# EVAL_COST_OUTPUT_PER_TOKEN, so a partial list is fine — nothing silently +# prices at zero. Listing no models at all (or mounting no file) leaves every +# model on that fallback. +# +# model input/token output/token +azure/gpt-5.4 0.0000025 0.0000150 +azure/gpt-5.5 0.0000050 0.0000300 +aws/claude-sonnet-5 0.0000030 0.0000150 +aws/claude-opus-5 0.0000150 0.0000750 +gcp/gemini-3-flash-preview 0.0000003 0.0000025 diff --git a/containers/gateways/bifrost/start b/containers/gateways/bifrost/start index 0b9bced1..3348a7b2 100644 --- a/containers/gateways/bifrost/start +++ b/containers/gateways/bifrost/start @@ -16,6 +16,19 @@ # EVAL_COST_INPUT_PER_TOKEN, EVAL_COST_OUTPUT_PER_TOKEN # what the upstream bills us per token (see the pricing # override below). Defaults match the IBM litellm proxy. +# EVAL_PRICING_FILE +# OPTIONAL price list, to charge different models different +# rates. Default /opt/gateway/data/pricing.tsv (absent in the +# image — mount one). Format, whitespace-separated: +# +# # model in/token out/token +# azure/gpt-5.5 0.0000050 0.0000300 +# aws/claude-opus-5 0.0000150 0.0000750 +# +# `#` comments and blank lines are ignored. The model is the +# BARE handle you'd put in EVAL_MODEL. Any model not listed +# still gets EVAL_COST_*_PER_TOKEN, so an incomplete file +# undercharges nothing — it just falls back. # Upstream auth is the provider-native env var (OPENAI_API_KEY etc.) — no # umbrella alias (gateways/RULES.md rule 3). @@ -77,7 +90,42 @@ if [ -f "$TEMPLATE" ]; then # unescaping (sed reduces `\"` to `"`). Hence the four backslashes: shell → # `\\"`, sed → `\"`, which is what the config file must contain. PRICING="{ \"id\": \"00000000-0000-4000-8000-000000000001\", \"name\": \"upstream-flat-rate\", \"scope_kind\": \"global\", \"match_type\": \"wildcard\", \"pattern\": \"*\", \"request_types\": [\"chat_completion\"], \"pricing_patch\": \"{\\\\\"input_cost_per_token\\\\\":${EVAL_COST_INPUT_PER_TOKEN},\\\\\"output_cost_per_token\\\\\":${EVAL_COST_OUTPUT_PER_TOKEN}}\" }" - GOVERNANCE="\"governance\": { \"routing_rules\": [${RULES}], \"pricing_overrides\": [${PRICING}] }," + # A price list, if one is mounted, becomes one `exact` override per model, + # layered on top of the wildcard above. Specificity decides which applies — + # a listed model gets its own rate, everything else falls back to the + # wildcard, so a partial list never prices anything at 0. (Order within the + # array is irrelevant; the wildcard does not shadow an exact entry.) + # + # `exact` matches the BARE requested model (`azure/gpt-5.5`), NOT the + # provider-prefixed key bifrost uses to index its own pricing.json + # (`openai/azure/gpt-5.5`). A prefixed pattern here matches nothing and the + # entry silently does nothing — hence the bare handle in the file format. + # + # awk, not jq/python: neither is in this alpine image (rule 20 — the image + # must run under plain `docker run`), so the file is whitespace-separated + # rather than JSON. Emitting one object per line and stripping newlines + # keeps the whole block a single sed replacement value. + : "${EVAL_PRICING_FILE:=/opt/gateway/data/pricing.tsv}" + if [ -f "$EVAL_PRICING_FILE" ]; then + CHART=$(awk ' + /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } + NF != 3 { + printf "pricing file line %d: expected 3 fields (model in out), got %d: %s\n", NR, NF, $0 > "/dev/stderr" + bad = 1; next + } + { + # Sequential uuid per entry; bifrost wants a distinct id on each. + n++ + printf ", { \"id\": \"00000000-0000-4000-8000-%012d\", \"name\": \"chart-%d\", \"scope_kind\": \"global\", \"match_type\": \"exact\", \"pattern\": \"%s\", \"request_types\": [\"chat_completion\"], \"pricing_patch\": \"{\\\\\"input_cost_per_token\\\\\":%s,\\\\\"output_cost_per_token\\\\\":%s}\" }", n + 1, n, $1, $2, $3 + } + END { if (bad) exit 2 } + ' "$EVAL_PRICING_FILE") || { + echo "EVAL_PRICING_FILE=$EVAL_PRICING_FILE is malformed (see above)" >&2; exit 2; } + echo "pricing: loaded $(grep -cvE '^[[:space:]]*(#|$)' "$EVAL_PRICING_FILE") model rate(s) from $EVAL_PRICING_FILE" >&2 + else + CHART="" + fi + GOVERNANCE="\"governance\": { \"routing_rules\": [${RULES}], \"pricing_overrides\": [${PRICING}${CHART}] }," : "${OTEL_EXPORTER_OTLP_ENDPOINT:=http://otelcol:4318}" OTEL_COLLECTOR_URL="${OTEL_EXPORTER_OTLP_ENDPOINT%/}/v1/traces" diff --git a/tests/run/gateways/test.rs b/tests/run/gateways/test.rs index b978946a..3208da0f 100644 --- a/tests/run/gateways/test.rs +++ b/tests/run/gateways/test.rs @@ -411,6 +411,65 @@ fn static_portkey_health_does_not_probe_removed_sidecar() { ); } +#[test] +fn static_bifrost_pricing_example_parses_as_the_start_script_reads_it() { + // The example file is the user-facing documentation of the price-list + // format, so it has to satisfy the same parse `start` applies: three + // whitespace-separated fields, `#`/blank ignored, model a BARE handle. + // A prefixed pattern is the trap worth guarding — bifrost's `exact` + // pricing override matches the bare requested model, so `openai/azure/…` + // matches nothing and the entry silently prices at 0. + let txt = std::fs::read_to_string( + test_support::repo_root().join("containers/gateways/bifrost/pricing.tsv.example"), + ) + .expect("read gateways/bifrost/pricing.tsv.example"); + let mut rows = 0; + for (i, line) in txt.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let f: Vec<&str> = line.split_whitespace().collect(); + assert_eq!( + f.len(), + 3, + "pricing.tsv.example line {}: expected 3 fields (model in out), got {}: {line}", + i + 1, + f.len() + ); + for price in &f[1..] { + let p: f64 = price + .parse() + .unwrap_or_else(|e| panic!("pricing.tsv.example line {}: {price} — {e}", i + 1)); + assert!( + p > 0.0, + "pricing.tsv.example line {}: price must be > 0, got {p}", + i + 1 + ); + // Per TOKEN, not per million. Anything at/above a cent per token + // is a per-1M figure someone forgot to divide. + assert!( + p < 0.01, + "pricing.tsv.example line {}: {p} looks like a per-1M price — \ + the file is per TOKEN (divide by 1e6)", + i + 1 + ); + } + assert!( + !f[0].starts_with("openai/") + && !f[0].starts_with("anthropic/") + && !f[0].starts_with("gemini/"), + "pricing.tsv.example line {}: `{}` is wire-prefixed. bifrost's `exact` \ + override matches the BARE handle, so a prefixed pattern never matches \ + and the model silently prices at 0.", + i + 1, + f[0] + ); + rows += 1; + } + assert!(rows > 0, "pricing.tsv.example has no price rows"); +} + // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ // Boot (runtime, no creds) — the gateway listens on :4000 after start. // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ @@ -695,6 +754,17 @@ const GEN_AI_REQUIRED_ATTRS: &[&str] = &[ async fn start_pod_with_otel( flavor: &str, host_output: &Path, +) -> (ContainerAsync, ContainerAsync) { + start_pod_with_otel_env(flavor, host_output, &[], None).await +} + +/// `start_pod_with_otel` plus extra gateway env and an optional host file +/// bind-mounted at `/etc/eval/pricing.tsv` (for the price-list tests). +async fn start_pod_with_otel_env( + flavor: &str, + host_output: &Path, + extra_env: &[(&str, &str)], + pricing_file: Option<&Path>, ) -> (ContainerAsync, ContainerAsync) { ensure_built().await; let (key, base) = upstream_creds(); @@ -735,7 +805,7 @@ async fn start_pod_with_otel( .expect("start otelcol"); let (name, tag) = gateway_image_ref(flavor); - let gw = GenericImage::new(name, tag) + let mut gw = GenericImage::new(name, tag) .with_exposed_port(ContainerPort::Tcp(4000)) .with_wait_for(WaitFor::Http(Box::new( HttpWaitStrategy::new(health_path(flavor)) @@ -764,10 +834,23 @@ async fn start_pod_with_otel( .with_env_var( "OTEL_EXPORTER_OTLP_ENDPOINT", format!("http://otelcol-{uniq}:4318"), - ) - .start() - .await - .expect("start gateway"); + ); + + // A price list, when the test supplies one: bind the host file and point + // the gateway at it, exactly as a user would with `docker run -v`. + if let Some(path) = pricing_file { + gw = gw + .with_mount(Mount::bind_mount( + path.to_str().expect("utf8 pricing path"), + "/etc/eval/pricing.tsv", + )) + .with_env_var("EVAL_PRICING_FILE", "/etc/eval/pricing.tsv"); + } + // Last, so a test can override any default set above. + for (k, v) in extra_env { + gw = gw.with_env_var(*k, *v); + } + let gw = gw.start().await.expect("start gateway"); (otel, gw) } @@ -932,6 +1015,113 @@ async fn otel_bifrost_span_cost_is_nonzero() { ); } +/// Write a price list to its own dir and return (dir, file path). The dir is +/// returned so the caller keeps it alive — dropping it deletes the file. +fn pricing_file(body: &str) -> (tempfile::TempDir, std::path::PathBuf) { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("pricing.tsv"); + std::fs::write(&path, body).expect("write pricing.tsv"); + (dir, path) +} + +/// Post one chat completion and return the largest span cost the collector saw. +async fn span_cost_of_one_call(gw: &ContainerAsync, output: &Path) -> f64 { + let port = gateway_port(gw).await; + let resp = http() + .post(format!( + "http://127.0.0.1:{port}/openai/v1/chat/completions" + )) + .json(&body_openai()) + .send() + .await + .expect("post chat"); + assert_eq!(resp.status(), 200, "precondition: gateway returned 200"); + + let mut traces = await_traces_with_gen_ai(output); + let path = output.join("traces.jsonl"); + for _ in 0..40 { + if let Some(c) = max_span_cost(&traces).filter(|c| *c > 0.0) { + return c; + } + std::thread::sleep(Duration::from_millis(500)); + traces = std::fs::read_to_string(&path).unwrap_or(traces); + } + panic!( + "no nonzero gen_ai.usage.cost span within 20s. First 600 bytes of \ + traces.jsonl: {}", + &traces[..traces.len().min(600)] + ); +} + +#[tokio::test] +#[ignore] +async fn otel_bifrost_prices_a_listed_model_from_the_mounted_price_file() { + // A mounted EVAL_PRICING_FILE is how a standalone `docker run` charges + // different models different rates. The listed rate must WIN over the + // flat EVAL_COST_* fallback — the two rates here differ by six orders of + // magnitude so the resulting cost says unambiguously which one applied. + // + // The file's model must be the BARE handle: bifrost's `exact` override + // matches the requested model, not the `openai/…`-prefixed key it uses + // to index its own pricing.json. A prefixed pattern matches nothing and + // silently leaves the model on the fallback, which is the whole reason + // this test asserts on the value rather than on the config rendering. + let (_pdir, prices) = pricing_file(&format!( + "# model in out\n\ + aws/claude-opus-5 0.0000150 0.0000750\n\ + {} 0.0010000 0.0020000\n", + eval_model("bifrost") + )); + let tmp = tempfile::tempdir().expect("tempdir"); + let (_otel, gw) = start_pod_with_otel_env( + "bifrost", + tmp.path(), + // Fallback set far below the file's rate: if the override didn't + // match, the cost lands near 1e-6 instead of near 1e-2. + &[ + ("EVAL_COST_INPUT_PER_TOKEN", "0.000000001"), + ("EVAL_COST_OUTPUT_PER_TOKEN", "0.000000001"), + ], + Some(&prices), + ) + .await; + + let cost = span_cost_of_one_call(&gw, tmp.path()).await; + assert!( + cost > 0.001, + "cost {cost} is the flat EVAL_COST_* fallback, not the mounted \ + price-file rate — the `exact` pricing override for `{}` did not \ + match. Check the pattern is the bare handle and that the awk block \ + in gateways/bifrost/start still renders valid JSON.", + eval_model("bifrost") + ); +} + +#[tokio::test] +#[ignore] +async fn otel_bifrost_falls_back_when_the_price_file_omits_the_model() { + // A partial price list must never price a model at 0 — the unlisted model + // stays on the flat EVAL_COST_* wildcard. This is what makes an + // incomplete file safe to mount, and it only holds because the wildcard + // override is layered alongside the per-model ones rather than replaced + // by them. + let (_pdir, prices) = pricing_file("some/other-model 0.0010000 0.0020000\n"); + let tmp = tempfile::tempdir().expect("tempdir"); + let (_otel, gw) = start_pod_with_otel_env("bifrost", tmp.path(), &[], Some(&prices)).await; + + // Nonzero is enforced by the helper (it panics if no span carries a + // cost). What this pins is *which* rate: the file's 0.001/token would put + // the cost three orders of magnitude above the flat default, so staying + // low proves the entry for another model didn't leak onto this one. + let cost = span_cost_of_one_call(&gw, tmp.path()).await; + assert!( + cost < 0.001, + "unlisted model priced at {cost}, which is the other model's \ + price-file rate — an `exact` override must not match a model it \ + doesn't name" + ); +} + #[tokio::test] #[ignore] async fn otel_litellm_gen_ai_attrs() { From 71df3cd544cead8fcfd489dd21b04cf6b323a323 Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Wed, 5 Aug 2026 11:40:24 +0300 Subject: [PATCH 3/3] refactor(bifrost): bake no price into the image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prices depend on provider, contract, account, and they change — so the image should not contain one. The previous commit left $2.50/$15 as a shell default in `start`, which made our IBM proxy's rate a property of the gateway image for everybody who runs it. Now nothing is baked. Both knobs are optional and independent: EVAL_COST_*_PER_TOKEN flat rate for every model, as a `*` wildcard EVAL_PRICING_FILE per-model rates, one `exact` override each Set both and the file wins for the models it names while the flat rate catches the rest. Set neither and we render no `pricing_overrides` at all, leaving bifrost's own bundled-catalog lookup exactly as it behaves out of the box — the unconfigured path is a working deployment, not a boot failure, so it warns rather than exits (models absent from that catalog report cost 0, which is otherwise invisible until someone reads a blank cost column). Two misconfigurations now fail loud instead of pricing half the bill at 0 or silently applying nothing (rule 22): one of the two flat knobs without the other, and an EVAL_PRICING_FILE path that doesn't exist — a typo in `-v` previously booted fine. Our own rate moves to deploy/values-openshift.yaml via the chart's existing gatewayExtraEnv hook, where changing it is an edit to a values file rather than an image rebuild. That is also the first thing that ever set it: the chart's gateway container had no cost env at all, so production was running on the image default. src/RULES.md 12 already says platform-specific settings belong in a composable values overlay. Tests: the two rate-dependent tests now pass the rate in the way a deployment does, plus a new one pinning that an unconfigured gateway still serves — an empty `pricing_overrides` array is easy to get wrong (a stray leading comma is invalid JSON and bifrost won't start). Verified by rendering all four combinations; file-only correctly strips the leading comma, and both misconfigurations exit 2. Signed-off-by: Elron Bandel --- containers/gateways/bifrost/Dockerfile | 8 +- .../gateways/bifrost/pricing.tsv.example | 26 +++-- containers/gateways/bifrost/start | 104 +++++++++++------- deploy/values-openshift.yaml | 19 ++++ tests/run/gateways/test.rs | 86 ++++++++++++--- 5 files changed, 170 insertions(+), 73 deletions(-) diff --git a/containers/gateways/bifrost/Dockerfile b/containers/gateways/bifrost/Dockerfile index 1dc00480..fc7e2f7b 100644 --- a/containers/gateways/bifrost/Dockerfile +++ b/containers/gateways/bifrost/Dockerfile @@ -56,10 +56,10 @@ COPY --from=bifrost-bin /app/main /opt/gateway/main COPY start /opt/gateway/start COPY health /opt/gateway/health COPY Caddyfile /opt/gateway/Caddyfile -# Documentation + starting point for a per-model price list. Shipped as -# `.example` so it is inert: `start` reads pricing.tsv (EVAL_PRICING_FILE), -# never this. Copying it in bakes no price into the image (rule 1) — a user -# mounts their own edited copy over the real path. +# Documentation + starting point for a per-model price list. Inert: `start` +# reads only the path EVAL_PRICING_FILE names, never this file, so the +# illustrative figures in it price nothing and no rate is baked into the image +# (rule 1). A user copies it, edits it, and mounts their own. COPY pricing.tsv.example /opt/gateway/data/pricing.tsv.example RUN chmod +x /opt/gateway/start /opt/gateway/health /opt/gateway/main \ && mkdir -p /opt/gateway/data diff --git a/containers/gateways/bifrost/pricing.tsv.example b/containers/gateways/bifrost/pricing.tsv.example index faac307b..24596486 100644 --- a/containers/gateways/bifrost/pricing.tsv.example +++ b/containers/gateways/bifrost/pricing.tsv.example @@ -1,29 +1,31 @@ -# Per-model token prices for the bifrost gateway. Copy, edit, and point the -# gateway at it: +# Per-model token prices for the bifrost gateway. NOT read by anything as +# shipped — it is documentation plus a starting point. Copy it, put YOUR prices +# in, and point the gateway at your copy: # # docker run -e EVAL_MODEL=azure/gpt-5.5 \ # -e EVAL_PRICING_FILE=/etc/eval/pricing.tsv \ -# -v ./pricing.tsv:/etc/eval/pricing.tsv:ro \ +# -v ./my-prices.tsv:/etc/eval/pricing.tsv:ro \ # ... ghcr.io/exgentic/models/bifrost:latest # -# Or mount it at the default path and skip the env var entirely: -# -v ./pricing.tsv:/opt/gateway/data/pricing.tsv:ro -# # The gateway renders these into bifrost's `governance.pricing_overrides` at # startup; the resulting cost lands on the OTel span as `gen_ai.usage.cost`, -# which is what the dashboard's cost column reads. +# which is what a dashboard's cost column reads. # # Format: three whitespace-separated fields. `#` comments and blank lines are # ignored. Prices are US dollars PER TOKEN, not per million — divide the usual # per-1M figure by 1,000,000 ($2.50/1M -> 0.0000025). # # The model is the BARE handle, exactly as you'd pass it in EVAL_MODEL -# (`azure/gpt-5.5`, not `openai/azure/gpt-5.5`). +# (`azure/gpt-5.5`, not `openai/azure/gpt-5.5`). A prefixed name matches nothing +# and silently leaves that model unpriced. +# +# A model NOT listed here is priced by whatever else is configured: +# EVAL_COST_{INPUT,OUTPUT}_PER_TOKEN if you set a flat rate alongside this file, +# otherwise bifrost's own bundled catalog — which reports cost 0 for any model +# it doesn't know. List the models you care about, or set the flat rate too. # -# Any model NOT listed here falls back to EVAL_COST_INPUT_PER_TOKEN / -# EVAL_COST_OUTPUT_PER_TOKEN, so a partial list is fine — nothing silently -# prices at zero. Listing no models at all (or mounting no file) leaves every -# model on that fallback. +# The figures below are ILLUSTRATIVE. They are not anyone's real prices; yours +# depend on your provider, contract, and account, and they change. # # model input/token output/token azure/gpt-5.4 0.0000025 0.0000150 diff --git a/containers/gateways/bifrost/start b/containers/gateways/bifrost/start index 3348a7b2..c5d5aa0a 100644 --- a/containers/gateways/bifrost/start +++ b/containers/gateways/bifrost/start @@ -13,22 +13,29 @@ # EVAL_MODEL_API OPTIONAL wire override: anthropic | openai | gemini. # OPENAI_API_BASE upstream root; each provider appends its native path. # LOG_LEVEL bifrost log level (default info). +# +# Pricing — what gen_ai.usage.cost on the OTel span gets multiplied by. Both +# knobs are OPTIONAL and no rate is baked into the image: a price belongs to the +# deployment, not to us. Set neither and bifrost prices from its own bundled +# catalog exactly as it does by default (see the pricing block below). # EVAL_COST_INPUT_PER_TOKEN, EVAL_COST_OUTPUT_PER_TOKEN -# what the upstream bills us per token (see the pricing -# override below). Defaults match the IBM litellm proxy. +# one flat rate for every model, in dollars PER TOKEN. For an +# upstream that bills the same regardless of model — the IBM +# litellm proxy bills $2.50/1M in, $15.00/1M out, i.e. +# 0.0000025 / 0.000015. Set both or neither. # EVAL_PRICING_FILE -# OPTIONAL price list, to charge different models different -# rates. Default /opt/gateway/data/pricing.tsv (absent in the -# image — mount one). Format, whitespace-separated: +# per-model rates. Format, whitespace-separated: # # # model in/token out/token # azure/gpt-5.5 0.0000050 0.0000300 # aws/claude-opus-5 0.0000150 0.0000750 # # `#` comments and blank lines are ignored. The model is the -# BARE handle you'd put in EVAL_MODEL. Any model not listed -# still gets EVAL_COST_*_PER_TOKEN, so an incomplete file -# undercharges nothing — it just falls back. +# BARE handle you'd put in EVAL_MODEL. A path that doesn't +# exist is an error (a mount that didn't land). +# +# Set both knobs and the file wins for the models it names while the flat rate +# catches everything else. # Upstream auth is the provider-native env var (OPENAI_API_KEY etc.) — no # umbrella alias (gateways/RULES.md rule 3). @@ -61,40 +68,44 @@ if [ -f "$TEMPLATE" ]; then RULES="$RULES{ \"id\": \"pin-$p\", \"name\": \"pin-$p\", \"enabled\": true, \"scope\": \"global\", \"priority\": $pr, \"cel_expression\": \"headers['x-eval-wire'] == '$p'\", \"targets\": [{ \"provider\": \"$p\"${MODEL}, \"weight\": 1.0 }] }" done fi - # PRICING: make gen_ai.usage.cost on the OTel span a real number. + # PRICING: what bifrost multiplies token counts by to fill gen_ai.usage.cost. + # + # Left alone, bifrost prices every call from its bundled pricing.json, keyed on + # the model name. That works for public catalog names and misses for anything + # else: route a proxy's own handles (`azure/gpt-5.5`, `rits/google/gemma-4-31B`) + # and the lookup finds nothing, so the span carries cost 0. There is no fallback + # layer inside bifrost — an unlisted model is silently free. # - # bifrost prices every call from its bundled pricing.json, keyed on the model - # name. Ours never matches: the handles we route are the upstream proxy's own - # (`azure/gpt-5.5`, `aws/claude-opus-5`, `rits/google/gemma-4-31B`), not - # litellm's public catalog names — so the lookup misses and bifrost writes - # gen_ai.usage.cost=0. Empirically that was 0 on every one of the ~2,100 - # provider-call spans in the results store, which is why the dashboard's cost - # column was blank. + # A price is a property of the deployment, not of this image: it depends on the + # provider, the contract, the account, and it changes without warning. So no + # rate is baked in. Configure it and we render `governance.pricing_overrides`; + # configure nothing and we render none, leaving bifrost's own pricing.json + # lookup exactly as it behaves out of the box. # - # The upstream DOES know the cost — it returns `x-litellm-response-cost` on - # every response — but that is an HTTP header, and bifrost's span carries the - # cost it computed itself, not one read off the wire. Rather than teach it to - # forward a header, give it the rate: a global wildcard pricing override makes - # its own arithmetic come out right. + # Two knobs, either or both: + # EVAL_COST_*_PER_TOKEN one flat rate for every model, as a `*` wildcard. + # Right for a proxy that bills one rate regardless of + # model (ours does). + # EVAL_PRICING_FILE per-model rates, one `exact` override each. + # Both set: the file wins for the models it names, the wildcard catches the + # rest. Specificity decides, and array order is irrelevant — a wildcard never + # shadows an exact entry. # - # One flat rate covers every model because the upstream bills one: probing it - # with gpt-5.4 / gpt-5.5 / gpt-4.1-mini / claude-opus-5 / claude-sonnet-5 / - # gemini-3-flash / gemma-4-31B returns the SAME cost for the same token counts. - # Solving two samples gives $2.50/1M in, $15.00/1M out, and that predicts - # `x-litellm-response-cost` exactly (to 1e-9) on every model tried. Overridable - # by env for when the contract changes. - : "${EVAL_COST_INPUT_PER_TOKEN:=0.0000025}" - : "${EVAL_COST_OUTPUT_PER_TOKEN:=0.000015}" # `pricing_patch` is JSON *inside* a JSON string, and this whole thing is a sed # replacement — so its inner quotes need a backslash that survives sed's own # unescaping (sed reduces `\"` to `"`). Hence the four backslashes: shell → # `\\"`, sed → `\"`, which is what the config file must contain. - PRICING="{ \"id\": \"00000000-0000-4000-8000-000000000001\", \"name\": \"upstream-flat-rate\", \"scope_kind\": \"global\", \"match_type\": \"wildcard\", \"pattern\": \"*\", \"request_types\": [\"chat_completion\"], \"pricing_patch\": \"{\\\\\"input_cost_per_token\\\\\":${EVAL_COST_INPUT_PER_TOKEN},\\\\\"output_cost_per_token\\\\\":${EVAL_COST_OUTPUT_PER_TOKEN}}\" }" - # A price list, if one is mounted, becomes one `exact` override per model, - # layered on top of the wildcard above. Specificity decides which applies — - # a listed model gets its own rate, everything else falls back to the - # wildcard, so a partial list never prices anything at 0. (Order within the - # array is irrelevant; the wildcard does not shadow an exact entry.) + PRICING="" + if [ -n "${EVAL_COST_INPUT_PER_TOKEN:-}" ] || [ -n "${EVAL_COST_OUTPUT_PER_TOKEN:-}" ]; then + # One without the other would silently price that half at 0 — worse than + # not configuring pricing at all, because the number looks real. + if [ -z "${EVAL_COST_INPUT_PER_TOKEN:-}" ] || [ -z "${EVAL_COST_OUTPUT_PER_TOKEN:-}" ]; then + echo "set BOTH EVAL_COST_INPUT_PER_TOKEN and EVAL_COST_OUTPUT_PER_TOKEN, or neither" >&2 + exit 2 + fi + PRICING="{ \"id\": \"00000000-0000-4000-8000-000000000001\", \"name\": \"flat-rate\", \"scope_kind\": \"global\", \"match_type\": \"wildcard\", \"pattern\": \"*\", \"request_types\": [\"chat_completion\"], \"pricing_patch\": \"{\\\\\"input_cost_per_token\\\\\":${EVAL_COST_INPUT_PER_TOKEN},\\\\\"output_cost_per_token\\\\\":${EVAL_COST_OUTPUT_PER_TOKEN}}\" }" + fi + # A price list becomes one `exact` override per model. # # `exact` matches the BARE requested model (`azure/gpt-5.5`), NOT the # provider-prefixed key bifrost uses to index its own pricing.json @@ -105,8 +116,12 @@ if [ -f "$TEMPLATE" ]; then # must run under plain `docker run`), so the file is whitespace-separated # rather than JSON. Emitting one object per line and stripping newlines # keeps the whole block a single sed replacement value. - : "${EVAL_PRICING_FILE:=/opt/gateway/data/pricing.tsv}" - if [ -f "$EVAL_PRICING_FILE" ]; then + CHART="" + if [ -n "${EVAL_PRICING_FILE:-}" ]; then + # An explicit path that isn't there is a mount that didn't land — a typo in + # `-v` would otherwise boot fine and price everything wrong (rule 22). + [ -f "$EVAL_PRICING_FILE" ] || { + echo "EVAL_PRICING_FILE=$EVAL_PRICING_FILE does not exist" >&2; exit 2; } CHART=$(awk ' /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } NF != 3 { @@ -122,10 +137,19 @@ if [ -f "$TEMPLATE" ]; then ' "$EVAL_PRICING_FILE") || { echo "EVAL_PRICING_FILE=$EVAL_PRICING_FILE is malformed (see above)" >&2; exit 2; } echo "pricing: loaded $(grep -cvE '^[[:space:]]*(#|$)' "$EVAL_PRICING_FILE") model rate(s) from $EVAL_PRICING_FILE" >&2 - else - CHART="" fi - GOVERNANCE="\"governance\": { \"routing_rules\": [${RULES}], \"pricing_overrides\": [${PRICING}${CHART}] }," + # Nothing configured -> no overrides -> bifrost's native pricing.json lookup, + # untouched. Say so, because the consequence (cost 0 for any model absent from + # that catalog) is otherwise invisible until someone reads a blank cost column. + if [ -z "$PRICING$CHART" ]; then + echo "pricing: none configured — bifrost prices from its bundled catalog; \ +models absent from it report cost 0. Set EVAL_COST_{INPUT,OUTPUT}_PER_TOKEN \ +and/or mount EVAL_PRICING_FILE to price them." >&2 + fi + # Strip the leading ", " the chart entries carry when no wildcard precedes them. + OVERRIDES="${PRICING}${CHART}" + case "$OVERRIDES" in ", "*) OVERRIDES="${OVERRIDES#, }" ;; esac + GOVERNANCE="\"governance\": { \"routing_rules\": [${RULES}], \"pricing_overrides\": [${OVERRIDES}] }," : "${OTEL_EXPORTER_OTLP_ENDPOINT:=http://otelcol:4318}" OTEL_COLLECTOR_URL="${OTEL_EXPORTER_OTLP_ENDPOINT%/}/v1/traces" diff --git a/deploy/values-openshift.yaml b/deploy/values-openshift.yaml index 77906554..201e2157 100644 --- a/deploy/values-openshift.yaml +++ b/deploy/values-openshift.yaml @@ -8,3 +8,22 @@ serviceAccountName: anyuid-sa # would serve a stale image cached on a node (e.g. an old core-otel that fails # the :13133 startup probe). Always re-pulls the current digest. imagePullPolicy: Always + +# What our upstream bills, so the gateway can put a real number in +# gen_ai.usage.cost (and therefore in the dashboard's cost column). Nothing is +# baked into the gateway image — a price is a property of this deployment, so it +# lives here where changing it is an edit to a values file, not a rebuild. +# +# One flat rate covers every model because the IBM litellm proxy bills one: +# probing it with gpt-5.4 / gpt-5.5 / gpt-4.1-mini / claude-opus-5 / +# claude-sonnet-5 / gemini-3-flash / gemma-4-31B returns the SAME cost for the +# same token counts. Solving two samples gives $2.50/1M in, $15.00/1M out, which +# then predicts the proxy's own `x-litellm-response-cost` to 1e-9 on every model +# tried (measured 2026-08). +# +# Per-token, not per-million. When the contract changes, change these. If the +# proxy ever starts billing per model, drop the flat pair and mount a price file +# instead (EVAL_PRICING_FILE — see containers/gateways/bifrost/pricing.tsv.example). +gatewayExtraEnv: + - { name: EVAL_COST_INPUT_PER_TOKEN, value: "0.0000025" } + - { name: EVAL_COST_OUTPUT_PER_TOKEN, value: "0.000015" } diff --git a/tests/run/gateways/test.rs b/tests/run/gateways/test.rs index 3208da0f..4c4da0ab 100644 --- a/tests/run/gateways/test.rs +++ b/tests/run/gateways/test.rs @@ -964,19 +964,30 @@ fn max_span_cost(traces: &str) -> Option { #[tokio::test] #[ignore] async fn otel_bifrost_span_cost_is_nonzero() { - // bifrost prices each call from its bundled pricing.json keyed on the - // model name, and the handles we route are the upstream proxy's own - // (`azure/gpt-5.4`, `aws/claude-opus-5`, …) — never in that catalog. - // So the lookup missed and every span carried gen_ai.usage.cost=0, - // which is what left the dashboard's cost column blank across the - // whole result history. gateways/bifrost/start now installs a - // governance pricing override with the rate the upstream actually - // bills; this pins that the span comes out with a real number on it. + // The original bug: bifrost prices each call from its bundled pricing.json + // keyed on the model name, and the handles we route are the upstream + // proxy's own (`azure/gpt-5.4`, `aws/claude-opus-5`, …) — never in that + // catalog. The lookup missed, every span carried gen_ai.usage.cost=0, and + // the dashboard's cost column was blank across the whole result history. + // + // Given a rate, the span must carry a real number. The rate is passed in + // here exactly the way a deployment passes it (deploy/values-openshift.yaml + // sets these two via gatewayExtraEnv) — the image itself has no default, so + // a test that set nothing would be testing bifrost's catalog, not us. // // Asserting the attribute is *populated*, not that it is under some // threshold — a cost SLO is forbidden (RULES.md rule 12). let tmp = tempfile::tempdir().expect("tempdir"); - let (_otel, gw) = start_pod_with_otel("bifrost", tmp.path()).await; + let (_otel, gw) = start_pod_with_otel_env( + "bifrost", + tmp.path(), + &[ + ("EVAL_COST_INPUT_PER_TOKEN", "0.0000025"), + ("EVAL_COST_OUTPUT_PER_TOKEN", "0.000015"), + ], + None, + ) + .await; let port = gateway_port(&gw).await; let resp = http() .post(format!( @@ -1100,19 +1111,28 @@ async fn otel_bifrost_prices_a_listed_model_from_the_mounted_price_file() { #[tokio::test] #[ignore] async fn otel_bifrost_falls_back_when_the_price_file_omits_the_model() { - // A partial price list must never price a model at 0 — the unlisted model - // stays on the flat EVAL_COST_* wildcard. This is what makes an - // incomplete file safe to mount, and it only holds because the wildcard - // override is layered alongside the per-model ones rather than replaced - // by them. + // A partial price list alongside a flat rate: the unlisted model must land + // on the flat wildcard — not on the other model's entry, and not on 0. + // This is what makes an incomplete file safe to mount, and it only holds + // because the wildcard override is layered alongside the per-model ones + // rather than replaced by them. let (_pdir, prices) = pricing_file("some/other-model 0.0010000 0.0020000\n"); let tmp = tempfile::tempdir().expect("tempdir"); - let (_otel, gw) = start_pod_with_otel_env("bifrost", tmp.path(), &[], Some(&prices)).await; + let (_otel, gw) = start_pod_with_otel_env( + "bifrost", + tmp.path(), + &[ + ("EVAL_COST_INPUT_PER_TOKEN", "0.0000025"), + ("EVAL_COST_OUTPUT_PER_TOKEN", "0.000015"), + ], + Some(&prices), + ) + .await; // Nonzero is enforced by the helper (it panics if no span carries a // cost). What this pins is *which* rate: the file's 0.001/token would put - // the cost three orders of magnitude above the flat default, so staying - // low proves the entry for another model didn't leak onto this one. + // the cost three orders of magnitude above the flat rate, so staying low + // proves the entry for another model didn't leak onto this one. let cost = span_cost_of_one_call(&gw, tmp.path()).await; assert!( cost < 0.001, @@ -1122,6 +1142,38 @@ async fn otel_bifrost_falls_back_when_the_price_file_omits_the_model() { ); } +#[tokio::test] +#[ignore] +async fn otel_bifrost_serves_with_no_pricing_configured() { + // No price is baked into the image, so the unconfigured case is a real + // deployment shape and must not be a boot failure: the gateway serves, and + // pricing is simply bifrost's own catalog lookup. Whether that lookup finds + // our handle is bifrost's business, not ours — the assertion is that the + // call succeeds, NOT that cost is any particular value. + // + // Guards the branch where both knobs are unset and `pricing_overrides` + // renders empty: an empty array is easy to get subtly wrong (a stray + // leading comma is invalid JSON and bifrost then fails to start). + let tmp = tempfile::tempdir().expect("tempdir"); + let (_otel, gw) = start_pod_with_otel_env("bifrost", tmp.path(), &[], None).await; + let port = gateway_port(&gw).await; + let resp = http() + .post(format!( + "http://127.0.0.1:{port}/openai/v1/chat/completions" + )) + .json(&body_openai()) + .send() + .await + .expect("post chat"); + assert_eq!( + resp.status(), + 200, + "unconfigured pricing must still serve — rendering an empty \ + pricing_overrides array broke the config. Body: {}", + resp.text().await.unwrap_or_default() + ); +} + #[tokio::test] #[ignore] async fn otel_litellm_gen_ai_attrs() {