From f43383b8d9cb7ec21047d36b18aebb0a1cde9637 Mon Sep 17 00:00:00 2001 From: usize Date: Wed, 12 Aug 2026 06:06:33 -0700 Subject: [PATCH 1/2] Add conformance test: inference fallback with protocol translation First conformance test for the AI gateway. Validates that an iterative_request_router composes correctly with responses_to_chat_completions protocol translation and credential_injection across a failover boundary. Two tests exercise the example config: - fallback_on_primary_503: primary returns 503, fallback receives the translated Chat Completions request with isolated credentials, client gets a Responses API resource. - primary_succeeds_no_fallback: primary returns 200, fallback receives no requests. Each IRR step re-runs openai_responses_format and openai_responses_validate because the IRR resets per-step metadata for credential isolation while preserving extensions. Assisted by Opus 4.6 Signed-off-by: usize --- .../inference/fallback-with-translation.yaml | 120 +++++++++++++ .../suite/examples/inference_fallback.rs | 169 ++++++++++++++++++ tests/integration/tests/suite/examples/mod.rs | 1 + 3 files changed, 290 insertions(+) create mode 100644 examples/configs/inference/fallback-with-translation.yaml create mode 100644 tests/integration/tests/suite/examples/inference_fallback.rs diff --git a/examples/configs/inference/fallback-with-translation.yaml b/examples/configs/inference/fallback-with-translation.yaml new file mode 100644 index 000000000..21e7e2c34 --- /dev/null +++ b/examples/configs/inference/fallback-with-translation.yaml @@ -0,0 +1,120 @@ +# Inference Fallback with Protocol Translation +# +# Demonstrates provider failover with Responses-to-Chat Completions +# protocol translation using the iterative_request_router. When the +# primary backend returns a retryable status, the request automatically +# retries against a fallback provider. Both backends receive translated +# Chat Completions requests with isolated credentials. +# +# Pipeline ordering (pre-IRR, runs once): +# 1. openai_responses_format classifies the client request and +# promotes header metadata. Rejects non-Responses requests early. +# 2. openai_responses_validate validates parameters, generates the +# ResponsesState extension shared across steps, and rejects +# malformed bodies before entering the IRR. +# +# IRR steps (each step re-runs openai_responses_format and +# openai_responses_validate because the IRR resets per-step metadata +# for credential isolation; extensions persist across steps): +# primary: classifies, validates, translates Responses to Chat +# Completions, rewrites path, routes, injects primary +# credentials, load-balances to primary backend. +# fallback: same pipeline, injects fallback credentials, routes +# to fallback backend. +# +# Transition rules: +# status [429, 502, 503, 504] -> next: fallback +# default -> done (return to client) +# +# Usage: +# cargo run -p praxis-ai-proxy -- -c examples/configs/inference/fallback-with-translation.yaml + +listeners: + - name: ai-gateway + address: "127.0.0.1:8080" + filter_chains: [inference-failover] + +filter_chains: + - name: inference-failover + filters: + - filter: openai_responses_format + on_invalid: reject + headers: + format: x-praxis-ai-format + model: x-praxis-ai-model + stream: x-praxis-ai-stream + + - filter: openai_responses_validate + + - filter: iterative_request_router + initial_step: primary + steps: + - name: primary + filters: + - filter: openai_responses_format + - filter: openai_responses_validate + - filter: responses_to_chat_completions + max_body_bytes: 67108864 + - filter: path_rewrite + replace: + pattern: "^/v1/responses/?$" + replacement: "/v1/chat/completions" + conditions: + - when: + path_prefix: "/v1/responses" + methods: [POST] + - filter: router + routes: + - path_prefix: "/" + cluster: primary-backend + - filter: credential_injection + clusters: + - name: primary-backend + header: Authorization + value: "primary-key" + header_prefix: "Bearer " + strip_client_credential: true + - filter: load_balancer + clusters: + - name: primary-backend + endpoints: + - "127.0.0.1:3001" + on_result: + - status: [429, 502, 503, 504] + next: fallback + - default: true + done: true + + - name: fallback + filters: + - filter: openai_responses_format + - filter: openai_responses_validate + - filter: responses_to_chat_completions + max_body_bytes: 67108864 + - filter: path_rewrite + replace: + pattern: "^/v1/responses/?$" + replacement: "/v1/chat/completions" + conditions: + - when: + path_prefix: "/v1/responses" + methods: [POST] + - filter: router + routes: + - path_prefix: "/" + cluster: fallback-backend + - filter: credential_injection + clusters: + - name: fallback-backend + header: Authorization + value: "fallback-key" + header_prefix: "Bearer " + strip_client_credential: true + - filter: load_balancer + clusters: + - name: fallback-backend + endpoints: + - "127.0.0.1:3002" + on_result: + - default: true + done: true diff --git a/tests/integration/tests/suite/examples/inference_fallback.rs b/tests/integration/tests/suite/examples/inference_fallback.rs new file mode 100644 index 000000000..9dc3a7cf2 --- /dev/null +++ b/tests/integration/tests/suite/examples/inference_fallback.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Praxis Contributors + +//! Conformance tests for inference failover with protocol translation. +//! +//! Validates that the iterative request router composes correctly with +//! Responses-to-Chat Completions translation and credential injection +//! across a failover boundary. + +use std::collections::HashMap; + +use praxis_test_utils::{ + StatefulCapturingBackend, free_port, http_send, json_post, parse_body, parse_status, start_proxy, +}; + +use super::load_example_config; + +const EXAMPLE: &str = "inference/fallback-with-translation.yaml"; + +fn chat_completions_response() -> String { + serde_json::json!({ + "id": "chatcmpl_test", + "object": "chat.completion", + "model": "gpt-4.1-mini", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "Hello from fallback."}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 5, "completion_tokens": 4, "total_tokens": 9} + }) + .to_string() +} + +fn responses_request() -> String { + serde_json::json!({ + "model": "gpt-4.1-mini", + "input": "Hello", + "stream": false, + "store": false + }) + .to_string() +} + +#[test] +fn fallback_on_primary_503() { + let primary = StatefulCapturingBackend::new(vec![(503, r#"{"error":"service unavailable"}"#.to_owned())]) + .start_with_shutdown(); + let fallback = StatefulCapturingBackend::new(vec![(200, chat_completions_response())]).start_with_shutdown(); + + let proxy_port = free_port(); + let config = load_example_config( + EXAMPLE, + proxy_port, + HashMap::from([("127.0.0.1:3001", primary.port()), ("127.0.0.1:3002", fallback.port())]), + ); + let proxy = start_proxy(&config); + + let raw = http_send(proxy.addr(), &json_post("/v1/responses", &responses_request())); + + // -- Primary assertions -- + let primary_requests = primary.requests(); + assert_eq!(primary_requests.len(), 1, "primary should receive exactly one request"); + let primary_req = &primary_requests[0]; + assert_eq!( + primary_req.uri, "/v1/chat/completions", + "primary path should be rewritten" + ); + let primary_body: serde_json::Value = + serde_json::from_str(&primary_req.body).expect("primary request body should be JSON"); + assert!( + primary_body["messages"].is_array(), + "primary should receive translated messages array" + ); + assert!( + primary_req.headers.contains("Bearer primary-key"), + "primary should receive primary credentials" + ); + + // -- Fallback assertions -- + let fallback_requests = fallback.requests(); + assert_eq!( + fallback_requests.len(), + 1, + "fallback should receive exactly one request" + ); + let fallback_req = &fallback_requests[0]; + assert_eq!( + fallback_req.uri, "/v1/chat/completions", + "fallback path should be rewritten" + ); + let fallback_body: serde_json::Value = + serde_json::from_str(&fallback_req.body).expect("fallback request body should be JSON"); + assert!( + fallback_body["messages"].is_array(), + "fallback should receive translated messages array" + ); + assert!( + fallback_req.headers.contains("Bearer fallback-key"), + "fallback should receive fallback credentials" + ); + + // -- Client response assertions -- + let status = parse_status(&raw); + assert_eq!(status, 200, "client should receive 200 after fallback succeeds"); + let response: serde_json::Value = serde_json::from_str(&parse_body(&raw)).expect("client response should be JSON"); + assert_eq!( + response["object"], "response", + "response should be a Responses API resource" + ); + assert_eq!( + response["output"][0]["content"][0]["text"], "Hello from fallback.", + "response text should match fallback backend" + ); + assert_eq!(response["usage"]["input_tokens"], 5); + assert_eq!(response["usage"]["output_tokens"], 4); +} + +#[test] +fn primary_succeeds_no_fallback() { + let primary_response = serde_json::json!({ + "id": "chatcmpl_primary", + "object": "chat.completion", + "model": "gpt-4.1-mini", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "Hello from primary."}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5} + }) + .to_string(); + + let primary = StatefulCapturingBackend::new(vec![(200, primary_response)]).start_with_shutdown(); + let fallback = StatefulCapturingBackend::new(vec![(200, chat_completions_response())]).start_with_shutdown(); + + let proxy_port = free_port(); + let config = load_example_config( + EXAMPLE, + proxy_port, + HashMap::from([("127.0.0.1:3001", primary.port()), ("127.0.0.1:3002", fallback.port())]), + ); + let proxy = start_proxy(&config); + + let raw = http_send(proxy.addr(), &json_post("/v1/responses", &responses_request())); + + // -- Primary assertions -- + let primary_requests = primary.requests(); + assert_eq!(primary_requests.len(), 1, "primary should receive exactly one request"); + + // -- Fallback assertions -- + let fallback_requests = fallback.requests(); + assert_eq!(fallback_requests.len(), 0, "fallback should receive zero requests"); + + // -- Client response assertions -- + let status = parse_status(&raw); + assert_eq!(status, 200, "client should receive 200 from primary"); + let response: serde_json::Value = serde_json::from_str(&parse_body(&raw)).expect("client response should be JSON"); + assert_eq!( + response["object"], "response", + "response should be a Responses API resource" + ); + assert_eq!( + response["output"][0]["content"][0]["text"], "Hello from primary.", + "response text should match primary backend" + ); + assert_eq!(response["usage"]["input_tokens"], 3); + assert_eq!(response["usage"]["output_tokens"], 2); +} diff --git a/tests/integration/tests/suite/examples/mod.rs b/tests/integration/tests/suite/examples/mod.rs index 0524479a1..547e808a8 100644 --- a/tests/integration/tests/suite/examples/mod.rs +++ b/tests/integration/tests/suite/examples/mod.rs @@ -16,6 +16,7 @@ mod credential_injection; mod file_search_callout; mod full_flow; mod guardrails; +mod inference_fallback; mod mcp_broker; mod model_to_header; mod openai_conversations; From 22965c7d7aa85d28bfeac672f89c4f1bb4d6d1dd Mon Sep 17 00:00:00 2001 From: usize Date: Wed, 12 Aug 2026 09:32:12 -0700 Subject: [PATCH 2/2] Document step boundary rules in config comments Move the metadata/extension persistence explanation into the example config header comments, where someone building a pipeline encounters it. Add a one-liner to ai-inference.md pointing to the example. Document real-world adaptation (model rewrite, TLS/SNI, env_var credentials, Host header) in the "Adapting for real providers" section. Assisted by Opus 4.6 Signed-off-by: usize --- docs/architecture/ai-inference.md | 7 +++ .../inference/fallback-with-translation.yaml | 63 ++++++++++++++----- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/docs/architecture/ai-inference.md b/docs/architecture/ai-inference.md index 324940544..d70c86280 100644 --- a/docs/architecture/ai-inference.md +++ b/docs/architecture/ai-inference.md @@ -78,6 +78,13 @@ All promoted values are validated against a 256-byte length limit and checked for control characters before propagation. +Inside `iterative_request_router` steps, metadata is +reset for credential isolation but extensions persist. +Filters that read classifier metadata must have the +classifier re-run inside their step. See +`examples/configs/inference/fallback-with-translation.yaml` +for the full pattern. + ### Stateful vs Stateless Responses API requests are classified as "stateful" diff --git a/examples/configs/inference/fallback-with-translation.yaml b/examples/configs/inference/fallback-with-translation.yaml index 21e7e2c34..6801db805 100644 --- a/examples/configs/inference/fallback-with-translation.yaml +++ b/examples/configs/inference/fallback-with-translation.yaml @@ -6,25 +6,54 @@ # retries against a fallback provider. Both backends receive translated # Chat Completions requests with isolated credentials. # -# Pipeline ordering (pre-IRR, runs once): -# 1. openai_responses_format classifies the client request and -# promotes header metadata. Rejects non-Responses requests early. -# 2. openai_responses_validate validates parameters, generates the -# ResponsesState extension shared across steps, and rejects -# malformed bodies before entering the IRR. +# Step boundary: the IRR resets per-step metadata for credential +# isolation but preserves extensions across steps. This means: # -# IRR steps (each step re-runs openai_responses_format and -# openai_responses_validate because the IRR resets per-step metadata -# for credential isolation; extensions persist across steps): -# primary: classifies, validates, translates Responses to Chat -# Completions, rewrites path, routes, injects primary -# credentials, load-balances to primary backend. -# fallback: same pipeline, injects fallback credentials, routes -# to fallback backend. +# - ResponsesState (extension, set by openai_responses_validate) +# survives step transitions. +# - openai_responses_format.format (metadata) does not. # -# Transition rules: -# status [429, 502, 503, 504] -> next: fallback -# default -> done (return to client) +# Because responses_to_chat_completions reads both metadata and +# extensions, each step must re-run openai_responses_format and +# openai_responses_validate to repopulate its metadata. Without +# them the translation filter returns 500: "request pipeline state +# is unavailable". +# +# Filter ordering within each step: +# +# 1. openai_responses_format classify, set metadata +# 2. openai_responses_model_rewrite rewrite model (optional) +# 3. openai_responses_validate parse body, set metadata + extension +# 4. responses_to_chat_completions translate body using both +# 5. path_rewrite /v1/responses -> /v1/chat/completions +# 6. router select cluster +# 7. credential_injection inject per-cluster credentials +# 8. load_balancer select endpoint +# +# Key ordering constraints: +# - 1 before 2: model rewrite reads openai_responses_format.stream +# - 2 before 3: validate parses after rewrite, so ResponsesState +# carries the effective model +# - 3 before 4: translation reads responses.response_id metadata +# and ResponsesState extension, both set by validate +# - 6 before 7: credential injection matches on ctx.cluster, +# which the router sets +# +# Pre-IRR filters: +# openai_responses_format rejects non-Responses requests early. +# openai_responses_validate rejects malformed bodies before the +# IRR allocates sub-request resources. +# +# Adapting for real providers: +# - Add openai_responses_model_rewrite between format and validate +# in the fallback step to map model names across providers. +# - Use env_var instead of value in credential_injection to read +# API keys from the environment at startup. +# - Add a headers filter (request_set Host) before load_balancer +# for external HTTPS upstreams. +# - Add tls: { sni: "hostname" } to the cluster for HTTPS. +# - Adjust the path_rewrite replacement per provider (e.g. +# /api/v1/chat/completions for OpenRouter). # # Usage: # cargo run -p praxis-ai-proxy -- -c examples/configs/inference/fallback-with-translation.yaml