Document and Test Provider Fallback Capabilities via Iterative Request Router

docs/architecture/ai-inference.md

@@ -78,6 +78,13 @@ All promoted values are validated against a 256-byte
 length limit and checked for control characters
 before propagation.
 
+Inside `iterative_request_router` steps, metadata is
+reset for credential isolation but extensions persist.
+Filters that read classifier metadata must have the
+classifier re-run inside their step. See
+`examples/configs/inference/fallback-with-translation.yaml`
+for the full pattern.
+
 ### Stateful vs Stateless
 
 Responses API requests are classified as "stateful"

examples/configs/inference/fallback-with-translation.yaml

@@ -0,0 +1,149 @@
+# Inference Fallback with Protocol Translation
+#
+# Demonstrates provider failover with Responses-to-Chat Completions
+# protocol translation using the iterative_request_router. When the
+# primary backend returns a retryable status, the request automatically
+# retries against a fallback provider. Both backends receive translated
+# Chat Completions requests with isolated credentials.
+#
+# Step boundary: the IRR resets per-step metadata for credential
+# isolation but preserves extensions across steps. This means:
+#
+#   - ResponsesState (extension, set by openai_responses_validate)
+#     survives step transitions.
+#   - openai_responses_format.format (metadata) does not.
+#
+# Because responses_to_chat_completions reads both metadata and
+# extensions, each step must re-run openai_responses_format and
+# openai_responses_validate to repopulate its metadata. Without
+# them the translation filter returns 500: "request pipeline state
+# is unavailable".
+#
+# Filter ordering within each step:
+#
+#   1. openai_responses_format         classify, set metadata
+#   2. openai_responses_model_rewrite  rewrite model (optional)
+#   3. openai_responses_validate       parse body, set metadata + extension
+#   4. responses_to_chat_completions   translate body using both
+#   5. path_rewrite                    /v1/responses -> /v1/chat/completions
+#   6. router                          select cluster
+#   7. credential_injection            inject per-cluster credentials
+#   8. load_balancer                   select endpoint
+#
+# Key ordering constraints:
+#   - 1 before 2: model rewrite reads openai_responses_format.stream
+#   - 2 before 3: validate parses after rewrite, so ResponsesState
+#     carries the effective model
+#   - 3 before 4: translation reads responses.response_id metadata
+#     and ResponsesState extension, both set by validate
+#   - 6 before 7: credential injection matches on ctx.cluster,
+#     which the router sets
+#
+# Pre-IRR filters:
+#   openai_responses_format rejects non-Responses requests early.
+#   openai_responses_validate rejects malformed bodies before the
+#   IRR allocates sub-request resources.
+#
+# Adapting for real providers:
+#   - Add openai_responses_model_rewrite between format and validate
+#     in the fallback step to map model names across providers.
+#   - Use env_var instead of value in credential_injection to read
+#     API keys from the environment at startup.
+#   - Add a headers filter (request_set Host) before load_balancer
+#     for external HTTPS upstreams.
+#   - Add tls: { sni: "hostname" } to the cluster for HTTPS.
+#   - Adjust the path_rewrite replacement per provider (e.g.
+#     /api/v1/chat/completions for OpenRouter).
+#
+# Usage:
+#   cargo run -p praxis-ai-proxy -- -c examples/configs/inference/fallback-with-translation.yaml
+
+listeners:
+  - name: ai-gateway
+    address: "127.0.0.1:8080"
+    filter_chains: [inference-failover]
+
+filter_chains:
+  - name: inference-failover
+    filters:
+      - filter: openai_responses_format
+        on_invalid: reject
+        headers:
+          format: x-praxis-ai-format
+          model: x-praxis-ai-model
+          stream: x-praxis-ai-stream
+
+      - filter: openai_responses_validate
+
+      - filter: iterative_request_router
+        initial_step: primary
+        steps:
+          - name: primary
+            filters:
+              - filter: openai_responses_format
+              - filter: openai_responses_validate
+              - filter: responses_to_chat_completions
+                max_body_bytes: 67108864
+              - filter: path_rewrite
+                replace:
+                  pattern: "^/v1/responses/?$"
+                  replacement: "/v1/chat/completions"
+                conditions:
+                  - when:
+                      path_prefix: "/v1/responses"
+                      methods: [POST]
+              - filter: router
+                routes:
+                  - path_prefix: "/"
+                    cluster: primary-backend
+              - filter: credential_injection
+                clusters:
+                  - name: primary-backend
+                    header: Authorization
+                    value: "primary-key"
+                    header_prefix: "Bearer "
+                    strip_client_credential: true
+              - filter: load_balancer
+                clusters:
+                  - name: primary-backend
+                    endpoints:
+                      - "127.0.0.1:3001"
+            on_result:
+              - status: [429, 502, 503, 504]
+                next: fallback
+              - default: true
+                done: true
+
+          - name: fallback
+            filters:
+              - filter: openai_responses_format
+              - filter: openai_responses_validate
+              - filter: responses_to_chat_completions
+                max_body_bytes: 67108864
+              - filter: path_rewrite
+                replace:
+                  pattern: "^/v1/responses/?$"
+                  replacement: "/v1/chat/completions"
+                conditions:
+                  - when:
+                      path_prefix: "/v1/responses"
+                      methods: [POST]
+              - filter: router
+                routes:
+                  - path_prefix: "/"
+                    cluster: fallback-backend
+              - filter: credential_injection
+                clusters:
+                  - name: fallback-backend
+                    header: Authorization
+                    value: "fallback-key"
+                    header_prefix: "Bearer "
+                    strip_client_credential: true
+              - filter: load_balancer
+                clusters:
+                  - name: fallback-backend
+                    endpoints:
+                      - "127.0.0.1:3002"
+            on_result:
+              - default: true
+                done: true

tests/integration/tests/suite/examples/inference_fallback.rs

@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 Praxis Contributors
+
+//! Conformance tests for inference failover with protocol translation.
+//!
+//! Validates that the iterative request router composes correctly with
+//! Responses-to-Chat Completions translation and credential injection
+//! across a failover boundary.
+
+use std::collections::HashMap;
+
+use praxis_test_utils::{
+    StatefulCapturingBackend, free_port, http_send, json_post, parse_body, parse_status, start_proxy,
+};
+
+use super::load_example_config;
+
+const EXAMPLE: &str = "inference/fallback-with-translation.yaml";
+
+fn chat_completions_response() -> String {
+    serde_json::json!({
+        "id": "chatcmpl_test",
+        "object": "chat.completion",
+        "model": "gpt-4.1-mini",
+        "choices": [{
+            "index": 0,
+            "message": {"role": "assistant", "content": "Hello from fallback."},
+            "finish_reason": "stop"
+        }],
+        "usage": {"prompt_tokens": 5, "completion_tokens": 4, "total_tokens": 9}
+    })
+    .to_string()
+}
+
+fn responses_request() -> String {
+    serde_json::json!({
+        "model": "gpt-4.1-mini",
+        "input": "Hello",
+        "stream": false,
+        "store": false
+    })
+    .to_string()
+}
+
+#[test]
+fn fallback_on_primary_503() {
+    let primary = StatefulCapturingBackend::new(vec![(503, r#"{"error":"service unavailable"}"#.to_owned())])
+        .start_with_shutdown();
+    let fallback = StatefulCapturingBackend::new(vec![(200, chat_completions_response())]).start_with_shutdown();
+
+    let proxy_port = free_port();
+    let config = load_example_config(
+        EXAMPLE,
+        proxy_port,
+        HashMap::from([("127.0.0.1:3001", primary.port()), ("127.0.0.1:3002", fallback.port())]),
+    );
+    let proxy = start_proxy(&config);
+
+    let raw = http_send(proxy.addr(), &json_post("/v1/responses", &responses_request()));
+
+    // -- Primary assertions --
+    let primary_requests = primary.requests();
+    assert_eq!(primary_requests.len(), 1, "primary should receive exactly one request");
+    let primary_req = &primary_requests[0];
+    assert_eq!(
+        primary_req.uri, "/v1/chat/completions",
+        "primary path should be rewritten"
+    );
+    let primary_body: serde_json::Value =
+        serde_json::from_str(&primary_req.body).expect("primary request body should be JSON");
+    assert!(
+        primary_body["messages"].is_array(),
+        "primary should receive translated messages array"
+    );
+    assert!(
+        primary_req.headers.contains("Bearer primary-key"),
+        "primary should receive primary credentials"
+    );
+
+    // -- Fallback assertions --
+    let fallback_requests = fallback.requests();
+    assert_eq!(
+        fallback_requests.len(),
+        1,
+        "fallback should receive exactly one request"
+    );
+    let fallback_req = &fallback_requests[0];
+    assert_eq!(
+        fallback_req.uri, "/v1/chat/completions",
+        "fallback path should be rewritten"
+    );
+    let fallback_body: serde_json::Value =
+        serde_json::from_str(&fallback_req.body).expect("fallback request body should be JSON");
+    assert!(
+        fallback_body["messages"].is_array(),
+        "fallback should receive translated messages array"
+    );
+    assert!(
+        fallback_req.headers.contains("Bearer fallback-key"),
+        "fallback should receive fallback credentials"
+    );
+
+    // -- Client response assertions --
+    let status = parse_status(&raw);
+    assert_eq!(status, 200, "client should receive 200 after fallback succeeds");
+    let response: serde_json::Value = serde_json::from_str(&parse_body(&raw)).expect("client response should be JSON");
+    assert_eq!(
+        response["object"], "response",
+        "response should be a Responses API resource"
+    );
+    assert_eq!(
+        response["output"][0]["content"][0]["text"], "Hello from fallback.",
+        "response text should match fallback backend"
+    );
+    assert_eq!(response["usage"]["input_tokens"], 5);
+    assert_eq!(response["usage"]["output_tokens"], 4);
+}
+
+#[test]
+fn primary_succeeds_no_fallback() {
+    let primary_response = serde_json::json!({
+        "id": "chatcmpl_primary",
+        "object": "chat.completion",
+        "model": "gpt-4.1-mini",
+        "choices": [{
+            "index": 0,
+            "message": {"role": "assistant", "content": "Hello from primary."},
+            "finish_reason": "stop"
+        }],
+        "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}
+    })
+    .to_string();
+
+    let primary = StatefulCapturingBackend::new(vec![(200, primary_response)]).start_with_shutdown();
+    let fallback = StatefulCapturingBackend::new(vec![(200, chat_completions_response())]).start_with_shutdown();
+
+    let proxy_port = free_port();
+    let config = load_example_config(
+        EXAMPLE,
+        proxy_port,
+        HashMap::from([("127.0.0.1:3001", primary.port()), ("127.0.0.1:3002", fallback.port())]),
+    );
+    let proxy = start_proxy(&config);
+
+    let raw = http_send(proxy.addr(), &json_post("/v1/responses", &responses_request()));
+
+    // -- Primary assertions --
+    let primary_requests = primary.requests();
+    assert_eq!(primary_requests.len(), 1, "primary should receive exactly one request");
+
+    // -- Fallback assertions --
+    let fallback_requests = fallback.requests();
+    assert_eq!(fallback_requests.len(), 0, "fallback should receive zero requests");
+
+    // -- Client response assertions --
+    let status = parse_status(&raw);
+    assert_eq!(status, 200, "client should receive 200 from primary");
+    let response: serde_json::Value = serde_json::from_str(&parse_body(&raw)).expect("client response should be JSON");
+    assert_eq!(
+        response["object"], "response",
+        "response should be a Responses API resource"
+    );
+    assert_eq!(
+        response["output"][0]["content"][0]["text"], "Hello from primary.",
+        "response text should match primary backend"
+    );
+    assert_eq!(response["usage"]["input_tokens"], 3);
+    assert_eq!(response["usage"]["output_tokens"], 2);
+}

tests/integration/tests/suite/examples/mod.rs

@@ -16,6 +16,7 @@ mod credential_injection;
 mod file_search_callout;
 mod full_flow;
 mod guardrails;
+mod inference_fallback;
 mod mcp_broker;
 mod model_to_header;
 mod openai_conversations;