Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions tests/samples/router_logs_contract.ndjson
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
INFO: 127.0.0.1:1 - "GET /health HTTP/1.1" 200 OK
{"event":"route_complete","route_id":"chat.strong","target_model":"pro-router","policy_id":"hard_rule","reason":"hard_rule:vip","request_id":"req_001","stream":true,"duration_ms":312.5,"upstream_status":200}
{"event":"route_error","route_id":"chat.strong","target_model":"pro-router","policy_id":"embedding_error","reason":"embedding_error","request_id":"req_002","stream":true,"duration_ms":489,"error_type":"UpstreamStatusError","upstream_status":503,"upstream_error":"gateway_timeout"}
{"event":"route_complete","route_id":null,"target_model":"base-router","policy_id":"passthrough","reason":"passthrough","request_id":"req_003","stream":false,"duration_ms":142,"upstream_status":200}
{"event":"route_error","target_model":"legacy-router","policy_id":"hard_rule","reason":"hard_rule:legacy","request_id":"req_004","stream":false,"duration_ms":220,"error_type":"RemoteProtocolError","upstream_status":502,"upstream_error":"legacy_bad_gateway"}
{"event":"startup"}
{"event":"route_error",
23 changes: 23 additions & 0 deletions tests/test_check_route_error_budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import json
import subprocess
import sys
from pathlib import Path

from scripts.check_route_error_budget import BudgetConfig, check_budget, format_budget_result, main
from scripts.router_log_summary import ParseDiagnostics, parse_route_records
Expand Down Expand Up @@ -635,3 +636,25 @@ def test_script_file_execution_returns_nonzero_for_failing_budget():
assert completed.returncode != 0
assert completed.stdout.startswith("FAIL route_error_budget\n")
assert "error_rate 1.0000 exceeds max_error_rate 0.0000" in completed.stdout

def test_contract_fixture_budget_includes_route_and_target_buckets():
fixture = Path("tests/samples/router_logs_contract.ndjson")
diagnostics = ParseDiagnostics()
records = list(parse_route_records(fixture.read_text().splitlines(), diagnostics=diagnostics))

result = check_budget(
records,
BudgetConfig(min_total=1, max_error_rate=1.0, max_target_error_rate=1.0),
parse_diagnostics=diagnostics,
)

assert result.passed is True
assert result.target_error_rates == {
"base-router": 0.0,
"legacy-router": 1.0,
"pro-router": 0.5,
}
assert result.route_error_rates == {"chat.strong": 0.5, "unknown": 0.5}
assert result.error_types == {"RemoteProtocolError": 1, "UpstreamStatusError": 1}
assert result.parse_diagnostics.malformed_json_lines == 1
assert result.parse_diagnostics.unknown_event_records == 1
33 changes: 33 additions & 0 deletions tests/test_router_log_summary.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import json
import subprocess
import sys
from pathlib import Path

from scripts.router_log_summary import (
ParseDiagnostics,
Expand Down Expand Up @@ -311,3 +312,35 @@ def test_main_short_json_flag_outputs_machine_readable_json():
},
"upstream_statuses": {"200": 1, "503": 1},
}

def test_contract_fixture_summary_separates_routes_and_targets():
fixture = Path("tests/samples/router_logs_contract.ndjson")
diagnostics = ParseDiagnostics()
records = list(parse_route_records(fixture.read_text().splitlines(), diagnostics=diagnostics))

summary = summarize_records(records, parse_diagnostics=diagnostics)

assert summary.total == 4
assert summary.completed == 2
assert summary.errors == 2
assert summary.routes == {"chat.strong": 2}
assert summary.targets == {"base-router": 1, "legacy-router": 1, "pro-router": 2}
assert summary.error_types == {"RemoteProtocolError": 1, "UpstreamStatusError": 1}
assert summary.parse_diagnostics.malformed_json_lines == 1
assert summary.parse_diagnostics.unknown_event_records == 1


def test_contract_fixture_has_no_sensitive_payload_fields():
fixture_text = Path("tests/samples/router_logs_contract.ndjson").read_text()

forbidden_patterns = [
"prompt",
"authorization",
"bearer ",
"request_body",
"raw_body",
]
lower_fixture = fixture_text.lower()

for pattern in forbidden_patterns:
assert pattern not in lower_fixture
Loading