diff --git a/tests/samples/router_logs_contract.ndjson b/tests/samples/router_logs_contract.ndjson new file mode 100644 index 0000000..fb67d34 --- /dev/null +++ b/tests/samples/router_logs_contract.ndjson @@ -0,0 +1,7 @@ +INFO: 127.0.0.1:1 - "GET /health HTTP/1.1" 200 OK +{"event":"route_complete","route_id":"chat.strong","target_model":"pro-router","policy_id":"hard_rule","reason":"hard_rule:vip","request_id":"req_001","stream":true,"duration_ms":312.5,"upstream_status":200} +{"event":"route_error","route_id":"chat.strong","target_model":"pro-router","policy_id":"embedding_error","reason":"embedding_error","request_id":"req_002","stream":true,"duration_ms":489,"error_type":"UpstreamStatusError","upstream_status":503,"upstream_error":"gateway_timeout"} +{"event":"route_complete","route_id":null,"target_model":"base-router","policy_id":"passthrough","reason":"passthrough","request_id":"req_003","stream":false,"duration_ms":142,"upstream_status":200} +{"event":"route_error","target_model":"legacy-router","policy_id":"hard_rule","reason":"hard_rule:legacy","request_id":"req_004","stream":false,"duration_ms":220,"error_type":"RemoteProtocolError","upstream_status":502,"upstream_error":"legacy_bad_gateway"} +{"event":"startup"} +{"event":"route_error", diff --git a/tests/test_check_route_error_budget.py b/tests/test_check_route_error_budget.py index 569b26e..b273508 100644 --- a/tests/test_check_route_error_budget.py +++ b/tests/test_check_route_error_budget.py @@ -3,6 +3,7 @@ import json import subprocess import sys +from pathlib import Path from scripts.check_route_error_budget import BudgetConfig, check_budget, format_budget_result, main from scripts.router_log_summary import ParseDiagnostics, parse_route_records @@ -635,3 +636,25 @@ def test_script_file_execution_returns_nonzero_for_failing_budget(): assert completed.returncode != 0 assert completed.stdout.startswith("FAIL route_error_budget\n") assert "error_rate 1.0000 exceeds max_error_rate 0.0000" in completed.stdout + +def test_contract_fixture_budget_includes_route_and_target_buckets(): + fixture = Path("tests/samples/router_logs_contract.ndjson") + diagnostics = ParseDiagnostics() + records = list(parse_route_records(fixture.read_text().splitlines(), diagnostics=diagnostics)) + + result = check_budget( + records, + BudgetConfig(min_total=1, max_error_rate=1.0, max_target_error_rate=1.0), + parse_diagnostics=diagnostics, + ) + + assert result.passed is True + assert result.target_error_rates == { + "base-router": 0.0, + "legacy-router": 1.0, + "pro-router": 0.5, + } + assert result.route_error_rates == {"chat.strong": 0.5, "unknown": 0.5} + assert result.error_types == {"RemoteProtocolError": 1, "UpstreamStatusError": 1} + assert result.parse_diagnostics.malformed_json_lines == 1 + assert result.parse_diagnostics.unknown_event_records == 1 diff --git a/tests/test_router_log_summary.py b/tests/test_router_log_summary.py index a5ce9cb..9dcb646 100644 --- a/tests/test_router_log_summary.py +++ b/tests/test_router_log_summary.py @@ -3,6 +3,7 @@ import json import subprocess import sys +from pathlib import Path from scripts.router_log_summary import ( ParseDiagnostics, @@ -311,3 +312,35 @@ def test_main_short_json_flag_outputs_machine_readable_json(): }, "upstream_statuses": {"200": 1, "503": 1}, } + +def test_contract_fixture_summary_separates_routes_and_targets(): + fixture = Path("tests/samples/router_logs_contract.ndjson") + diagnostics = ParseDiagnostics() + records = list(parse_route_records(fixture.read_text().splitlines(), diagnostics=diagnostics)) + + summary = summarize_records(records, parse_diagnostics=diagnostics) + + assert summary.total == 4 + assert summary.completed == 2 + assert summary.errors == 2 + assert summary.routes == {"chat.strong": 2} + assert summary.targets == {"base-router": 1, "legacy-router": 1, "pro-router": 2} + assert summary.error_types == {"RemoteProtocolError": 1, "UpstreamStatusError": 1} + assert summary.parse_diagnostics.malformed_json_lines == 1 + assert summary.parse_diagnostics.unknown_event_records == 1 + + +def test_contract_fixture_has_no_sensitive_payload_fields(): + fixture_text = Path("tests/samples/router_logs_contract.ndjson").read_text() + + forbidden_patterns = [ + "prompt", + "authorization", + "bearer ", + "request_body", + "raw_body", + ] + lower_fixture = fixture_text.lower() + + for pattern in forbidden_patterns: + assert pattern not in lower_fixture