From ca15f43161e1757cc97bc24443adadb63af05e63 Mon Sep 17 00:00:00 2001 From: mahsudbekyuldoshev Date: Tue, 14 Jul 2026 21:42:39 +0500 Subject: [PATCH 1/3] feat: add unit tests for audit command closes #103 --- tests/unit/test_cli/test_audit.py | 334 ++++++++++++++++++++++++++++++ uv.lock | 20 +- 2 files changed, 344 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_cli/test_audit.py diff --git a/tests/unit/test_cli/test_audit.py b/tests/unit/test_cli/test_audit.py new file mode 100644 index 0000000..1f6a5f8 --- /dev/null +++ b/tests/unit/test_cli/test_audit.py @@ -0,0 +1,334 @@ +"""Tests for the audit CLI command.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from typer.testing import CliRunner + +from openagent_eval.cli.main import app +from openagent_eval.corpus.models import IssueSeverity +from openagent_eval.exceptions.corpus import CorpusAuditError, CorpusNotFoundError + +runner = CliRunner() + + +class _IssueTypeStub(str): + """Hashable stand-in for an issue-type enum member. + + Behaves like a string for dict-keying/equality purposes but also + exposes ``.value`` the way the real enum does, since the CLI code + reads ``issue.issue_type.value``. + """ + + @property + def value(self) -> str: + return str(self) + + +def make_issue( + severity: IssueSeverity = IssueSeverity.MEDIUM, + issue_type: str = "duplicate", + title: str = "Sample issue", + document_ids: list[str] | None = None, + metadata: dict | None = None, + description: str = "A sample issue description.", +) -> SimpleNamespace: + """Build a lightweight stand-in for an audit issue.""" + return SimpleNamespace( + severity=severity, + issue_type=_IssueTypeStub(issue_type), + title=title, + document_ids=document_ids if document_ids is not None else ["doc1", "doc2"], + metadata=metadata if metadata is not None else {}, + description=description, + ) + + +def make_report( + health_score: float = 0.95, + summary: str = "Corpus looks healthy.", + issues: list | None = None, + checks_performed: list[str] | None = None, + total_documents: int = 10, +) -> SimpleNamespace: + """Build a lightweight stand-in for an AuditReport.""" + issues = issues if issues is not None else [] + by_type: dict = {} + for issue in issues: + by_type.setdefault(issue.issue_type, []).append(issue) + + return SimpleNamespace( + health_score=health_score, + summary=summary, + issues=issues, + checks_performed=checks_performed if checks_performed is not None else ["contradiction", "staleness"], + total_documents=total_documents, + issues_by_type=by_type, + ) + + +@pytest.fixture +def corpus_dir(tmp_path): + """A temporary directory standing in for a corpus path.""" + path = tmp_path / "corpus" + path.mkdir() + (path / "doc1.md").write_text("Some content.", encoding="utf-8") + return path + + +class TestAuditHelp: + """Tests for --help output of the audit command.""" + + def test_audit_help(self): + result = runner.invoke(app, ["audit", "--help"]) + assert result.exit_code == 0 + assert "audit" in result.output.lower() + + def test_audit_help_shows_checks_option(self): + result = runner.invoke(app, ["audit", "--help"]) + assert result.exit_code == 0 + assert "--checks" in result.output + + def test_audit_help_shows_output_option(self): + result = runner.invoke(app, ["audit", "--help"]) + assert result.exit_code == 0 + assert "--output" in result.output + + +class TestAuditArgumentParsing: + """Tests for command-line argument parsing.""" + + def test_audit_missing_corpus_path(self): + result = runner.invoke(app, ["audit"]) + assert result.exit_code != 0 + + def test_audit_nonexistent_path(self, tmp_path): + missing = tmp_path / "does_not_exist" + result = runner.invoke(app, ["audit", str(missing)]) + assert result.exit_code == 2 + assert "does not exist" in result.output.lower() + + def test_audit_unknown_check_rejected(self, corpus_dir): + result = runner.invoke(app, ["audit", str(corpus_dir), "--checks", "bogus"]) + assert result.exit_code == 2 + assert "unknown check" in result.output.lower() + + def test_audit_partial_unknown_check_rejected(self, corpus_dir): + result = runner.invoke( + app, ["audit", str(corpus_dir), "--checks", "contradiction,bogus"] + ) + assert result.exit_code == 2 + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_valid_checks_parsed(self, mock_auditor_cls, corpus_dir): + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock(return_value=make_report()) + + result = runner.invoke( + app, ["audit", str(corpus_dir), "--checks", "contradiction, staleness"] + ) + + assert result.exit_code == 0 + _, kwargs = mock_auditor_cls.call_args + assert kwargs["checks"] == ["contradiction", "staleness"] + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_default_options(self, mock_auditor_cls, corpus_dir): + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock(return_value=make_report()) + + result = runner.invoke(app, ["audit", str(corpus_dir)]) + + assert result.exit_code == 0 + _, kwargs = mock_auditor_cls.call_args + assert kwargs["checks"] is None + assert kwargs["staleness_days"] == 365 + assert kwargs["similarity_threshold"] == pytest.approx(0.92) + assert kwargs["max_documents"] == 1000 + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_custom_options(self, mock_auditor_cls, corpus_dir): + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock(return_value=make_report()) + + result = runner.invoke( + app, + [ + "audit", + str(corpus_dir), + "--staleness-days", + "30", + "--similarity-threshold", + "0.5", + "--max-documents", + "50", + ], + ) + + assert result.exit_code == 0 + _, kwargs = mock_auditor_cls.call_args + assert kwargs["staleness_days"] == 30 + assert kwargs["similarity_threshold"] == pytest.approx(0.5) + assert kwargs["max_documents"] == 50 + + +class TestAuditIntegration: + """Tests for integration with the corpus audit module.""" + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_calls_auditor_with_corpus_path(self, mock_auditor_cls, corpus_dir): + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock(return_value=make_report()) + + result = runner.invoke(app, ["audit", str(corpus_dir)]) + + assert result.exit_code == 0 + mock_auditor.audit.assert_awaited_once_with(str(corpus_dir)) + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_corpus_not_found_error(self, mock_auditor_cls, corpus_dir): + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock( + side_effect=CorpusNotFoundError("corpus missing") + ) + + result = runner.invoke(app, ["audit", str(corpus_dir)]) + + assert result.exit_code == 2 + assert "error" in result.output.lower() + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_corpus_audit_error(self, mock_auditor_cls, corpus_dir): + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock(side_effect=CorpusAuditError("audit failed")) + + result = runner.invoke(app, ["audit", str(corpus_dir)]) + + assert result.exit_code == 3 + assert "error" in result.output.lower() + + +class TestAuditErrorHandling: + """Tests for error handling on non-existent or invalid corpus paths.""" + + def test_audit_error_message_includes_path(self, tmp_path): + missing = tmp_path / "missing_corpus" + result = runner.invoke(app, ["audit", str(missing)]) + assert str(missing) in result.output + + def test_audit_unknown_check_lists_valid_checks(self, corpus_dir): + result = runner.invoke(app, ["audit", str(corpus_dir), "--checks", "bogus"]) + output_lower = result.output.lower() + assert "contradiction" in output_lower + assert "staleness" in output_lower + assert "duplicate" in output_lower + assert "coverage" in output_lower + + +class TestAuditOutputFormatting: + """Tests for terminal and JSON output formatting.""" + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_terminal_output_healthy(self, mock_auditor_cls, corpus_dir): + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock(return_value=make_report(health_score=0.9)) + + result = runner.invoke(app, ["audit", str(corpus_dir)]) + + assert result.exit_code == 0 + assert "Healthy" in result.output + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_terminal_output_needs_attention(self, mock_auditor_cls, corpus_dir): + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock(return_value=make_report(health_score=0.6)) + + result = runner.invoke(app, ["audit", str(corpus_dir)]) + + assert result.exit_code == 0 + assert "Needs Attention" in result.output + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_terminal_output_unhealthy(self, mock_auditor_cls, corpus_dir): + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock(return_value=make_report(health_score=0.2)) + + result = runner.invoke(app, ["audit", str(corpus_dir)]) + + assert result.exit_code == 0 + assert "Unhealthy" in result.output + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_no_issues_found(self, mock_auditor_cls, corpus_dir): + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock(return_value=make_report(issues=[])) + + result = runner.invoke(app, ["audit", str(corpus_dir)]) + + assert result.exit_code == 0 + assert "No issues found" in result.output + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_issues_listed_in_table(self, mock_auditor_cls, corpus_dir): + issue = make_issue(severity=IssueSeverity.HIGH, title="Duplicate content detected") + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock(return_value=make_report(issues=[issue])) + + result = runner.invoke(app, ["audit", str(corpus_dir)]) + + assert result.exit_code == 0 + assert "Duplicate content detected" in result.output + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_verbose_shows_checks_and_documents(self, mock_auditor_cls, corpus_dir): + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock( + return_value=make_report( + checks_performed=["contradiction", "duplicate"], + total_documents=42, + ) + ) + + result = runner.invoke(app, ["audit", str(corpus_dir), "--verbose"]) + + assert result.exit_code == 0 + assert "contradiction" in result.output.lower() + assert "42" in result.output + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_json_output_structure(self, mock_auditor_cls, corpus_dir): + # 1. Kontekstni to'g'rilash (JSON formatda chiqishini kafolatlash uchun) + from openagent_eval.cli.context import CLIContext, set_context + set_context(CLIContext(json_output=True, quiet=True)) + + issue = make_issue( + severity=IssueSeverity.LOW, + issue_type="staleness", + title="Stale document", + document_ids=["docA"], + ) + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock( + return_value=make_report(health_score=0.75, issues=[issue]) + ) + + # 2. Buyruqni chaqirish (Global flaglarni buyruqdan oldinga qo'yamiz) + result = runner.invoke( + app, ["--quiet", "audit", str(corpus_dir), "--output", "json"] + ) + + # 3. Natijani tekshirish (Agar xato qilsa, nima xatoligini ekranga chiqaradi) + assert result.exit_code == 0, f"Buyruq xato bilan yakunlandi: {result.output}" + + # 4. JSON formatini tekshirish + payload = json.loads(result.output) + assert payload["status"] == "success" + assert payload["health_score"] == pytest.approx(0.75) + assert payload["issues_count"] == 1 + assert payload["issues"][0]["title"] == "Stale document" + assert payload["issues"][0]["type"] == "staleness" + assert "elapsed_seconds" in payload diff --git a/uv.lock b/uv.lock index 33e51a1..7f2605a 100644 --- a/uv.lock +++ b/uv.lock @@ -642,7 +642,7 @@ name = "cuda-bindings" version = "12.9.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/45/e7/b47792cc2d01c7e1d37c32402182524774dadd2d26339bd224e0e913832e/cuda_bindings-12.9.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c912a3d9e6b6651853eed8eed96d6800d69c08e94052c292fec3f282c5a817c9", size = 12210593, upload-time = "2025-10-21T14:51:36.574Z" }, @@ -2262,7 +2262,7 @@ name = "nvidia-cudnn-cu12" version = "9.10.2.21" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cublas-cu12", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, @@ -2273,7 +2273,7 @@ name = "nvidia-cufft-cu12" version = "11.3.3.83" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, @@ -2300,9 +2300,9 @@ name = "nvidia-cusolver-cu12" version = "11.7.3.90" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, - { name = "nvidia-cusparse-cu12" }, - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-cublas-cu12", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cusparse-cu12", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, @@ -2313,7 +2313,7 @@ name = "nvidia-cusparse-cu12" version = "12.5.8.93" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, @@ -2361,7 +2361,7 @@ wheels = [ [[package]] name = "openagent-eval" -version = "0.3.0" +version = "0.4.4" source = { editable = "." } dependencies = [ { name = "httpx" }, @@ -3886,7 +3886,7 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -3971,7 +3971,7 @@ resolution-markers = [ "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ From a92a10b59f87d6a4bedeb18458bd5bca495a16ca Mon Sep 17 00:00:00 2001 From: mahsudbekyuldoshev Date: Thu, 16 Jul 2026 15:09:57 +0500 Subject: [PATCH 2/3] feat: add unit tests for audit command closes #103 --- tests/unit/test_cli/test_audit.py | 144 +++++++++++++++++++----------- 1 file changed, 92 insertions(+), 52 deletions(-) diff --git a/tests/unit/test_cli/test_audit.py b/tests/unit/test_cli/test_audit.py index 1f6a5f8..4e8d3aa 100644 --- a/tests/unit/test_cli/test_audit.py +++ b/tests/unit/test_cli/test_audit.py @@ -3,71 +3,77 @@ from __future__ import annotations import json -from types import SimpleNamespace from unittest.mock import AsyncMock, patch import pytest from typer.testing import CliRunner +from openagent_eval.cli.context import get_context, reset_context from openagent_eval.cli.main import app -from openagent_eval.corpus.models import IssueSeverity -from openagent_eval.exceptions.corpus import CorpusAuditError, CorpusNotFoundError +from openagent_eval.corpus.models import AuditReport, CorpusIssue, IssueSeverity, IssueType +from openagent_eval.exceptions.corpus import ( + CorpusAuditError, + CorpusNotFoundError, + CorpusValidationError, +) runner = CliRunner() -class _IssueTypeStub(str): - """Hashable stand-in for an issue-type enum member. +@pytest.fixture(autouse=True) +def _reset_cli_context(): + """Reset the global CLI context before and after every test. - Behaves like a string for dict-keying/equality purposes but also - exposes ``.value`` the way the real enum does, since the CLI code - reads ``issue.issue_type.value``. + The CLI context (quiet/json/verbose flags) lives in a module-level + global, so leaving it dirty between tests can make later tests pass + or fail depending on run order. """ - - @property - def value(self) -> str: - return str(self) + reset_context() + yield + reset_context() def make_issue( - severity: IssueSeverity = IssueSeverity.MEDIUM, - issue_type: str = "duplicate", - title: str = "Sample issue", - document_ids: list[str] | None = None, - metadata: dict | None = None, - description: str = "A sample issue description.", -) -> SimpleNamespace: - """Build a lightweight stand-in for an audit issue.""" - return SimpleNamespace( + severity: IssueSeverity = IssueSeverity.MEDIUM, + issue_type: IssueType = IssueType.DUPLICATE, + title: str = "Sample issue", + document_ids: list[str] | None = None, + metadata: dict | None = None, + description: str = "A sample issue description.", +) -> CorpusIssue: + """Build a real CorpusIssue for use in test fixtures.""" + return CorpusIssue( + issue_type=issue_type, severity=severity, - issue_type=_IssueTypeStub(issue_type), title=title, + description=description, document_ids=document_ids if document_ids is not None else ["doc1", "doc2"], metadata=metadata if metadata is not None else {}, - description=description, ) def make_report( - health_score: float = 0.95, - summary: str = "Corpus looks healthy.", - issues: list | None = None, - checks_performed: list[str] | None = None, - total_documents: int = 10, -) -> SimpleNamespace: - """Build a lightweight stand-in for an AuditReport.""" - issues = issues if issues is not None else [] - by_type: dict = {} - for issue in issues: - by_type.setdefault(issue.issue_type, []).append(issue) - - return SimpleNamespace( + health_score: float = 0.95, + summary: str = "Corpus looks healthy.", + issues: list[CorpusIssue] | None = None, + checks_performed: list[str] | None = None, + total_documents: int = 10, + corpus_path: str = "corpus", +) -> AuditReport: + """Build a real AuditReport for use in test fixtures. + + Grouping (``issues_by_type``, ``issues_by_severity``, etc.) is a + computed property on the real model, so we don't need to duplicate + that logic here — if the model's grouping logic changes, these + tests will actually notice. + """ + return AuditReport( + corpus_path=corpus_path, + total_documents=total_documents, + issues=issues if issues is not None else [], health_score=health_score, - summary=summary, - issues=issues, checks_performed=checks_performed if checks_performed is not None else ["contradiction", "staleness"], - total_documents=total_documents, - issues_by_type=by_type, + summary=summary, ) @@ -193,7 +199,7 @@ def test_audit_calls_auditor_with_corpus_path(self, mock_auditor_cls, corpus_dir def test_audit_corpus_not_found_error(self, mock_auditor_cls, corpus_dir): mock_auditor = mock_auditor_cls.return_value mock_auditor.audit = AsyncMock( - side_effect=CorpusNotFoundError("corpus missing") + side_effect=CorpusNotFoundError(corpus_path=str(corpus_dir)) ) result = runner.invoke(app, ["audit", str(corpus_dir)]) @@ -204,13 +210,35 @@ def test_audit_corpus_not_found_error(self, mock_auditor_cls, corpus_dir): @patch("openagent_eval.cli.commands.audit.CorpusAuditor") def test_audit_corpus_audit_error(self, mock_auditor_cls, corpus_dir): mock_auditor = mock_auditor_cls.return_value - mock_auditor.audit = AsyncMock(side_effect=CorpusAuditError("audit failed")) + mock_auditor.audit = AsyncMock( + side_effect=CorpusAuditError(message="audit failed", corpus_path=str(corpus_dir)) + ) result = runner.invoke(app, ["audit", str(corpus_dir)]) assert result.exit_code == 3 assert "error" in result.output.lower() + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_empty_corpus_validation_error(self, mock_auditor_cls, corpus_dir): + """CorpusValidationError (raised by the real auditor when no readable + documents are found) is NOT one of the exception types the command + explicitly catches (only CorpusNotFoundError / CorpusAuditError are). + This pins down current behavior so a future fix is a visible diff + here rather than a silent regression either way. + """ + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock( + side_effect=CorpusValidationError( + message="No readable documents found", + corpus_path=str(corpus_dir), + ) + ) + + result = runner.invoke(app, ["audit", str(corpus_dir)]) + + assert result.exit_code != 0 + class TestAuditErrorHandling: """Tests for error handling on non-existent or invalid corpus paths.""" @@ -298,16 +326,16 @@ def test_audit_verbose_shows_checks_and_documents(self, mock_auditor_cls, corpus assert result.exit_code == 0 assert "contradiction" in result.output.lower() assert "42" in result.output + # The command sets ctx.verbose = True when --verbose is passed; + # confirm that actually happens rather than just checking the + # downstream output it enables. + assert get_context().verbose is True @patch("openagent_eval.cli.commands.audit.CorpusAuditor") def test_audit_json_output_structure(self, mock_auditor_cls, corpus_dir): - # 1. Kontekstni to'g'rilash (JSON formatda chiqishini kafolatlash uchun) - from openagent_eval.cli.context import CLIContext, set_context - set_context(CLIContext(json_output=True, quiet=True)) - issue = make_issue( severity=IssueSeverity.LOW, - issue_type="staleness", + issue_type=IssueType.STALENESS, title="Stale document", document_ids=["docA"], ) @@ -316,15 +344,11 @@ def test_audit_json_output_structure(self, mock_auditor_cls, corpus_dir): return_value=make_report(health_score=0.75, issues=[issue]) ) - # 2. Buyruqni chaqirish (Global flaglarni buyruqdan oldinga qo'yamiz) result = runner.invoke( app, ["--quiet", "audit", str(corpus_dir), "--output", "json"] ) - # 3. Natijani tekshirish (Agar xato qilsa, nima xatoligini ekranga chiqaradi) - assert result.exit_code == 0, f"Buyruq xato bilan yakunlandi: {result.output}" - - # 4. JSON formatini tekshirish + assert result.exit_code == 0, f"Command failed: {result.output}" payload = json.loads(result.output) assert payload["status"] == "success" assert payload["health_score"] == pytest.approx(0.75) @@ -332,3 +356,19 @@ def test_audit_json_output_structure(self, mock_auditor_cls, corpus_dir): assert payload["issues"][0]["title"] == "Stale document" assert payload["issues"][0]["type"] == "staleness" assert "elapsed_seconds" in payload + + @patch("openagent_eval.cli.commands.audit.CorpusAuditor") + def test_audit_json_output_without_quiet(self, mock_auditor_cls, corpus_dir): + """Without --quiet, a banner is printed before the JSON payload. + The JSON itself should still be well-formed once you skip past it. + """ + mock_auditor = mock_auditor_cls.return_value + mock_auditor.audit = AsyncMock(return_value=make_report(health_score=0.75)) + + result = runner.invoke(app, ["audit", str(corpus_dir), "--output", "json"]) + + assert result.exit_code == 0 + json_start = result.output.index("{") + payload = json.loads(result.output[json_start:]) + assert payload["status"] == "success" + assert payload["health_score"] == pytest.approx(0.75) \ No newline at end of file From 41e346ef6380d576ec1a423fdd204a4ed7f6b5b2 Mon Sep 17 00:00:00 2001 From: mahsudbekyuldoshev Date: Fri, 17 Jul 2026 15:00:02 +0500 Subject: [PATCH 3/3] feat: add unit tests for audit command closes #103 --- tests/unit/test_cli/test_audit.py | 53 ++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/tests/unit/test_cli/test_audit.py b/tests/unit/test_cli/test_audit.py index 4e8d3aa..39b7058 100644 --- a/tests/unit/test_cli/test_audit.py +++ b/tests/unit/test_cli/test_audit.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re from unittest.mock import AsyncMock, patch import pytest @@ -20,6 +21,19 @@ runner = CliRunner() +def strip_ansi(text: str) -> str: + """Strip ANSI escape codes from text. + + Rich emits color codes even when CliRunner's output isn't a real + terminal, which can split option names like ``--checks`` across + escape sequences (``-`` + reset + ``-checks``). Assertions on raw + CLI output should strip these first, matching the helper already + used in test_cli_improvements.py. + """ + ansi_escape = re.compile(r'\x1b\[[0-9;]*m') + return ansi_escape.sub('', text) + + @pytest.fixture(autouse=True) def _reset_cli_context(): """Reset the global CLI context before and after every test. @@ -34,12 +48,12 @@ def _reset_cli_context(): def make_issue( - severity: IssueSeverity = IssueSeverity.MEDIUM, - issue_type: IssueType = IssueType.DUPLICATE, - title: str = "Sample issue", - document_ids: list[str] | None = None, - metadata: dict | None = None, - description: str = "A sample issue description.", + severity: IssueSeverity = IssueSeverity.MEDIUM, + issue_type: IssueType = IssueType.DUPLICATE, + title: str = "Sample issue", + document_ids: list[str] | None = None, + metadata: dict | None = None, + description: str = "A sample issue description.", ) -> CorpusIssue: """Build a real CorpusIssue for use in test fixtures.""" return CorpusIssue( @@ -53,12 +67,12 @@ def make_issue( def make_report( - health_score: float = 0.95, - summary: str = "Corpus looks healthy.", - issues: list[CorpusIssue] | None = None, - checks_performed: list[str] | None = None, - total_documents: int = 10, - corpus_path: str = "corpus", + health_score: float = 0.95, + summary: str = "Corpus looks healthy.", + issues: list[CorpusIssue] | None = None, + checks_performed: list[str] | None = None, + total_documents: int = 10, + corpus_path: str = "corpus", ) -> AuditReport: """Build a real AuditReport for use in test fixtures. @@ -97,12 +111,12 @@ def test_audit_help(self): def test_audit_help_shows_checks_option(self): result = runner.invoke(app, ["audit", "--help"]) assert result.exit_code == 0 - assert "--checks" in result.output + assert "--checks" in strip_ansi(result.output) def test_audit_help_shows_output_option(self): result = runner.invoke(app, ["audit", "--help"]) assert result.exit_code == 0 - assert "--output" in result.output + assert "--output" in strip_ansi(result.output) class TestAuditArgumentParsing: @@ -246,7 +260,7 @@ class TestAuditErrorHandling: def test_audit_error_message_includes_path(self, tmp_path): missing = tmp_path / "missing_corpus" result = runner.invoke(app, ["audit", str(missing)]) - assert str(missing) in result.output + assert str(missing) in strip_ansi(result.output) def test_audit_unknown_check_lists_valid_checks(self, corpus_dir): result = runner.invoke(app, ["audit", str(corpus_dir), "--checks", "bogus"]) @@ -349,7 +363,7 @@ def test_audit_json_output_structure(self, mock_auditor_cls, corpus_dir): ) assert result.exit_code == 0, f"Command failed: {result.output}" - payload = json.loads(result.output) + payload = json.loads(strip_ansi(result.output)) assert payload["status"] == "success" assert payload["health_score"] == pytest.approx(0.75) assert payload["issues_count"] == 1 @@ -368,7 +382,8 @@ def test_audit_json_output_without_quiet(self, mock_auditor_cls, corpus_dir): result = runner.invoke(app, ["audit", str(corpus_dir), "--output", "json"]) assert result.exit_code == 0 - json_start = result.output.index("{") - payload = json.loads(result.output[json_start:]) + clean_output = strip_ansi(result.output) + json_start = clean_output.index("{") + payload = json.loads(clean_output[json_start:]) assert payload["status"] == "success" - assert payload["health_score"] == pytest.approx(0.75) \ No newline at end of file + assert payload["health_score"] == pytest.approx(0.75)