diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6a3d8b..8f9ec28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,6 @@ jobs: run: | python -m pip install --upgrade pip pip install -e .[dev] - pip install pytest ruff mypy - name: Run linter run: ruff check . - name: Run type checks @@ -25,3 +24,26 @@ jobs: PYTHONPATH=src mypy --ignore-missing-imports src tests - name: Run tests run: pytest -q + + # Build with coverage for PRs only + build-with-coverage: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[dev] + - name: Run tests with coverage + run: | + PYTHONPATH=src python -m pytest --cov=src --cov-report=term-missing --cov-report=html --cov-fail-under=39 + - name: Upload coverage reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: coverage_html_report/ diff --git a/pyproject.toml b/pyproject.toml index 9e84e7e..3c7c460 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,13 +14,16 @@ dependencies = [ "anthropic>=0.7.0", "openai>=1.0.0", "google-generativeai>=0.3.0", - "types-requests>=2.32", ] [project.optional-dependencies] dev = [ + "pytest>=7.0", "pytest-cov>=5.0", - "pre-commit>=3.0", + "ruff>=0.3.0", + "mypy>=1.0", + "types-requests>=2.32", + "pre-commit>=3.0" ] [project.scripts] @@ -33,3 +36,38 @@ packages = ["src"] [tool.ruff] line-length = 120 target-version = "py311" + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] +addopts = "--strict-markers --strict-config" + +[tool.coverage.run] +source = ["src"] +branch = true +omit = [ + "*/tests/*", + "*/test_*", + "*/__pycache__/*", + "*/conftest.py", + "*/cli/*", # Exclude CLI tools + "*/bench/*", # Exclude benchmark tools + "*/validators/*", # Exclude validators (placeholder code) +] + +[tool.coverage.report] +fail_under = 39 +show_missing = true +skip_covered = false +precision = 2 +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "@abstract", +] + +[tool.coverage.html] +directory = "coverage_html_report" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a68f158..0000000 --- a/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Core requirements -pydantic>=2.0.0 -types-requests>=2.32 - - -# Models: -anthropic>=0.7.0 -openai>=1.0.0 -google-generativeai>=0.3.0 - -# Benchmarks: -swebench>=1.7.0 diff --git a/tests/agentNodes/__init__.py b/tests/agentNodes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/agentNodes/test_hld_designer.py b/tests/agentNodes/test_hld_designer.py index 67c8627..04bb295 100644 --- a/tests/agentNodes/test_hld_designer.py +++ b/tests/agentNodes/test_hld_designer.py @@ -1,13 +1,13 @@ -from agentNodes.hld_designer import HLDDesigner +from src.agentNodes.hld_designer import HLDDesigner from pydantic import TypeAdapter -from dataModel.task import Task, TaskType -from dataModel.model_response import ( +from src.dataModel.task import Task, TaskType +from src.dataModel.model_response import ( DecomposedResponse, ImplementedResponse, - DesignerResponse, + ModelResponse, ) -from modelAccessors.base_accessor import BaseModelAccessor +from src.modelAccessors.base_accessor import BaseModelAccessor class _StubAccessor(BaseModelAccessor): @@ -18,12 +18,12 @@ def call_model( self, prompt: str, *, - adapter: TypeAdapter[DesignerResponse], + adapter: TypeAdapter[ModelResponse], schema: dict, model: str = "gpt-4", system_prompt: str = "", tools=None, - ) -> DesignerResponse: + ) -> ModelResponse: return self._result diff --git a/tests/agentNodes/test_implementer.py b/tests/agentNodes/test_implementer.py index dc8e5a0..11ec557 100644 --- a/tests/agentNodes/test_implementer.py +++ b/tests/agentNodes/test_implementer.py @@ -1,9 +1,9 @@ from pydantic import TypeAdapter -from agentNodes.implementer import Implementer -from modelAccessors.base_accessor import BaseModelAccessor -from dataModel.model_response import ImplementedResponse -from dataModel.task import Task, TaskType +from src.agentNodes.implementer import Implementer +from src.modelAccessors.base_accessor import BaseModelAccessor +from src.dataModel.model_response import ImplementedResponse, ModelResponse +from src.dataModel.task import Task, TaskType class _StubAccessor(BaseModelAccessor): @@ -14,12 +14,12 @@ def call_model( self, prompt: str, *, - adapter: TypeAdapter[ImplementedResponse], + adapter: TypeAdapter[ModelResponse], schema: dict, model: str = "gpt-4", system_prompt: str = "", tools=None, - ) -> ImplementedResponse: + ) -> ModelResponse: return self._result diff --git a/tests/agentNodes/test_lld_designer.py b/tests/agentNodes/test_lld_designer.py index d8dfa7f..da812f6 100644 --- a/tests/agentNodes/test_lld_designer.py +++ b/tests/agentNodes/test_lld_designer.py @@ -1,10 +1,10 @@ import pytest from pydantic import TypeAdapter, ValidationError -from agentNodes.lld_designer import LLDDesigner -from dataModel.model_response import ImplementedResponse -from dataModel.task import Task, TaskType -from modelAccessors.base_accessor import BaseModelAccessor +from src.agentNodes.lld_designer import LLDDesigner +from src.dataModel.model_response import ImplementedResponse, ModelResponse +from src.dataModel.task import Task, TaskType +from src.modelAccessors.base_accessor import BaseModelAccessor class _StubAccessor(BaseModelAccessor): @@ -15,12 +15,12 @@ def call_model( self, prompt: str, *, - adapter: TypeAdapter[ImplementedResponse], + adapter: TypeAdapter[ModelResponse], schema: dict, model: str = "gpt-4", system_prompt: str = "", tools=None, - ) -> ImplementedResponse: + ) -> ModelResponse: return self._result diff --git a/tests/agentNodes/test_researcher.py b/tests/agentNodes/test_researcher.py index 267397e..65d644b 100644 --- a/tests/agentNodes/test_researcher.py +++ b/tests/agentNodes/test_researcher.py @@ -1,11 +1,11 @@ import pytest from pydantic import TypeAdapter, ValidationError -from agentNodes.researcher import Researcher -from tools.web_search import WEB_SEARCH_TOOL -from dataModel.model_response import ImplementedResponse -from dataModel.task import Task, TaskType -from modelAccessors.base_accessor import BaseModelAccessor +from src.agentNodes.researcher import Researcher +from src.tools.web_search import WEB_SEARCH_TOOL +from src.dataModel.model_response import ImplementedResponse, ModelResponse +from src.dataModel.task import Task, TaskType +from src.modelAccessors.base_accessor import BaseModelAccessor class _StubAccessor(BaseModelAccessor): @@ -16,12 +16,12 @@ def call_model( self, prompt: str, *, - adapter: TypeAdapter[ImplementedResponse], + adapter: TypeAdapter[ModelResponse], schema: dict, model: str = "gpt-4", system_prompt: str = "", tools=None, - ) -> ImplementedResponse: + ) -> ModelResponse: return self._result diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/dataBuilders/__init__.py b/tests/dataBuilders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/dataManagement/__init__.py b/tests/dataManagement/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_project_manager.py b/tests/dataManagement/test_project_manager.py similarity index 100% rename from tests/test_project_manager.py rename to tests/dataManagement/test_project_manager.py diff --git a/tests/dataModel/__init__.py b/tests/dataModel/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_models.py b/tests/dataModel/test_models.py similarity index 100% rename from tests/test_models.py rename to tests/dataModel/test_models.py diff --git a/tests/dataModel/test_validation_result.py b/tests/dataModel/test_validation_result.py new file mode 100644 index 0000000..77af7c3 --- /dev/null +++ b/tests/dataModel/test_validation_result.py @@ -0,0 +1,51 @@ +import pytest + +from src.dataModel.validation_result import ValidationResult + + +def test_validation_result_valid_with_no_errors(): + """Test creating a valid ValidationResult with no errors.""" + result = ValidationResult(is_valid=True, errors=None) + assert result.is_valid is True + assert result.errors is None + + +def test_validation_result_valid_with_empty_errors(): + """Test creating a valid ValidationResult with empty errors list.""" + result = ValidationResult(is_valid=True, errors=[]) + assert result.is_valid is True + assert result.errors == [] + + +def test_validation_result_invalid_with_errors(): + """Test creating an invalid ValidationResult with errors.""" + errors = ["Error 1", "Error 2"] + result = ValidationResult(is_valid=False, errors=errors) + assert result.is_valid is False + assert result.errors == errors + + +def test_validation_result_invalid_with_single_error(): + """Test creating an invalid ValidationResult with a single error.""" + error = ["Single error"] + result = ValidationResult(is_valid=False, errors=error) + assert result.is_valid is False + assert result.errors == error + + +def test_validation_result_invalid_valid_true_with_errors_raises_error(): + """Test that ValidationResult raises error when is_valid=True but errors are present.""" + with pytest.raises(ValueError, match="Cannot have both is_valid=True and an error_message"): + ValidationResult(is_valid=True, errors=["Some error"]) + + +def test_validation_result_invalid_valid_false_with_no_errors_raises_error(): + """Test that ValidationResult raises error when is_valid=False but no errors provided.""" + with pytest.raises(ValueError, match="Must provide an error_message when is_valid=False"): + ValidationResult(is_valid=False, errors=None) + + +def test_validation_result_invalid_valid_false_with_empty_errors_raises_error(): + """Test that ValidationResult raises error when is_valid=False but empty errors provided.""" + with pytest.raises(ValueError, match="Must provide an error_message when is_valid=False"): + ValidationResult(is_valid=False, errors=[]) \ No newline at end of file diff --git a/tests/modelAccessors/__init__.py b/tests/modelAccessors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/modelAccessors/test_anthropic_accessor.py b/tests/modelAccessors/test_anthropic_accessor.py new file mode 100644 index 0000000..bc010b6 --- /dev/null +++ b/tests/modelAccessors/test_anthropic_accessor.py @@ -0,0 +1,151 @@ +import pytest +from unittest.mock import Mock, patch +from pydantic import TypeAdapter + +from src.modelAccessors.anthropic_accessor import AnthropicAccessor +from src.dataModel.model_response import ImplementedResponse + + +@patch('src.modelAccessors.anthropic_accessor.Anthropic') +@patch('src.modelAccessors.anthropic_accessor.environ.get') +def test_anthropic_accessor_init(mock_env_get, mock_anthropic): + """Test AnthropicAccessor initialization.""" + mock_env_get.return_value = "test_api_key" + mock_client = Mock() + mock_anthropic.return_value = mock_client + + accessor = AnthropicAccessor() + + mock_env_get.assert_called_once_with("ANTHROPIC_API_KEY") + mock_anthropic.assert_called_once_with(api_key="test_api_key") + assert accessor.client == mock_client + assert accessor.tool_supported_models == [ + "claude-3-opus-20240229", + "claude-3-sonnet-20240229", + "claude-3-5-sonnet-20240620" + ] + + +@patch('src.modelAccessors.anthropic_accessor.Anthropic') +@patch('src.modelAccessors.anthropic_accessor.environ.get') +def test_call_model_without_tools(mock_env_get, mock_anthropic): + """Test calling model without tools.""" + mock_env_get.return_value = "test_api_key" + + # Mock client and response + mock_client = Mock() + mock_anthropic.return_value = mock_client + + mock_response = Mock() + mock_content = Mock() + mock_content.text = '{"content": "test response", "artifacts": []}' + mock_response.content = [mock_content] + mock_client.messages.create.return_value = mock_response + + accessor = AnthropicAccessor() + adapter = TypeAdapter(ImplementedResponse) + + result = accessor.call_model( + "Test prompt", + adapter=adapter, + schema={"type": "object"}, + model="claude-3-sonnet-20240229" + ) + + assert isinstance(result, ImplementedResponse) + assert result.content == "test response" + assert result.artifacts == [] + + +@patch('src.modelAccessors.anthropic_accessor.Anthropic') +@patch('src.modelAccessors.anthropic_accessor.environ.get') +def test_call_model_with_tools(mock_env_get, mock_anthropic): + """Test calling model with tools.""" + mock_env_get.return_value = "test_api_key" + + # Mock client and response + mock_client = Mock() + mock_anthropic.return_value = mock_client + + mock_response = Mock() + mock_content = Mock() + mock_content.text = '{"content": "test response with tools", "artifacts": ["tool_result"]}' + mock_response.content = [mock_content] + mock_client.messages.create.return_value = mock_response + + accessor = AnthropicAccessor() + adapter = TypeAdapter(ImplementedResponse) + + mock_tool = Mock() + mock_tool.to_anthropic_tool.return_value = {"name": "test_tool", "input_schema": {}} + + result = accessor.call_model( + "Test prompt", + adapter=adapter, + schema={"type": "object"}, + model="claude-3-sonnet-20240229", + tools=[mock_tool] + ) + + assert isinstance(result, ImplementedResponse) + assert result.content == "test response with tools" + assert result.artifacts == ["tool_result"] + + +@patch('src.modelAccessors.anthropic_accessor.Anthropic') +@patch('src.modelAccessors.anthropic_accessor.environ.get') +def test_call_model_uses_default_model(mock_env_get, mock_anthropic): + """Test that call_model uses default model when none specified.""" + mock_env_get.return_value = "test_api_key" + + # Mock client and response + mock_client = Mock() + mock_anthropic.return_value = mock_client + + mock_response = Mock() + mock_content = Mock() + mock_content.text = '{"content": "test", "artifacts": []}' + mock_response.content = [mock_content] + mock_client.messages.create.return_value = mock_response + + accessor = AnthropicAccessor() + adapter = TypeAdapter(ImplementedResponse) + + accessor.call_model( + "Test prompt", + adapter=adapter, + schema={"type": "object"} + ) + + # Should call messages.create with default model + mock_client.messages.create.assert_called_once() + call_args = mock_client.messages.create.call_args + assert call_args[1]['model'] == "claude-3-opus-20240229" # First in tool_supported_models + + +@patch('src.modelAccessors.anthropic_accessor.Anthropic') +@patch('src.modelAccessors.anthropic_accessor.environ.get') +def test_call_model_handles_json_parsing_error(mock_env_get, mock_anthropic): + """Test that call_model handles JSON parsing errors gracefully.""" + mock_env_get.return_value = "test_api_key" + + # Mock client and response with invalid JSON + mock_client = Mock() + mock_anthropic.return_value = mock_client + + mock_response = Mock() + mock_content = Mock() + mock_content.text = 'invalid json response' + mock_response.content = [mock_content] + mock_client.messages.create.return_value = mock_response + + accessor = AnthropicAccessor() + adapter = TypeAdapter(ImplementedResponse) + + # Should handle the JSON parsing error appropriately + with pytest.raises(Exception): # The exact exception depends on implementation + accessor.call_model( + "Test prompt", + adapter=adapter, + schema={"type": "object"} + ) \ No newline at end of file diff --git a/tests/modelAccessors/test_gemini_accessor.py b/tests/modelAccessors/test_gemini_accessor.py new file mode 100644 index 0000000..b07e51e --- /dev/null +++ b/tests/modelAccessors/test_gemini_accessor.py @@ -0,0 +1,111 @@ +from unittest.mock import Mock, patch +from pydantic import TypeAdapter + +from src.modelAccessors.gemini_accessor import GeminiAccessor +from src.dataModel.model_response import ImplementedResponse + + +class MockModel: + def __init__(self, response_text): + self.response_text = response_text + + def generate_content(self, prompt, generation_config=None, tools=None): + mock_response = Mock() + mock_response.text = self.response_text + mock_response.parts = [Mock()] + mock_response.parts[0].function_call = None + return mock_response + + +@patch('src.modelAccessors.gemini_accessor.genai.configure') +@patch('src.modelAccessors.gemini_accessor.environ.get') +def test_gemini_accessor_init(mock_env_get, mock_configure): + """Test GeminiAccessor initialization.""" + mock_env_get.return_value = "test_api_key" + + accessor = GeminiAccessor() + + mock_env_get.assert_called_once_with("GOOGLE_API_KEY") + mock_configure.assert_called_once_with(api_key="test_api_key") + assert accessor.tool_supported_models == ["gemini-1.5-pro", "gemini-1.5-flash", "gemini-pro"] + + +@patch('src.modelAccessors.gemini_accessor.genai.configure') +@patch('src.modelAccessors.gemini_accessor.environ.get') +@patch('src.modelAccessors.gemini_accessor.genai.GenerativeModel') +def test_call_model_without_tools(mock_generative_model, mock_env_get, mock_configure): + """Test calling model without tools.""" + mock_env_get.return_value = "test_api_key" + + # Mock response + response_text = '{"content": "test response", "artifacts": []}' + mock_model_instance = MockModel(response_text) + mock_generative_model.return_value = mock_model_instance + + accessor = GeminiAccessor() + adapter = TypeAdapter(ImplementedResponse) + + result = accessor.call_model( + "Test prompt", + adapter=adapter, + schema={"type": "object"}, + model="gemini-1.5-pro" + ) + + assert isinstance(result, ImplementedResponse) + assert result.content == "test response" + assert result.artifacts == [] + + +@patch('src.modelAccessors.gemini_accessor.genai.configure') +@patch('src.modelAccessors.gemini_accessor.environ.get') +@patch('src.modelAccessors.gemini_accessor.genai.GenerativeModel') +def test_call_model_with_tools(mock_generative_model, mock_env_get, mock_configure): + """Test calling model with tools.""" + mock_env_get.return_value = "test_api_key" + + # Mock response + response_text = '{"content": "test response with tools", "artifacts": ["tool_result"]}' + mock_model_instance = MockModel(response_text) + mock_generative_model.return_value = mock_model_instance + + accessor = GeminiAccessor() + adapter = TypeAdapter(ImplementedResponse) + + mock_tool = Mock() + mock_tool.to_gemini_tool.return_value = {"function": {"name": "test_tool"}} + + result = accessor.call_model( + "Test prompt", + adapter=adapter, + schema={"type": "object"}, + model="gemini-1.5-pro", + tools=[mock_tool] + ) + + assert isinstance(result, ImplementedResponse) + assert result.content == "test response with tools" + assert result.artifacts == ["tool_result"] + + +@patch('src.modelAccessors.gemini_accessor.genai.configure') +@patch('src.modelAccessors.gemini_accessor.environ.get') +def test_call_model_uses_default_model(mock_env_get, mock_configure): + """Test that call_model uses default model when none specified.""" + mock_env_get.return_value = "test_api_key" + + with patch('src.modelAccessors.gemini_accessor.genai.GenerativeModel') as mock_gen_model: + mock_model_instance = MockModel('{"content": "test", "artifacts": []}') + mock_gen_model.return_value = mock_model_instance + + accessor = GeminiAccessor() + adapter = TypeAdapter(ImplementedResponse) + + accessor.call_model( + "Test prompt", + adapter=adapter, + schema={"type": "object"} + ) + + # Should use default model (first in tool_supported_models) + mock_gen_model.assert_called_with("gemini-1.5-pro") \ No newline at end of file diff --git a/tests/test_mock_accessor.py b/tests/modelAccessors/test_mock_accessor.py similarity index 100% rename from tests/test_mock_accessor.py rename to tests/modelAccessors/test_mock_accessor.py diff --git a/tests/test_openai_accessor.py b/tests/modelAccessors/test_openai_accessor.py similarity index 100% rename from tests/test_openai_accessor.py rename to tests/modelAccessors/test_openai_accessor.py diff --git a/tests/orchestrator/__init__.py b/tests/orchestrator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_agent_orchestrator.py b/tests/orchestrator/test_agent_orchestrator.py similarity index 100% rename from tests/test_agent_orchestrator.py rename to tests/orchestrator/test_agent_orchestrator.py diff --git a/tests/test_nodes_e2e.py b/tests/orchestrator/test_nodes_e2e.py similarity index 100% rename from tests/test_nodes_e2e.py rename to tests/orchestrator/test_nodes_e2e.py diff --git a/tests/search/__init__.py b/tests/search/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_file_loader.py b/tests/search/test_file_loader.py similarity index 100% rename from tests/test_file_loader.py rename to tests/search/test_file_loader.py diff --git a/tests/tools/__init__.py b/tests/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/tools/test_env_tools.py b/tests/tools/test_env_tools.py index 9dcd9ea..74cff62 100644 --- a/tests/tools/test_env_tools.py +++ b/tests/tools/test_env_tools.py @@ -1,6 +1,6 @@ import subprocess -from tools.env_tools import EnvManager +from src.tools.env_tools import EnvManager def test_npm_install_runs_command(monkeypatch): diff --git a/tests/tools/test_file_io.py b/tests/tools/test_file_io.py index 0c13e2a..4ca39f8 100644 --- a/tests/tools/test_file_io.py +++ b/tests/tools/test_file_io.py @@ -5,7 +5,7 @@ import time from contextlib import contextmanager -import tools.file_io as file_io +import src.tools.file_io as file_io def test_read_file_returns_contents(monkeypatch): def fake_open(path, mode="r", encoding=None): diff --git a/tests/tools/test_web_search.py b/tests/tools/test_web_search.py index d24eac7..d9520a2 100644 --- a/tests/tools/test_web_search.py +++ b/tests/tools/test_web_search.py @@ -1,6 +1,6 @@ import requests -from tools.web_search import web_search +from src.tools.web_search import web_search class _Resp: diff --git a/tests/treeagent/__init__.py b/tests/treeagent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/validators/__init__.py b/tests/validators/__init__.py new file mode 100644 index 0000000..e69de29