Unit testing improvements

.github/workflows/functional_test.yml

@@ -27,5 +27,4 @@ jobs:
       - name: Run Tests
         run: |
           uv run --python ${{ matrix.env }} pytest \
-            tests/test_classes \
             --color=yes

collectoss/tasks/github/__init__.py

@@ -1,7 +0,0 @@
-from collectoss.tasks.github.contributors import *
-from collectoss.tasks.github.events import *
-from collectoss.tasks.github.issues import *
-from collectoss.tasks.github.messages import *
-from collectoss.tasks.github.pull_requests.tasks import *
-from collectoss.tasks.github.repo_info.tasks import *
-from collectoss.tasks.github.releases.tasks import *

collectoss/tasks/start_tasks.py

@@ -7,7 +7,13 @@
 import sqlalchemy as s
 
 
-from collectoss.tasks.github import *
+from collectoss.tasks.github.contributors import *
+from collectoss.tasks.github.events import *
+from collectoss.tasks.github.issues import *
+from collectoss.tasks.github.messages import *
+from collectoss.tasks.github.pull_requests.tasks import *
+from collectoss.tasks.github.repo_info.tasks import *
+from collectoss.tasks.github.releases.tasks import *
 if os.environ.get('AUGUR_DOCKER_DEPLOY') != "1":
     from collectoss.tasks.data_analysis import *
 from collectoss.tasks.github.detect_move.tasks import detect_github_repo_move_core, detect_github_repo_move_secondary

pyproject.toml

@@ -146,14 +146,21 @@ addopts = "-ra -s"
 testpaths = [
     "tests/test_classes",
     "tests/test_application/test_cli/test_csv_utils.py",
-    # "tests/test_routes", # runs, but needs a fixture for connecting to the web interface of CollectOSS
+    "tests/test_tasks/test_task_utilities/test_util/test_worker_util.py",
+    "tests/test_tasks/test_task_utilities/test_util/test_contributor_uuid.py",
+    # "tests/test_routes", # runs, but needs a fixture for connecting to the web interface of Augur
     # "tests/test_metrics",
     # "tests/test_tasks",
     # "tests/test_application",
     # "tests/test_workers",
     # "tests/test_workers/worker_persistence/",
     # "tests/test_routes/runner.py"
 ]
+markers = [
+    "unit: pure logic tests with no external dependencies",
+    "integration: tests requiring a database, Redis, or network access",
+]
+
 
 [tool.mypy]
 files = ['collectoss/application/db/*.py']

tests/test_application/test_cli/test_csv_utils.py

@@ -14,7 +14,7 @@
     MAX_FILE_SIZE_BYTES,
 )
 
-
+@pytest.mark.unit
 class TestValidateGitUrl:
     """Tests for validate_git_url function"""
 
@@ -40,7 +40,7 @@ def test_whitespace_handling(self):
         """Test that whitespace is properly stripped"""
         assert validate_git_url("  https://github.com/chaoss/collectoss  ")
 
-
+@pytest.mark.unit
 class TestValidatePositiveInt:
     """Tests for validate_positive_int function"""
 
@@ -71,7 +71,7 @@ def test_whitespace_handling(self):
         """Test that whitespace is properly stripped"""
         assert validate_positive_int("  42  ")
 
-
+@pytest.mark.unit
 class TestDetectColumnOrder:
     """Tests for detect_column_order function"""
 
@@ -153,7 +153,7 @@ def test_no_match_found_raises_error(self):
         with pytest.raises(ValueError, match="Could not detect column"):
             detect_column_order(sample_rows, validators)
 
-
+@pytest.mark.unit
 class TestProcessCsv:
     """Tests for process_csv function"""
 
@@ -252,7 +252,7 @@ def test_whitespace_in_values(self, tmp_path):
         result = process_csv(str(csv_file), validators)
         assert result[0] == {"repo_url": "https://github.com/chaoss/collectoss", "repo_group_id": "10"}
 
-
+@pytest.mark.unit
 class TestProcessRepoCsv:
     """Tests for process_repo_csv function"""
 
@@ -275,6 +275,7 @@ def test_process_repo_csv_without_headers(self, tmp_path):
         assert len(result) == 2
 
 
+@pytest.mark.unit
 class TestProcessRepoGroupCsv:
     """Tests for process_repo_group_csv function"""
 
@@ -310,6 +311,7 @@ def test_empty_group_name_invalid(self, tmp_path):
         assert len(result) >= 1
 
 
+@pytest.mark.unit
 class TestEdgeCases:
     """Tests for edge cases and error conditions"""
 

tests/test_classes/test_config_stores.py

@@ -14,75 +14,128 @@
     return Mock()
 
 
-def test_jsonconfig_readonly_flags(mock_logger):
-    cfg = JsonConfig({"A": {"x": 1}}, mock_logger)
-    assert cfg.writable is False
-    assert cfg.empty is False
-
-
-def test_jsonconfig_empty_true_false(mock_logger):
-    assert JsonConfig({}, mock_logger).empty is True
-    assert JsonConfig({"A": {}}, mock_logger).empty is False
-
-
-def test_jsonconfig_write_protection(mock_logger):
-    # JsonConfig should be not writeable by default, so we should be unable to change
-    # its values, even by abusing references
-
-    data = {"Alpha": {"a": 1, "b": "str"}, "Beta": {}}
-    cfg = JsonConfig(data, mock_logger)
-
-    # mutation via input
-    data["Alpha"]["a"] = 2
-
-    config_test = cfg.retrieve_dict() 
-    assert config_test != data # the data in the config should not change
-
-    # mutation via output
-    config_test["Alpha"]["a"] = 3
-
-    config_test = cfg.retrieve_dict() 
-    assert config_test != data # the data in the config should not change
-
-def test_jsonconfig_retrieve_has_get(mock_logger):
-    data = {"Alpha": {"a": 1, "b": "str"}, "Beta": {}}
-    cfg = JsonConfig(data, mock_logger)
-
-    # retrieve full dict
-    assert cfg.retrieve_dict() == data
-
-    # has/get section
-    assert cfg.has_section("Alpha") is True
-    assert cfg.has_section("Missing") is False
-    assert cfg.get_section("Alpha") == {"a": 1, "b": "str"}
-    assert cfg.get_section("Missing") is None
-
-    # has/get value
-    assert cfg.has_value("Alpha", "a") is True
-    assert cfg.has_value("Alpha", "missing") is False
-    assert cfg.has_value("Missing", "a") is False
-    assert cfg.get_value("Alpha", "a") == 1
-    assert cfg.get_value("Alpha", "missing") is None
-    assert cfg.get_value("Missing", "a") is None
-
-
-@pytest.mark.parametrize(
-    "callable_name, args, kwargs",
-    [
-        ("load_dict", ({"X": {"y": 2}},), {"ignore_existing": False}),
-        ("clear", tuple(), {}),
-        ("remove_section", ("X",), {}),
-        ("create_section", ("X", {"y": 2}), {"ignore_existing": False}),
-        ("remove_value", ("X", "y"), {}),
-        ("add_value", ("X", "y", 2), {"ignore_existing": False}),
-    ],
-)
-def test_jsonconfig_mutations_raise_not_writable(mock_logger, callable_name, args, kwargs):
-    cfg = JsonConfig({"A": {"x": 1}}, mock_logger)
-    with pytest.raises(NotWriteableException):
-        getattr(cfg, callable_name)(*args, **kwargs)
+class TestJSONConfig:
 
+    def test_jsonconfig_readonly_flags(self, mock_logger):
+        cfg = JsonConfig({"A": {"x": 1}}, mock_logger)
+        assert cfg.writable is False
+        assert cfg.empty is False
 
+
+    def test_jsonconfig_empty_true_false(self, mock_logger):
+        assert JsonConfig({}, mock_logger).empty is True
+        assert JsonConfig({"A": {}}, mock_logger).empty is False
+
+
+    def test_jsonconfig_write_protection(self, mock_logger):
+        # JsonConfig should be not writeable by default, so we should be unable to change
+        # its values, even by abusing references
+
+        data = {"Alpha": {"a": 1, "b": "str"}, "Beta": {}}
+        cfg = JsonConfig(data, mock_logger)
+
+        # mutation via input
+        data["Alpha"]["a"] = 2
+
+        config_test = cfg.retrieve_dict() 
+        assert config_test != data # the data in the config should not change
+
+        # mutation via output
+        config_test["Alpha"]["a"] = 3
+
+        config_test = cfg.retrieve_dict() 
+        assert config_test != data # the data in the config should not change
+
+    def test_jsonconfig_retrieve_has_get(self, mock_logger):
+        data = {"Alpha": {"a": 1, "b": "str"}, "Beta": {}}
+        cfg = JsonConfig(data, mock_logger)
+
+        # retrieve full dict
+        assert cfg.retrieve_dict() == data
+
+        # has/get section
+        assert cfg.has_section("Alpha") is True
+        assert cfg.has_section("Missing") is False
+        assert cfg.get_section("Alpha") == {"a": 1, "b": "str"}
+        assert cfg.get_section("Missing") is None
+
+        # has/get value
+        assert cfg.has_value("Alpha", "a") is True
+        assert cfg.has_value("Alpha", "missing") is False
+        assert cfg.has_value("Missing", "a") is False
+        assert cfg.get_value("Alpha", "a") == 1
+        assert cfg.get_value("Alpha", "missing") is None
+        assert cfg.get_value("Missing", "a") is None
+
+
+    @pytest.mark.parametrize(
+        "callable_name, args, kwargs",
+        [
+            ("load_dict", ({"X": {"y": 2}},), {"ignore_existing": False}),
+            ("clear", tuple(), {}),
+            ("remove_section", ("X",), {}),
+            ("create_section", ("X", {"y": 2}), {"ignore_existing": False}),
+            ("remove_value", ("X", "y"), {}),
+            ("add_value", ("X", "y", 2), {"ignore_existing": False}),
+        ],
+    )
+    def test_jsonconfig_mutations_raise_not_writable(self, mock_logger, callable_name, args, kwargs):
+        cfg = JsonConfig({"A": {"x": 1}}, mock_logger)
+        with pytest.raises(NotWriteableException):
+            getattr(cfg, callable_name)(*args, **kwargs)
+
+
+    def test_fetching_real_defaults(self, mock_logger, mock_session):
+        cfg = SystemConfig(mock_logger, mock_session)
+        cfg.config_sources = [JsonConfig(default_config, mock_logger)]
+
+        assert cfg.get_value("Redis", "cache_group") == 0
+
+
+    def test_load_config_utilizes_hierarchy(self):
+
+        default_dict = {
+            "Section1": {"alpha": 1, "beta": "x"},
+            "Section2": {"gamma": False, "delta": 3.14},
+        }
+
+        override_dict = {
+            "Section1": {"beta": "y"},
+            "Section2": {"Epsilon": True, "delta": 6.28},
+            "Section3": {"hi": "there"}
+        }
+
+        cfg = SystemConfig(None, None, [JsonConfig(default_dict, mock_logger), JsonConfig(override_dict, mock_logger)])
+
+        expected_dict = {
+            "Section1": {"alpha": 1, "beta": "y"},
+            "Section2": {"gamma": False, "Epsilon": True, "delta": 6.28},
+            "Section3": {"hi": "there"} # test that new sections are accounted for too
+        }
+
+        assert cfg.load_config() == expected_dict
+
+
+    def test_get_section_incorporates_hierarchy(self):
+
+        default_dict = {
+            "Section1": {"alpha": 1, "beta": "x"},
+            "Section2": {"gamma": False, "delta": 3.14},
+        }
+
+        override_dict = {
+            "Section1": {"beta": "y"},
+            "Section2": {"gamma": False, "delta": 3.14},
+        }
+
+        cfg = SystemConfig(None, None, [JsonConfig(default_dict, mock_logger), JsonConfig(override_dict, mock_logger)])
+
+        expected_dict = {"alpha": 1, "beta": "y"}
+
+        assert cfg.get_section("Section1") == expected_dict
+
+
+@pytest.mark.unit
 def test_dict_to_config_table_happy_path():
     input_dict = {
         "Section1": {"alpha": 1, "beta": "x"},
@@ -122,53 +175,3 @@
     assert rows == expected
 
 
-
-def test_fetching_real_defaults(mock_logger, mock_session):
-    cfg = SystemConfig(mock_logger, mock_session)
-    cfg.config_sources = [JsonConfig(default_config, mock_logger)]
-
-    assert cfg.get_value("Redis", "cache_group") == 0
-
-
-def test_load_config_utilizes_hierarchy():
-
-    default_dict = {
-        "Section1": {"alpha": 1, "beta": "x"},
-        "Section2": {"gamma": False, "delta": 3.14},
-    }
-
-    override_dict = {
-        "Section1": {"beta": "y"},
-        "Section2": {"Epsilon": True, "delta": 6.28},
-        "Section3": {"hi": "there"}
-    }
-
-    cfg = SystemConfig(None, None, [JsonConfig(default_dict, mock_logger), JsonConfig(override_dict, mock_logger)])
-
-    expected_dict = {
-        "Section1": {"alpha": 1, "beta": "y"},
-        "Section2": {"gamma": False, "Epsilon": True, "delta": 6.28},
-        "Section3": {"hi": "there"} # test that new sections are accounted for too
-    }
-
-    assert cfg.load_config() == expected_dict
-
-
-def test_get_section_incorporates_hierarchy():
-
-    default_dict = {
-        "Section1": {"alpha": 1, "beta": "x"},
-        "Section2": {"gamma": False, "delta": 3.14},
-    }
-
-    override_dict = {
-        "Section1": {"beta": "y"},
-        "Section2": {"gamma": False, "delta": 3.14},
-    }
-
-    cfg = SystemConfig(None, None, [JsonConfig(default_dict, mock_logger), JsonConfig(override_dict, mock_logger)])
-
-    expected_dict = {"alpha": 1, "beta": "y"}
-
-    assert cfg.get_section("Section1") == expected_dict
-