evaluate env markers for requirements

morgan/__init__.py

@@ -10,7 +10,7 @@
 import urllib.parse
 import urllib.request
 import zipfile
-from typing import Dict, Iterable, Tuple
+from typing import Dict, Iterable, Optional, Tuple
 
 import packaging.requirements
 import packaging.specifiers
@@ -20,7 +20,7 @@
 
 from morgan import configurator, metadata, server
 from morgan.__about__ import __version__
-from morgan.utils import Cache, to_single_dash
+from morgan.utils import Cache, is_requirement_relevant, to_single_dash
 
 PYPI_ADDRESS = "https://pypi.org/simple/"
 PREFERRED_HASH_ALG = "sha256"
@@ -125,10 +125,16 @@ def _mirror(
         self,
         requirement: packaging.requirements.Requirement,
         required_by: packaging.requirements.Requirement = None,
-    ) -> dict:
+    ) -> Optional[dict]:
         if self._processed_pkgs.check(requirement):
             return None
 
+        # Check if requirement is relevant for any environment
+        if not is_requirement_relevant(requirement, self.envs.values()):
+            print(f"\tSkipping {requirement}, not relevant for any environment")
+            self._processed_pkgs.add(requirement)  # Mark as processed
+            return None
+
         if required_by:
             print("[{}]: {}".format(required_by, requirement))
         else:

morgan/metadata.py

@@ -9,6 +9,7 @@
 from packaging.markers import Marker, Variable as MarkerVariable
 import tomli
 
+from morgan.utils import filter_relevant_requirements
 
 METADATA_VERSION_11 = Version("1.1")
 METADATA_VERSION_12 = Version("1.2")
@@ -190,24 +191,7 @@ def dependencies(
             elif extra in extras:
                 deps |= self.optional_dependencies[extra]
 
-        irrelevant_deps = set()
-        for dep in deps:
-            relevant = True
-            if dep.marker:
-                relevant = False
-                for env in envs:
-                    env["extra"] = ",".join(extras)
-                    if dep.marker.evaluate(env):
-                        relevant = True
-                        break
-
-            if not relevant:
-                irrelevant_deps.add(dep)
-                continue
-
-        deps -= irrelevant_deps
-
-        return deps
+        return filter_relevant_requirements(deps, envs, extras)
 
     def _add_core_requirements(self, reqs):
         self.core_dependencies |= set([Requirement(dep) for dep in reqs])

morgan/utils.py

@@ -1,4 +1,6 @@
 import re
+from collections import OrderedDict
+from typing import Dict, Iterable, Optional, Set
 
 from packaging.requirements import Requirement
 
@@ -42,3 +44,55 @@ def is_simple_case(self, req):
             if all(spec.operator in ('>', '>=') for spec in specifier._specs):
                 return True
         return False
+
+
+def is_requirement_relevant(
+    requirement: Requirement, envs: Iterable[Dict], extras: Optional[Set[str]] = None
+) -> bool:
+    """Determines if a requirement is relevant for any of the provided environments.
+
+    Args:
+        requirement: The requirement to evaluate.
+        envs: The environments to check against.
+        extras: Optional extras to consider during evaluation.
+
+    Returns:
+        True if the requirement has no marker or if its marker evaluates to
+        True for at least one environment, False otherwise.
+    """
+    if not requirement.marker:
+        return True
+
+    # If no environments specified, assume relevant
+    if not envs:
+        return True
+
+    for env in envs:
+        # Create a copy of the environment to avoid modifying the original
+        env_copy = env.copy()
+        env_copy.setdefault("extra", "")
+        if extras:
+            env_copy["extra"] = ",".join(extras)
+
+        if requirement.marker.evaluate(env_copy):
+            return True
+
+    return False
+
+
+def filter_relevant_requirements(
+    requirements: Iterable[Requirement],
+    envs: Iterable[Dict],
+    extras: Optional[Set[str]] = None,
+) -> Set[Requirement]:
+    """Filters a collection of requirements to only those relevant for the provided environments.
+
+    Args:
+        requirements: Requirements to filter.
+        envs: The environments to check against.
+        extras: Optional extras to consider during evaluation.
+
+    Returns:
+        Set of requirements relevant for at least one environment.
+    """
+    return {req for req in requirements if is_requirement_relevant(req, envs, extras)}

tests/test_utils.py

@@ -0,0 +1,194 @@
+import pytest
+from packaging.requirements import Requirement
+
+from morgan.utils import filter_relevant_requirements, is_requirement_relevant
+
+
+class TestEnvironmentEvaluation:
+    @pytest.fixture
+    def python_environments(self):
+        """Sample environments with different Python versions"""
+        return [
+            {
+                "python_version": "3.7",
+                "sys_platform": "linux",
+                "platform_machine": "x86_64",
+            },
+            {
+                "python_version": "3.8",
+                "sys_platform": "linux",
+                "platform_machine": "x86_64",
+            },
+            {
+                "python_version": "3.9",
+                "sys_platform": "linux",
+                "platform_machine": "x86_64",
+            },
+        ]
+
+    @pytest.fixture
+    def platform_environments(self):
+        """Sample environments with different platforms"""
+        return [
+            {
+                "python_version": "3.8",
+                "sys_platform": "linux",
+                "platform_machine": "x86_64",
+            },
+            {
+                "python_version": "3.8",
+                "sys_platform": "win32",
+                "platform_machine": "x86_64",
+            },
+            {
+                "python_version": "3.8",
+                "sys_platform": "darwin",
+                "platform_machine": "x86_64",
+            },
+        ]
+
+    def test_simple_requirement_always_relevant(self, python_environments):
+        """Test that a requirement without markers is always relevant"""
+        req = Requirement("simple-package")
+
+        result = is_requirement_relevant(req, python_environments)
+
+        assert result
+
+    def test_simple_requirement_with_empty_environments(self):
+        """Test that a requirement without markers is always relevant even with empty environments"""
+        req = Requirement("simple-package")
+
+        result = is_requirement_relevant(req, [])
+
+        assert result
+
+    @pytest.mark.parametrize(
+        "requirement_str,expected",
+        [
+            ('package; python_version < "3.8"', True),
+            ('package; python_version > "3.9"', False),
+            ('package; python_version < "3.6"', False),
+        ],
+        ids=["py37_only", "py37_and_above", "py35_and_below"],
+    )
+    def test_requirement_with_python_version_marker(
+        self, requirement_str, expected, python_environments
+    ):
+        """Test requirements with Python version markers"""
+        req = Requirement(requirement_str)
+
+        result = is_requirement_relevant(req, python_environments)
+
+        assert result == expected
+
+    @pytest.mark.parametrize(
+        "requirement_str,expected",
+        [
+            ('package; sys_platform == "linux"', True),
+            ('package; sys_platform == "linux" or sys_platform == "win32"', True),
+            ('package; sys_platform == "freebsd"', False),
+        ],
+        ids=["linux_only", "linux_or_windows", "freebsd_only"],
+    )
+    def test_requirement_with_platform_marker(
+        self, requirement_str, expected, platform_environments
+    ):
+        """Test requirements with platform markers"""
+        req = Requirement(requirement_str)
+
+        result = is_requirement_relevant(req, platform_environments)
+
+        assert result == expected
+
+    @pytest.mark.parametrize(
+        "extras,expected",
+        [
+            ({"test"}, True),  # With matching extra
+            ({"other"}, False),  # With non-matching extra
+            (None, False),  # No extras provided
+        ],
+        ids=["with_test_extra", "with_other_extra", "no_extras"],
+    )
+    def test_requirement_with_extra_marker(self, extras, expected, python_environments):
+        """Test requirements with extra markers"""
+        req = Requirement('package; extra == "test"')
+
+        result = is_requirement_relevant(req, python_environments, extras=extras)
+
+        assert result == expected
+
+    @pytest.mark.parametrize(
+        "requirement_str,extras,expected",
+        [
+            ('package; python_version >= "3.8" and extra == "test"', {"test"}, True),
+            ('package; python_version >= "3.8" and extra == "test"', {"other"}, False),
+            ('package; python_version >= "3.9" and extra == "test"', {"test"}, True),
+            ('package; python_version >= "3.9" or extra == "test"', {"test"}, True),
+            ('package; python_version >= "3.9" or extra == "test"', None, True),
+        ],
+        ids=[
+            "py38_plus_with_test_extra",
+            "py38_plus_with_wrong_extra",
+            "py39_plus_with_test_extra",
+            "py39_or_test_extra_with_extra",
+            "py39_or_test_extra_no_extras",
+        ],
+    )
+    def test_complex_requirement_with_combined_markers(
+        self, requirement_str, extras, expected, python_environments
+    ):
+        """Test requirements with combined markers"""
+        req = Requirement(requirement_str)
+
+        result = is_requirement_relevant(req, python_environments, extras=extras)
+
+        assert result == expected
+
+    @pytest.mark.parametrize(
+        "extras,expected_count,expected_names",
+        [
+            (None, 3, {"always-relevant", "py37-only", "py38-plus"}),
+            ({"test"}, 4, {"always-relevant", "py37-only", "py38-plus", "test-extra"}),
+        ],
+        ids=["no_extras", "with_test_extra"],
+    )
+    def test_filter_relevant_requirements(
+        self, extras, expected_count, expected_names, python_environments
+    ):
+        """Test filtering a collection of requirements"""
+        requirements = [
+            Requirement("always-relevant"),
+            Requirement('py37-only; python_version < "3.8"'),
+            Requirement('py38-plus; python_version >= "3.8"'),
+            Requirement('test-extra; extra == "test"'),
+            Requirement('not-relevant; python_version < "3.6"'),
+        ]
+
+        filtered = filter_relevant_requirements(
+            requirements, python_environments, extras=extras
+        )
+
+        assert len(filtered) == expected_count
+        assert {req.name for req in filtered} == expected_names
+
+    def test_filter_with_empty_requirements(self):
+        """Test filtering with empty requirements list"""
+        requirements = []
+        environments = [{"python_version": "3.8"}]
+
+        filtered = filter_relevant_requirements(requirements, environments)
+
+        assert len(filtered) == 0
+
+    def test_filter_with_empty_environments(self):
+        """Test filtering with empty environments list"""
+        requirements = [
+            Requirement("package1"),
+            Requirement('package2; python_version >= "3.8"'),
+        ]
+        environments = []
+
+        filtered = filter_relevant_requirements(requirements, environments)
+
+        assert len(filtered) == 2