diff --git a/morgan/__init__.py b/morgan/__init__.py index 755fed9..32f7d8f 100644 --- a/morgan/__init__.py +++ b/morgan/__init__.py @@ -10,7 +10,7 @@ import urllib.parse import urllib.request import zipfile -from typing import Dict, Iterable, Tuple +from typing import Dict, Iterable, Optional, Tuple import packaging.requirements import packaging.specifiers @@ -20,7 +20,7 @@ from morgan import configurator, metadata, server from morgan.__about__ import __version__ -from morgan.utils import Cache, to_single_dash +from morgan.utils import Cache, is_requirement_relevant, to_single_dash PYPI_ADDRESS = "https://pypi.org/simple/" PREFERRED_HASH_ALG = "sha256" @@ -125,10 +125,16 @@ def _mirror( self, requirement: packaging.requirements.Requirement, required_by: packaging.requirements.Requirement = None, - ) -> dict: + ) -> Optional[dict]: if self._processed_pkgs.check(requirement): return None + # Check if requirement is relevant for any environment + if not is_requirement_relevant(requirement, self.envs.values()): + print(f"\tSkipping {requirement}, not relevant for any environment") + self._processed_pkgs.add(requirement) # Mark as processed + return None + if required_by: print("[{}]: {}".format(required_by, requirement)) else: diff --git a/morgan/metadata.py b/morgan/metadata.py index 0d0c6e3..b62f4b7 100644 --- a/morgan/metadata.py +++ b/morgan/metadata.py @@ -9,6 +9,7 @@ from packaging.markers import Marker, Variable as MarkerVariable import tomli +from morgan.utils import filter_relevant_requirements METADATA_VERSION_11 = Version("1.1") METADATA_VERSION_12 = Version("1.2") @@ -190,24 +191,7 @@ def dependencies( elif extra in extras: deps |= self.optional_dependencies[extra] - irrelevant_deps = set() - for dep in deps: - relevant = True - if dep.marker: - relevant = False - for env in envs: - env["extra"] = ",".join(extras) - if dep.marker.evaluate(env): - relevant = True - break - - if not relevant: - irrelevant_deps.add(dep) - continue - - deps -= irrelevant_deps - - return deps + return filter_relevant_requirements(deps, envs, extras) def _add_core_requirements(self, reqs): self.core_dependencies |= set([Requirement(dep) for dep in reqs]) diff --git a/morgan/utils.py b/morgan/utils.py index 425efc1..89c1463 100644 --- a/morgan/utils.py +++ b/morgan/utils.py @@ -1,4 +1,6 @@ import re +from collections import OrderedDict +from typing import Dict, Iterable, Optional, Set from packaging.requirements import Requirement @@ -42,3 +44,55 @@ def is_simple_case(self, req): if all(spec.operator in ('>', '>=') for spec in specifier._specs): return True return False + + +def is_requirement_relevant( + requirement: Requirement, envs: Iterable[Dict], extras: Optional[Set[str]] = None +) -> bool: + """Determines if a requirement is relevant for any of the provided environments. + + Args: + requirement: The requirement to evaluate. + envs: The environments to check against. + extras: Optional extras to consider during evaluation. + + Returns: + True if the requirement has no marker or if its marker evaluates to + True for at least one environment, False otherwise. + """ + if not requirement.marker: + return True + + # If no environments specified, assume relevant + if not envs: + return True + + for env in envs: + # Create a copy of the environment to avoid modifying the original + env_copy = env.copy() + env_copy.setdefault("extra", "") + if extras: + env_copy["extra"] = ",".join(extras) + + if requirement.marker.evaluate(env_copy): + return True + + return False + + +def filter_relevant_requirements( + requirements: Iterable[Requirement], + envs: Iterable[Dict], + extras: Optional[Set[str]] = None, +) -> Set[Requirement]: + """Filters a collection of requirements to only those relevant for the provided environments. + + Args: + requirements: Requirements to filter. + envs: The environments to check against. + extras: Optional extras to consider during evaluation. + + Returns: + Set of requirements relevant for at least one environment. + """ + return {req for req in requirements if is_requirement_relevant(req, envs, extras)} diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..bf296cd --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,194 @@ +import pytest +from packaging.requirements import Requirement + +from morgan.utils import filter_relevant_requirements, is_requirement_relevant + + +class TestEnvironmentEvaluation: + @pytest.fixture + def python_environments(self): + """Sample environments with different Python versions""" + return [ + { + "python_version": "3.7", + "sys_platform": "linux", + "platform_machine": "x86_64", + }, + { + "python_version": "3.8", + "sys_platform": "linux", + "platform_machine": "x86_64", + }, + { + "python_version": "3.9", + "sys_platform": "linux", + "platform_machine": "x86_64", + }, + ] + + @pytest.fixture + def platform_environments(self): + """Sample environments with different platforms""" + return [ + { + "python_version": "3.8", + "sys_platform": "linux", + "platform_machine": "x86_64", + }, + { + "python_version": "3.8", + "sys_platform": "win32", + "platform_machine": "x86_64", + }, + { + "python_version": "3.8", + "sys_platform": "darwin", + "platform_machine": "x86_64", + }, + ] + + def test_simple_requirement_always_relevant(self, python_environments): + """Test that a requirement without markers is always relevant""" + req = Requirement("simple-package") + + result = is_requirement_relevant(req, python_environments) + + assert result + + def test_simple_requirement_with_empty_environments(self): + """Test that a requirement without markers is always relevant even with empty environments""" + req = Requirement("simple-package") + + result = is_requirement_relevant(req, []) + + assert result + + @pytest.mark.parametrize( + "requirement_str,expected", + [ + ('package; python_version < "3.8"', True), + ('package; python_version > "3.9"', False), + ('package; python_version < "3.6"', False), + ], + ids=["py37_only", "py37_and_above", "py35_and_below"], + ) + def test_requirement_with_python_version_marker( + self, requirement_str, expected, python_environments + ): + """Test requirements with Python version markers""" + req = Requirement(requirement_str) + + result = is_requirement_relevant(req, python_environments) + + assert result == expected + + @pytest.mark.parametrize( + "requirement_str,expected", + [ + ('package; sys_platform == "linux"', True), + ('package; sys_platform == "linux" or sys_platform == "win32"', True), + ('package; sys_platform == "freebsd"', False), + ], + ids=["linux_only", "linux_or_windows", "freebsd_only"], + ) + def test_requirement_with_platform_marker( + self, requirement_str, expected, platform_environments + ): + """Test requirements with platform markers""" + req = Requirement(requirement_str) + + result = is_requirement_relevant(req, platform_environments) + + assert result == expected + + @pytest.mark.parametrize( + "extras,expected", + [ + ({"test"}, True), # With matching extra + ({"other"}, False), # With non-matching extra + (None, False), # No extras provided + ], + ids=["with_test_extra", "with_other_extra", "no_extras"], + ) + def test_requirement_with_extra_marker(self, extras, expected, python_environments): + """Test requirements with extra markers""" + req = Requirement('package; extra == "test"') + + result = is_requirement_relevant(req, python_environments, extras=extras) + + assert result == expected + + @pytest.mark.parametrize( + "requirement_str,extras,expected", + [ + ('package; python_version >= "3.8" and extra == "test"', {"test"}, True), + ('package; python_version >= "3.8" and extra == "test"', {"other"}, False), + ('package; python_version >= "3.9" and extra == "test"', {"test"}, True), + ('package; python_version >= "3.9" or extra == "test"', {"test"}, True), + ('package; python_version >= "3.9" or extra == "test"', None, True), + ], + ids=[ + "py38_plus_with_test_extra", + "py38_plus_with_wrong_extra", + "py39_plus_with_test_extra", + "py39_or_test_extra_with_extra", + "py39_or_test_extra_no_extras", + ], + ) + def test_complex_requirement_with_combined_markers( + self, requirement_str, extras, expected, python_environments + ): + """Test requirements with combined markers""" + req = Requirement(requirement_str) + + result = is_requirement_relevant(req, python_environments, extras=extras) + + assert result == expected + + @pytest.mark.parametrize( + "extras,expected_count,expected_names", + [ + (None, 3, {"always-relevant", "py37-only", "py38-plus"}), + ({"test"}, 4, {"always-relevant", "py37-only", "py38-plus", "test-extra"}), + ], + ids=["no_extras", "with_test_extra"], + ) + def test_filter_relevant_requirements( + self, extras, expected_count, expected_names, python_environments + ): + """Test filtering a collection of requirements""" + requirements = [ + Requirement("always-relevant"), + Requirement('py37-only; python_version < "3.8"'), + Requirement('py38-plus; python_version >= "3.8"'), + Requirement('test-extra; extra == "test"'), + Requirement('not-relevant; python_version < "3.6"'), + ] + + filtered = filter_relevant_requirements( + requirements, python_environments, extras=extras + ) + + assert len(filtered) == expected_count + assert {req.name for req in filtered} == expected_names + + def test_filter_with_empty_requirements(self): + """Test filtering with empty requirements list""" + requirements = [] + environments = [{"python_version": "3.8"}] + + filtered = filter_relevant_requirements(requirements, environments) + + assert len(filtered) == 0 + + def test_filter_with_empty_environments(self): + """Test filtering with empty environments list""" + requirements = [ + Requirement("package1"), + Requirement('package2; python_version >= "3.8"'), + ] + environments = [] + + filtered = filter_relevant_requirements(requirements, environments) + + assert len(filtered) == 2