Merge pull request #53 from nexB/52

Ensure to filter out top level dependencies on the basis of their environment markers

Author: TG1999

src/python_inspector/resolution.py

@@ -101,6 +101,17 @@ def get_requirements_from_distribution(
     return list(get_requirements_from_dependencies(dependencies=deps[0].dependencies))
 
 
+def get_environment_marker_from_environment(environment):
+    return {
+        "extra": "",
+        "python_version": get_python_version_from_env_tag(
+            python_version=environment.python_version
+        ),
+        "platform_system": environment.operating_system.capitalize(),
+        "sys_platform": environment.operating_system,
+    }
+
+
 def is_requirements_file_in_setup_files(setup_files: List[str]) -> bool:
     """
     Return True if the string ``requirements.txt`` is found in any of the ``setup_files`` location
@@ -220,6 +231,7 @@ def remove_extras(identifier: str) -> str:
 class PythonInputProvider(AbstractProvider):
     def __init__(self, environment=None, repos=tuple()):
         self.environment = environment
+        self.environment_marker = get_environment_marker_from_environment(environment)
         self.repos = repos or []
         self.versions_by_package = {}
         self.dependencies_by_purl = {}
@@ -484,16 +496,7 @@ def _iter_dependencies(self, candidate: Candidate) -> Generator[Requirement, Non
             if r.marker is None:
                 yield r
             else:
-                if r.marker.evaluate(
-                    {
-                        "extra": "",
-                        "python_version": get_python_version_from_env_tag(
-                            python_version=self.environment.python_version
-                        ),
-                        "platform_system": self.environment.operating_system.capitalize(),
-                        "sys_platform": self.environment.operating_system,
-                    }
-                ):
+                if r.marker.evaluate(self.environment_marker):
                     yield r
 
     def get_dependencies(self, candidate: Candidate) -> List[Requirement]:

src/python_inspector/resolve_cli.py

@@ -11,6 +11,7 @@
 
 import json
 import os
+from typing import Dict
 from typing import List
 
 import click
@@ -24,6 +25,7 @@
 from python_inspector import utils
 from python_inspector import utils_pypi
 from python_inspector.cli_utils import FileOptionType
+from python_inspector.resolution import get_environment_marker_from_environment
 from python_inspector.resolution import get_resolved_dependencies
 
 TRACE = False
@@ -351,7 +353,13 @@ def resolve(
     If empty, use instead the PyPI.org JSON API exclusively.
     """
 
-    requirements = list(get_requirements_from_direct_dependencies(direct_dependencies))
+    environment_marker = get_environment_marker_from_environment(environment)
+
+    requirements = list(
+        get_requirements_from_direct_dependencies(
+            direct_dependencies=direct_dependencies, environment_marker=environment_marker
+        )
+    )
 
     resolved_dependencies = get_resolved_dependencies(
         requirements=requirements,
@@ -369,15 +377,20 @@ def resolve(
 
 
 def get_requirements_from_direct_dependencies(
-    direct_dependencies: List[DependentPackage],
+    direct_dependencies: List[DependentPackage], environment_marker: Dict
 ) -> List[Requirement]:
     for dependency in direct_dependencies:
         # FIXME We are skipping editable requirements
         # and other pip options for now
         # https://github.com/nexB/python-inspector/issues/41
         if not can_process_dependent_package(dependency):
             continue
-        yield Requirement(requirement_string=dependency.extracted_requirement)
+        req = Requirement(requirement_string=dependency.extracted_requirement)
+        if req.marker is None:
+            yield req
+        else:
+            if req.marker.evaluate(environment_marker):
+                yield req
 
 
 def write_output(headers, requirements, resolved_dependencies, json_output, pdt_output=False):

tests/data/environment-marker-test-requirements.txt

@@ -0,0 +1,7 @@
+click>6,<6.8
+Flask==1.0
+itsdangerous<0.25
+license-expression ; platform_system == "Windows"
+Jinja2==2.11.3
+MarkupSafe==1.0
+Werkzeug==0.15.3

tests/data/pdt-expected.json renamed to tests/data/environment-marker-test-requirements.txt-expected.json

@@ -0,0 +1,40 @@
+[
+  {
+    "key": "flask",
+    "package_name": "flask",
+    "installed_version": "1.0",
+    "dependencies": [
+      {
+        "key": "click",
+        "package_name": "click",
+        "installed_version": "6.7",
+        "dependencies": []
+      },
+      {
+        "key": "itsdangerous",
+        "package_name": "itsdangerous",
+        "installed_version": "0.24",
+        "dependencies": []
+      },
+      {
+        "key": "jinja2",
+        "package_name": "jinja2",
+        "installed_version": "2.11.3",
+        "dependencies": [
+          {
+            "key": "markupsafe",
+            "package_name": "markupsafe",
+            "installed_version": "1.0",
+            "dependencies": []
+          }
+        ]
+      },
+      {
+        "key": "werkzeug",
+        "package_name": "werkzeug",
+        "installed_version": "0.15.3",
+        "dependencies": []
+      }
+    ]
+  }
+]

tests/data/frozen-requirements-pdt-expected.json renamed to tests/data/frozen-requirements.txt-expected.json

@@ -0,0 +1,29 @@
+aboutcode-toolkit==7.0.2
+attrs==21.4.0
+beautifulsoup4==4.11.1
+certifi==2022.5.18.1
+charset-normalizer==2.0.12
+click==8.0.4
+colorama==0.4.4
+commoncode==30.2.0
+dparse2==0.6.1
+idna==3.3
+importlib-metadata==4.8.3
+intbitset==3.0.1
+packageurl-python==0.9.9
+packaging==21.3
+-e git+https://github.com/nexB/python-inspector@18baae17824d6bacb4b1d519b10a0d0e50775884#egg=python_inspector
+pip-requirements-parser==31.2.0
+pkginfo2==30.0.0
+pyparsing==3.0.9
+PyYAML==6.0
+requests==2.27.1
+resolvelib==0.8.1
+saneyaml==0.5.2
+soupsieve==2.3.2.post1
+text-unidecode==1.3
+toml==0.10.2
+typing==3.6.6
+typing_extensions==4.1.1
+urllib3==1.26.9
+zipp==3.6.0

tests/data/pdt-requirements.txt-expected.json

@@ -37,18 +37,18 @@
       "sdist_url": "https://files.pythonhosted.org/packages/59/87/84326af34517fca8c58418d148f2403df25303e02736832403587318e9e8/click-8.1.3.tar.gz"
     },
     {
-      "package": "pkg:pypi/[email protected].1",
+      "package": "pkg:pypi/[email protected].2",
       "dependencies": [
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
-        "pkg:pypi/[email protected].1"
+        "pkg:pypi/[email protected].2"
       ],
       "wheel_urls": [
-        "https://files.pythonhosted.org/packages/3c/96/6c896f80f466b7f5e2cfd6d632fe5b0464dcb412757c595a663e59589a93/Flask-2.2.1-py3-none-any.whl"
+        "https://files.pythonhosted.org/packages/0f/43/15f4f9ab225b0b25352412e8daa3d0e3d135fcf5e127070c74c3632c8b4c/Flask-2.2.2-py3-none-any.whl"
       ],
-      "sdist_url": "https://files.pythonhosted.org/packages/4b/4f/50888944490f2263bd70171e8298c9626675fd3dfd750694a7beaa3484fb/Flask-2.2.1.tar.gz"
+      "sdist_url": "https://files.pythonhosted.org/packages/69/b6/53cfa30eed5aa7343daff36622843688ba8c6fe9829bb2b92e193ab1163f/Flask-2.2.2.tar.gz"
     },
     {
       "package": "pkg:pypi/[email protected]",
@@ -87,14 +87,14 @@
       "sdist_url": "https://files.pythonhosted.org/packages/1d/97/2288fe498044284f39ab8950703e88abbac2abbdf65524d576157af70556/MarkupSafe-2.1.1.tar.gz"
     },
     {
-      "package": "pkg:pypi/[email protected].1",
+      "package": "pkg:pypi/[email protected].2",
       "dependencies": [
         "pkg:pypi/[email protected]"
       ],
       "wheel_urls": [
-        "https://files.pythonhosted.org/packages/c0/93/d6d60870e47162ea6a1bbdd787649eea776b2a70618dd66ed87cb2238543/Werkzeug-2.2.1-py3-none-any.whl"
+        "https://files.pythonhosted.org/packages/c8/27/be6ddbcf60115305205de79c29004a0c6bc53cec814f733467b1bb89386d/Werkzeug-2.2.2-py3-none-any.whl"
       ],
-      "sdist_url": "https://files.pythonhosted.org/packages/32/19/a92cdbd9fb795928dfca1031278ae8a7f051e78a2c057c224ad2d4cdd95e/Werkzeug-2.2.1.tar.gz"
+      "sdist_url": "https://files.pythonhosted.org/packages/f8/c1/1c8e539f040acd80f844c69a5ef8e2fccdf8b442dabb969e497b55d544e1/Werkzeug-2.2.2.tar.gz"
     },
     {
       "package": "pkg:pypi/[email protected]",

tests/data/pinned-pdt-requirements.txt

@@ -45,7 +45,7 @@ def test_cli_with_default_urls():
 @pytest.mark.online
 def test_pdt_output():
     requirements_file = test_env.get_test_loc("pdt-requirements.txt")
-    expected_file = test_env.get_test_loc("pdt-expected.json", must_exist=False)
+    expected_file = test_env.get_test_loc("pdt-requirements.txt-expected.json", must_exist=False)
     extra_options = []
     check_requirements_resolution(
         requirements_file=requirements_file,
@@ -58,8 +58,10 @@ def test_pdt_output():
 
 @pytest.mark.online
 def test_pdt_output_with_pinned_requirements():
-    requirements_file = test_env.get_test_loc("pinned-requirements.txt")
-    expected_file = test_env.get_test_loc("pinned-requirements-pdt-expected.json", must_exist=False)
+    requirements_file = test_env.get_test_loc("pinned-pdt-requirements.txt")
+    expected_file = test_env.get_test_loc(
+        "pinned-pdt-requirements.txt-expected.json", must_exist=False
+    )
     extra_options = []
     check_requirements_resolution(
         requirements_file=requirements_file,
@@ -73,7 +75,7 @@ def test_pdt_output_with_pinned_requirements():
 @pytest.mark.online
 def test_pdt_output_with_frozen_requirements():
     requirements_file = test_env.get_test_loc("frozen-requirements.txt")
-    expected_file = test_env.get_test_loc("frozen-requirements-pdt-expected.json", must_exist=False)
+    expected_file = test_env.get_test_loc("frozen-requirements.txt-expected.json", must_exist=False)
     extra_options = []
     check_requirements_resolution(
         requirements_file=requirements_file,
@@ -137,6 +139,27 @@ def test_cli_with_multiple_index_url_and_tilde_req():
     )
 
 
+@pytest.mark.online
+def test_cli_with_environment_marker_and_complex_ranges():
+    requirements_file = test_env.get_test_loc("environment-marker-test-requirements.txt")
+    expected_file = test_env.get_test_loc(
+        "environment-marker-test-requirements.txt-expected.json", must_exist=False
+    )
+    extra_options = [
+        "--operating-system",
+        "linux",
+        "--python-version",
+        "37",
+    ]
+    check_requirements_resolution(
+        requirements_file=requirements_file,
+        expected_file=expected_file,
+        extra_options=extra_options,
+        pdt_output=True,
+        regen=REGEN_TEST_FIXTURES,
+    )
+
+
 @pytest.mark.online
 def test_cli_with_multiple_index_url_and_tilde_req_with_max_rounds():
     expected_file = test_env.get_test_loc("tilde_req-expected.json", must_exist=False)
@@ -345,13 +368,23 @@ def test_get_requirements_from_direct_dependencies():
         )
     ]
 
-    requirements = [str(r) for r in get_requirements_from_direct_dependencies(direct_dependencies)]
+    requirements = [
+        str(r)
+        for r in get_requirements_from_direct_dependencies(
+            direct_dependencies=direct_dependencies, environment_marker={}
+        )
+    ]
 
     assert requirements == ["django>=1.11.11"]
 
 
 def test_get_requirements_from_direct_dependencies_with_empty_list():
-    assert list(get_requirements_from_direct_dependencies(direct_dependencies=[])) == []
+    assert (
+        list(
+            get_requirements_from_direct_dependencies(direct_dependencies=[], environment_marker={})
+        )
+        == []
+    )
 
 
 def test_get_requirements_from_direct_dependencies_with_editable_requirements():
@@ -378,6 +411,11 @@ def test_get_requirements_from_direct_dependencies_with_editable_requirements():
         )
     ]
 
-    requirements = [str(r) for r in get_requirements_from_direct_dependencies(direct_dependencies)]
+    requirements = [
+        str(r)
+        for r in get_requirements_from_direct_dependencies(
+            direct_dependencies=direct_dependencies, environment_marker={}
+        )
+    ]
 
     assert requirements == []

tests/data/pinned-requirements-pdt-expected.json renamed to tests/data/pinned-pdt-requirements.txt-expected.json

@@ -40,7 +40,7 @@ def test_get_resolved_dependencies_with_flask_and_python_310():
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
-        "pkg:pypi/[email protected].1",
+        "pkg:pypi/[email protected].2",
     ]
 
 
@@ -65,7 +65,7 @@ def test_get_resolved_dependencies_with_flask_and_python_310_windows():
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
-        "pkg:pypi/[email protected].1",
+        "pkg:pypi/[email protected].2",
     ]
 
 
@@ -118,7 +118,7 @@ def test_get_resolved_dependencies_with_tilde_requirement_using_json_api():
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
-        "pkg:pypi/[email protected].1",
+        "pkg:pypi/[email protected].2",
         "pkg:pypi/[email protected]",
     ]
 
@@ -145,7 +145,7 @@ def test_without_supported_wheels():
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
-        "pkg:pypi/setuptools@63.4.1",
+        "pkg:pypi/setuptools@65.0.0",
         "pkg:pypi/[email protected]",
     ]