Add Cython to requirements.txt

CHANGELOG.rst

@@ -2,6 +2,18 @@ Changelog
 =========
 
 
+v0.9.7
+-------------
+
+- Fix resolution of setup files which partially have dynamic dependencies.
+
+
+v0.9.6
+-------------
+
+- Mock the actual setup provider defined in setup.py.
+- Update dependency resolvelib to latest.
+
 v0.9.5
 -------------
 

requirements.txt

@@ -5,6 +5,7 @@ charset-normalizer==2.1.0
 click==8.1.3
 colorama==0.4.5
 commoncode==30.2.0
+Cython==0.29.35
 dparse2==0.7.0
 idna==3.3
 importlib-metadata==4.12.0
@@ -17,7 +18,7 @@ pkginfo2==30.0.0
 pyparsing==3.0.9
 PyYAML==6.0
 requests==2.28.1
-resolvelib==0.8.1
+resolvelib >= 1.0.0
 saneyaml==0.5.2
 soupsieve==2.3.2.post1
 text-unidecode==1.3

setup.cfg

@@ -64,7 +64,7 @@ install_requires =
     pkginfo2 >= 30.0.0
     pip-requirements-parser >= 32.0.1
     requests >= 2.18.0
-    resolvelib >= 0.8.1
+    resolvelib >= 1.0.0
     saneyaml >= 0.5.2
     toml >= 0.10.0
     mock >= 3.0.5

src/python_inspector/api.py

@@ -37,6 +37,7 @@
 from python_inspector.resolution import get_environment_marker_from_environment
 from python_inspector.resolution import get_package_list
 from python_inspector.resolution import get_python_version_from_env_tag
+from python_inspector.resolution import get_reqs_insecurely
 from python_inspector.resolution import get_requirements_from_python_manifest
 from python_inspector.utils_pypi import PLATFORMS_BY_OS
 from python_inspector.utils_pypi import PYPI_SIMPLE_URL
@@ -175,22 +176,30 @@ def resolve_dependencies(
                 f"is not compatible with setup.py {setup_py_file} "
                 f"python_requires {python_requires}",
             )
-
-        setup_py_file_deps = package_data.dependencies
-        for dep in package_data.dependencies:
-            # TODO : we need to handle to all the scopes
-            if dep.scope == "install":
-                direct_dependencies.append(dep)
-
-        if not package_data.dependencies:
-            reqs = get_requirements_from_python_manifest(
-                sdist_location=os.path.dirname(setup_py_file),
-                setup_py_location=setup_py_file,
-                files=[setup_py_file],
-                analyze_setup_py_insecurely=analyze_setup_py_insecurely,
+        if analyze_setup_py_insecurely:
+            reqs = list(
+                get_reqs_insecurely(
+                    setup_py_location=setup_py_file,
+                )
             )
             setup_py_file_deps = list(get_dependent_packages_from_reqs(reqs))
             direct_dependencies.extend(setup_py_file_deps)
+        else:
+            setup_py_file_deps = package_data.dependencies
+            for dep in package_data.dependencies:
+                # TODO : we need to handle to all the scopes
+                if dep.scope == "install":
+                    direct_dependencies.append(dep)
+
+            if not package_data.dependencies:
+                reqs = get_requirements_from_python_manifest(
+                    sdist_location=os.path.dirname(setup_py_file),
+                    setup_py_location=setup_py_file,
+                    files=[setup_py_file],
+                    analyze_setup_py_insecurely=analyze_setup_py_insecurely,
+                )
+                setup_py_file_deps = list(get_dependent_packages_from_reqs(reqs))
+                direct_dependencies.extend(setup_py_file_deps)
 
         package_data.dependencies = setup_py_file_deps
         file_package_data = [package_data.to_dict()]

src/python_inspector/resolution.py

@@ -493,25 +493,30 @@ def get_requirements_for_package_from_pypi_simple(
                 "setup.cfg",
             )
 
-            requirements = list(
-                get_setup_requirements(
-                    sdist_location=sdist_location,
+            if self.analyze_setup_py_insecurely:
+                yield from get_reqs_insecurely(
                     setup_py_location=setup_py_location,
-                    setup_cfg_location=setup_cfg_location,
                 )
-            )
-            if requirements:
-                yield from requirements
             else:
-                # Look in requirements file if and only if thy are refered in setup.py or setup.cfg
-                # And no deps have been yielded by requirements file
-
-                yield from get_requirements_from_python_manifest(
-                    sdist_location=sdist_location,
-                    setup_py_location=setup_py_location,
-                    files=[setup_cfg_location, setup_py_location],
-                    analyze_setup_py_insecurely=self.analyze_setup_py_insecurely,
+                requirements = list(
+                    get_setup_requirements(
+                        sdist_location=sdist_location,
+                        setup_py_location=setup_py_location,
+                        setup_cfg_location=setup_cfg_location,
+                    )
                 )
+                if requirements:
+                    yield from requirements
+                else:
+                    # Look in requirements file if and only if thy are refered in setup.py or setup.cfg
+                    # And no deps have been yielded by requirements file
+
+                    yield from get_requirements_from_python_manifest(
+                        sdist_location=sdist_location,
+                        setup_py_location=setup_py_location,
+                        files=[setup_cfg_location, setup_py_location],
+                        analyze_setup_py_insecurely=self.analyze_setup_py_insecurely,
+                    )
 
     def get_requirements_for_package_from_pypi_json_api(
         self, purl: PackageURL

src/python_inspector/resolve_cli.py

@@ -20,7 +20,7 @@
 
 TRACE = False
 
-__version__ = "0.9.4"
+__version__ = "0.9.7"
 
 DEFAULT_PYTHON_VERSION = "38"
 PYPI_SIMPLE_URL = "https://pypi.org/simple"

src/python_inspector/setup_py_live_eval.py

@@ -9,15 +9,17 @@
 #
 """Generate requirements from `setup.py` and `requirements-devel.txt`."""
 
+import ast
 import os
-import re
 import sys
 
 try:
     import configparser
 except ImportError:  # pragma: no cover
     import ConfigParser as configparser
 
+import distutils.core
+
 import mock
 import setuptools
 from commoncode.command import pushd
@@ -54,11 +56,65 @@ def iter_requirements(level, extras, setup_file):
     setup_requires = {}
     # change directory to setup.py path
     with pushd(os.path.dirname(setup_file)):
-        with mock.patch.object(setuptools, "setup") as mock_setup:
-            sys.path.append(os.path.dirname(setup_file))
-            g = {"__file__": setup_file, "__name__": "__main__"}
-            with open(setup_file) as sf:
-                exec(sf.read(), g)
+        with open(setup_file) as sf:
+            file_contents = sf.read()
+            node = ast.parse(file_contents)
+            asnames = {}
+            imports = []
+            for elem in ast.walk(node):
+                # save the asnames to parse aliases later
+                if isinstance(elem, ast.Import):
+                    for n in elem.names:
+                        asnames[(n.asname if n.asname is not None else n.name)] = n.name
+            for elem in ast.walk(node):
+                # for function imports, e.g. from setuptools import setup; setup()
+                if isinstance(elem, ast.ImportFrom) and "setup" in [e.name for e in elem.names]:
+                    imports.append(elem.module)
+                # for module imports, e.g. import setuptools; setuptools.setup(...)
+                elif (
+                    isinstance(elem, ast.Expr)
+                    and isinstance(elem.value, ast.Call)
+                    and isinstance(elem.value.func, ast.Attribute)
+                    and isinstance(elem.value.func.value, ast.Name)
+                    and elem.value.func.attr == "setup"
+                ):
+                    name = elem.value.func.value.id
+                    if name in asnames.keys():
+                        name = asnames[name]
+                    imports.append(name)
+                # for module imports, e.g. import disttools.core; disttools.core.setup(...)
+                elif (
+                    isinstance(elem, ast.Expr)
+                    and isinstance(elem.value, ast.Call)
+                    and isinstance(elem.value.func, ast.Attribute)
+                    and isinstance(elem.value.func.value, ast.Attribute)
+                    and elem.value.func.attr == "setup"
+                ):
+                    name = (
+                        str(elem.value.func.value.value.id) + "." + str(elem.value.func.value.attr)
+                    )
+                    if name in asnames.keys():
+                        name = asnames[name]
+                    imports.append(name)
+            setup_providers = [i for i in imports if i in ["distutils.core", "setuptools"]]
+            if len(setup_providers) == 0:
+                print(
+                    f"Warning: unable to recognize setup provider in {setup_file}: "
+                    "defaulting to 'distutils.core'."
+                )
+                setup_provider = "distutils.core"
+            elif len(setup_providers) == 1:
+                setup_provider = setup_providers[0]
+            else:
+                print(
+                    f"Warning: ambiguous setup provider in {setup_file}: candidates are {setup_providers}"
+                    "defaulting to 'distutils.core'."
+                )
+                setup_provider = "distutils.core"
+            with mock.patch.object(eval(setup_provider), "setup") as mock_setup:
+                sys.path.append(os.path.dirname(setup_file))
+                g = {"__file__": setup_file, "__name__": "__main__"}
+                exec(file_contents, g)
             sys.path.pop()
             # removing the assertion `assert g["setup"]`` since this is not true for all cases
             # for example when setuptools.setup() is called instead of setup()

tests/data/default-url-expected.json

@@ -2,7 +2,7 @@
   "headers": {
     "tool_name": "python-inspector",
     "tool_homepageurl": "https://github.com/nexB/python-inspector",
-    "tool_version": "0.9.4",
+    "tool_version": "0.9.7",
     "options": [
       "--specifier zipp==3.8.0",
       "--index-url https://pypi.org/simple",

tests/data/environment-marker-test-requirements.txt-expected.json

@@ -2,7 +2,7 @@
   "headers": {
     "tool_name": "python-inspector",
     "tool_homepageurl": "https://github.com/nexB/python-inspector",
-    "tool_version": "0.9.4",
+    "tool_version": "0.9.7",
     "options": [
       "--requirement /home/tg1999/Desktop/python-inspector-1/tests/data/environment-marker-test-requirements.txt",
       "--index-url https://pypi.org/simple",

tests/data/frozen-requirements.txt-expected.json

@@ -2,7 +2,7 @@
   "headers": {
     "tool_name": "python-inspector",
     "tool_homepageurl": "https://github.com/nexB/python-inspector",
-    "tool_version": "0.9.4",
+    "tool_version": "0.9.7",
     "options": [
       "--requirement /home/tg1999/Desktop/python-inspector-1/tests/data/frozen-requirements.txt",
       "--index-url https://pypi.org/simple",