Merge calizarr/fix-180-env-vars

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

resolve_cli.python

@@ -5,7 +5,7 @@
 # ScanCode is a trademark of nexB Inc.
 # SPDX-License-Identifier: Apache-2.0
 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
-# See https://aboutcode-orgnexB/python-inspector for support or download.
+# See https://aboutcode-org/python-inspector for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 import asyncio

src/python_inspector/api.py

@@ -91,6 +91,7 @@ def print_version(ctx, param, value):
 @click.option(
     "--index-url",
     "index_urls",
+    envvar="PYINSP_INDEX_URL",
     type=str,
     metavar="INDEX",
     show_default=True,
@@ -122,6 +123,7 @@ def print_version(ctx, param, value):
     "--netrc",
     "netrc_file",
     type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False),
+    envvar="PYINSP_NETRC_FILE",
     metavar="NETRC-FILE",
     hidden=True,
     required=False,
@@ -163,6 +165,7 @@ def print_version(ctx, param, value):
 )
 @click.option(
     "--verbose",
+    envvar="PYINSP_VERBOSE",
     is_flag=True,
     help="Enable verbose debug output.",
 )

src/python_inspector/resolve_cli.py

@@ -252,6 +252,7 @@ async def download_wheel(
                 )
             continue
         for wheel in supported_and_valid_wheels:
+            wheel.credentials = repo.credentials
             fetched_wheel_filename = await wheel.download(
                 dest_dir=dest_dir,
                 verbose=verbose,

src/python_inspector/utils_pypi.py

@@ -0,0 +1,323 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/python-inspector for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import os
+from unittest.mock import patch
+
+import packvers
+import pytest
+from commoncode.system import on_mac
+from commoncode.testcase import FileDrivenTesting
+from packvers.requirements import Requirement
+from test_cli import check_data_results
+
+from _packagedcode import models
+from python_inspector.api import get_resolved_dependencies
+from python_inspector.error import NoVersionsFound
+from python_inspector.resolution import PythonInputProvider
+from python_inspector.resolution import get_requirements_from_dependencies
+from python_inspector.resolution import get_requirements_from_python_manifest
+from python_inspector.resolution import is_valid_version
+from python_inspector.resolution import parse_reqs_from_setup_py_insecurely
+from python_inspector.utils_pypi import Environment
+from python_inspector.utils_pypi import PypiSimpleRepository
+from python_inspector.utils_pypi import get_current_indexes
+
+# Used for tests to regenerate fixtures with regen=True
+REGEN_TEST_FIXTURES = os.getenv("PYINSP_REGEN_TEST_FIXTURES", False)
+
+setup_test_env = FileDrivenTesting()
+setup_test_env.test_data_dir = os.path.join(os.path.dirname(__file__), "data")
+
+
+def check_get_resolved_dependencies(
+    requirement: Requirement,
+    expected_file,
+    python_version,
+    operating_system,
+    repos=None,
+    as_tree=False,
+    regen=REGEN_TEST_FIXTURES,
+):
+    env = Environment(python_version=python_version, operating_system=operating_system)
+
+    results = list(
+        get_resolved_dependencies(
+            requirements=[requirement],
+            environment=env,
+            repos=repos or get_current_indexes(),
+            as_tree=as_tree,
+        )
+    )
+    check_data_results(results=results, expected_file=expected_file, regen=regen)
+
+
+@pytest.mark.online
+def test_get_resolved_dependencies_with_flask_and_python_310():
+    req = Requirement("flask==2.1.2")
+    req.is_requirement_resolved = True
+
+    expected_file = setup_test_env.get_test_loc(
+        "resolved_deps/flask-310-expected.json", must_exist=False
+    )
+
+    check_get_resolved_dependencies(
+        req,
+        expected_file=expected_file,
+        python_version="310",
+        operating_system="linux",
+        as_tree=False,
+    )
+
+
+@pytest.mark.online
+def test_get_resolved_dependencies_with_flask_and_python_310_windows():
+    req = Requirement("flask==2.1.2")
+    req.is_requirement_resolved = True
+
+    expected_file = setup_test_env.get_test_loc(
+        "resolved_deps/flask-310-win-expected.json", must_exist=False
+    )
+
+    check_get_resolved_dependencies(
+        req,
+        expected_file=expected_file,
+        python_version="310",
+        operating_system="windows",
+        as_tree=False,
+    )
+
+
+@pytest.mark.online
+def test_get_resolved_dependencies_with_flask_and_python_36():
+    req = Requirement("flask")
+    req.is_requirement_resolved = False
+
+    expected_file = setup_test_env.get_test_loc(
+        "resolved_deps/flask-36-expected.json", must_exist=False
+    )
+
+    check_get_resolved_dependencies(
+        req,
+        expected_file=expected_file,
+        python_version="36",
+        operating_system="linux",
+        as_tree=False,
+    )
+
+
+@pytest.mark.online
+def test_get_resolved_dependencies_with_tilde_requirement_using_json_api():
+    req = Requirement("flask~=2.1.2")
+    req.is_requirement_resolved = False
+
+    expected_file = setup_test_env.get_test_loc(
+        "resolved_deps/flask-39-expected.json", must_exist=False
+    )
+
+    check_get_resolved_dependencies(
+        req,
+        expected_file=expected_file,
+        python_version="39",
+        operating_system="linux",
+        as_tree=False,
+    )
+
+
+@pytest.mark.online
+@pytest.mark.skipif(on_mac, reason="torch is only available for linux and windows.")
+def test_get_resolved_dependencies_for_version_containing_local_version_identifier():
+    req = Requirement("torchcodec==0.2.0+cu124")
+    req.is_requirement_resolved = True
+
+    repos = [PypiSimpleRepository(index_url="https://download.pytorch.org/whl")]
+    expected_file = setup_test_env.get_test_loc(
+        "resolved_deps/torch-312-expected.json", must_exist=False
+    )
+
+    check_get_resolved_dependencies(
+        req,
+        expected_file=expected_file,
+        python_version="312",
+        operating_system="linux",
+        repos=repos,
+        as_tree=False,
+    )
+
+
+@pytest.mark.online
+def test_without_supported_wheels():
+    req = Requirement("autobahn==22.3.2")
+    req.is_requirement_resolved = True
+    expected_file = setup_test_env.get_test_loc(
+        "resolved_deps/autobahn-310-expected.json", must_exist=False
+    )
+
+    check_get_resolved_dependencies(
+        req,
+        expected_file=expected_file,
+        python_version="39",
+        operating_system="linux",
+        as_tree=False,
+    )
+
+
+def test_is_valid_version():
+    parsed_version = packvers.version.parse("2.1.2")
+    requirements = {"flask": [Requirement("flask>2.0.0")]}
+    bad_versions = []
+    identifier = "flask"
+    assert is_valid_version(parsed_version, requirements, identifier, bad_versions)
+
+
+def test_is_valid_version_with_no_specifier():
+    parsed_version = packvers.version.parse("2.1.2")
+    requirements = {"flask": [Requirement("flask")]}
+    bad_versions = []
+    identifier = "flask"
+    assert is_valid_version(parsed_version, requirements, identifier, bad_versions)
+
+
+def test_is_valid_version_with_no_specifier_and_pre_release():
+    parsed_version = packvers.version.parse("1.0.0b4")
+    requirements = {"flask": [Requirement("flask")]}
+    bad_versions = []
+    identifier = "flask"
+    assert is_valid_version(parsed_version, requirements, identifier, bad_versions)
+
+
+def test_get_requirements_from_dependencies():
+    dependencies = [
+        models.DependentPackage(
+            purl="pkg:pypi/django",
+            scope="install",
+            is_runtime=True,
+            is_optional=False,
+            is_resolved=False,
+            extracted_requirement="django>=1.11.11",
+            extra_data=dict(
+                is_editable=False,
+                link=None,
+                hash_options=[],
+                is_constraint=False,
+                is_archive=False,
+                is_wheel=False,
+                is_url=False,
+                is_vcs_url=False,
+                is_name_at_url=False,
+                is_local_path=False,
+            ),
+        )
+    ]
+
+    requirements = [str(r) for r in get_requirements_from_dependencies(dependencies)]
+
+    assert requirements == ["django>=1.11.11"]
+
+
+def test_get_requirements_from_dependencies_with_empty_list():
+    assert list(get_requirements_from_dependencies(dependencies=[])) == []
+
+
+def test_get_requirements_from_dependencies_with_editable_requirements():
+    dependencies = [
+        models.DependentPackage(
+            purl="pkg:pypi/django",
+            scope="install",
+            is_runtime=True,
+            is_optional=False,
+            is_resolved=False,
+            extracted_requirement="django>=1.11.11",
+            extra_data=dict(
+                is_editable=True,
+                link=None,
+                hash_options=[],
+                is_constraint=False,
+                is_archive=False,
+                is_wheel=False,
+                is_url=False,
+                is_vcs_url=False,
+                is_name_at_url=False,
+                is_local_path=False,
+            ),
+        )
+    ]
+
+    requirements = [str(r) for r in get_requirements_from_dependencies(dependencies)]
+
+    assert requirements == []
+
+
+def test_get_requirements_from_python_manifest_securely():
+    sdist_location = "tests/data/secure-setup"
+    setup_py_emptyrequires = "setup-emptyrequires.py"
+    setup_py_norequires = "setup-norequires.py"
+    setup_py_requires = "setup-requires.py"
+    analyze_setup_py_insecurely = False
+    try:
+        ret = list(
+            get_requirements_from_python_manifest(
+                sdist_location,
+                sdist_location + "/" + setup_py_norequires,
+                [sdist_location + "/" + setup_py_norequires],
+                analyze_setup_py_insecurely,
+            )
+        )
+        assert ret == []
+    except Exception:
+        pytest.fail("Failure parsing setup.py where requirements are not provided.")
+    try:
+        ret = list(
+            get_requirements_from_python_manifest(
+                sdist_location,
+                sdist_location + "/" + setup_py_emptyrequires,
+                [sdist_location + "/" + setup_py_emptyrequires],
+                analyze_setup_py_insecurely,
+            )
+        )
+        assert ret == []
+    except Exception:
+        pytest.fail("Failure getting empty requirements securely from setup.py.")
+    with pytest.raises(Exception):
+        ret = list(
+            get_requirements_from_python_manifest(
+                sdist_location,
+                sdist_location + "/" + setup_py_requires,
+                [sdist_location + "/" + setup_py_requires],
+                analyze_setup_py_insecurely,
+            ).next()
+        )
+
+
+def test_setup_py_parsing_insecure():
+    setup_py_file = setup_test_env.get_test_loc("insecure-setup/setup.py")
+    reqs = [str(req) for req in list(parse_reqs_from_setup_py_insecurely(setup_py=setup_py_file))]
+    assert reqs == ["isodate", "pyparsing", "six"]
+
+
+def test_setup_py_parsing_insecure_testpkh():
+    setup_py_file = setup_test_env.get_test_loc("insecure-setup-2/setup.py")
+    reqs = [str(req) for req in list(parse_reqs_from_setup_py_insecurely(setup_py=setup_py_file))]
+    assert reqs == [
+        "CairoSVG<2.0.0,>=1.0.20",
+        "click>=5.0.0",
+        "invenio[auth,base,metadata]>=3.0.0",
+        "invenio-records==1.0.*,>=1.0.0",
+        "mock>=1.3.0",
+    ]
+
+
+@patch("python_inspector.resolution.PythonInputProvider.get_versions_for_package")
+def test_iter_matches(mock_versions):
+    repos = get_current_indexes()
+    mock_versions.return_value = []
+    provider = PythonInputProvider(repos=repos)
+    with pytest.raises(NoVersionsFound):
+        list(provider._iter_matches("foo-bar", {"foo-bar": []}, {"foo-bar": []}))