Merge pull request #201 from ukaj/fix/unsupported_url_scheme

fix: Incorrect parsing of index_urls in requirements.txt

Author: TG1999

src/python_inspector/api.py

@@ -155,8 +155,7 @@ def resolve_dependencies(
     for req_file in requirement_files:
         deps = dependencies.get_dependencies_from_requirements(requirements_file=req_file)
         for extra_data in dependencies.get_extra_data_from_requirements(requirements_file=req_file):
-            index_urls = (*index_urls, *tuple(extra_data.get("extra_index_urls") or []))
-            index_urls = (*index_urls, *tuple(extra_data.get("index_url") or []))
+            index_urls = get_index_urls(index_urls, extra_data)
         direct_dependencies.extend(deps)
         package_data = [
             pkg_data.to_dict() for pkg_data in PipRequirementsFileHandler.parse(location=req_file)
@@ -320,6 +319,25 @@ async def get_pypi_data(package):
     )
 
 
+def get_index_urls(index_urls: Tuple, extra_data: Dict) -> Tuple:
+    """
+    Return a list of index URLs from the extra_data.
+    The extra_data is a dictionary that may contain
+    "extra_index_urls" and "index_url" keys.
+    """
+    if isinstance(index_urls, str):
+        index_urls = [index_urls]
+    if isinstance(index_urls, tuple):
+        index_urls = list(index_urls)
+    extra_index_urls = extra_data.get("extra_index_urls") or []
+    index_url = extra_data.get("index_url") or []
+    if isinstance(extra_index_urls, list):
+        index_urls = (*index_urls, *tuple(extra_index_urls))
+    if isinstance(index_url, str):
+        index_urls = (*index_urls, *tuple([index_url]))
+    return tuple(set(index_urls))
+
+
 resolver_api = resolve_dependencies
 
 

tests/data/azure-devops.req-310-expected.json

@@ -224,12 +224,12 @@
       "type": "pypi",
       "namespace": null,
       "name": "packaging",
-      "version": "24.2",
+      "version": "25.0",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "Core utilities for Python packages\npackaging\n=========\n\n.. start-intro\n\nReusable core utilities for various Python Packaging\n`interoperability specifications <https://packaging.python.org/specifications/>`_.\n\nThis library provides utilities that implement the interoperability\nspecifications which have clearly one correct behaviour (eg: :pep:`440`)\nor benefit greatly from having a single shared implementation (eg: :pep:`425`).\n\n.. end-intro\n\nThe ``packaging`` project includes the following: version handling, specifiers,\nmarkers, requirements, tags, utilities.\n\nDocumentation\n-------------\n\nThe `documentation`_ provides information and the API for the following:\n\n- Version Handling\n- Specifiers\n- Markers\n- Requirements\n- Tags\n- Utilities\n\nInstallation\n------------\n\nUse ``pip`` to install these utilities::\n\n    pip install packaging\n\nThe ``packaging`` library uses calendar-based versioning (``YY.N``).\n\nDiscussion\n----------\n\nIf you run into bugs, you can file them in our `issue tracker`_.\n\nYou can also join ``#pypa`` on Freenode to ask questions or get involved.\n\n\n.. _`documentation`: https://packaging.pypa.io/\n.. _`issue tracker`: https://github.com/pypa/packaging/issues\n\n\nCode of Conduct\n---------------\n\nEveryone interacting in the packaging project's codebases, issue trackers, chat\nrooms, and mailing lists is expected to follow the `PSF Code of Conduct`_.\n\n.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md\n\nContributing\n------------\n\nThe ``CONTRIBUTING.rst`` file outlines how to contribute to this project as\nwell as how to report a potential security issue. The documentation for this\nproject also covers information about `project development`_ and `security`_.\n\n.. _`project development`: https://packaging.pypa.io/en/latest/development/\n.. _`security`: https://packaging.pypa.io/en/latest/security/\n\nProject History\n---------------\n\nPlease review the ``CHANGELOG.rst`` file or the `Changelog documentation`_ for\nrecent changes and project history.\n\n.. _`Changelog documentation`: https://packaging.pypa.io/en/latest/changelog/",
-      "release_date": "2024-11-08T09:47:44",
+      "release_date": "2025-04-19T11:48:57",
       "parties": [
         {
           "type": "person",
@@ -256,11 +256,11 @@
         "Typing :: Typed"
       ],
       "homepage_url": null,
-      "download_url": "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl",
-      "size": 65451,
+      "download_url": "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl",
+      "size": 66469,
       "sha1": null,
-      "md5": "137b07612433f1ad2cd27dd8ab38ce49",
-      "sha256": "09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759",
+      "md5": "5fa4842e2eb0d7883b4b0e7c42d6229e",
+      "sha256": "29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484",
       "sha512": null,
       "bug_tracking_url": null,
       "code_view_url": "https://github.com/pypa/packaging",
@@ -280,9 +280,9 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/packaging/24.2/json",
+      "api_data_url": "https://pypi.org/pypi/packaging/25.0/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/packaging@24.2"
+      "purl": "pkg:pypi/packaging@25.0"
     },
     {
       "type": "pypi",
@@ -491,7 +491,7 @@
       "dependencies": []
     },
     {
-      "package": "pkg:pypi/packaging@24.2",
+      "package": "pkg:pypi/packaging@25.0",
       "dependencies": []
     },
     {
@@ -503,7 +503,7 @@
       "dependencies": [
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
-        "pkg:pypi/packaging@24.2",
+        "pkg:pypi/packaging@25.0",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]"
       ]

tests/data/azure-devops.req-312-expected.json

@@ -879,12 +879,12 @@
       "type": "pypi",
       "namespace": null,
       "name": "boolean-py",
-      "version": "4.0",
+      "version": "5.0",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "Define boolean algebras, create and parse boolean expressions and create custom boolean DSL.\nThis library helps you deal with boolean expressions and algebra with variables\nand the boolean functions AND, OR, NOT.\n\nYou can parse expressions from strings and simplify and compare expressions.\nYou can also easily create your custom algreba and mini DSL and create custom\ntokenizers to handle custom expressions.\n\nFor extensive documentation look either into the docs directory or view it online, at\nhttps://booleanpy.readthedocs.org/en/latest/\n\nhttps://github.com/bastikr/boolean.py\n\nCopyright (c) 2009-2020 Sebastian Kraemer, [email protected] and others\nSPDX-License-Identifier: BSD-2-Clause",
-      "release_date": "2022-05-05T08:18:58",
+      "release_date": "2025-04-03T10:39:48",
       "parties": [
         {
           "type": "person",
@@ -910,11 +910,11 @@
         "Topic :: Utilities"
       ],
       "homepage_url": "https://github.com/bastikr/boolean.py",
-      "download_url": "https://files.pythonhosted.org/packages/3f/02/6389ef0529af6da0b913374dedb9bbde8eabfe45767ceec38cc37801b0bd/boolean.py-4.0-py3-none-any.whl",
-      "size": 25909,
+      "download_url": "https://files.pythonhosted.org/packages/e5/ca/78d423b324b8d77900030fa59c4aa9054261ef0925631cd2501dd015b7b7/boolean_py-5.0-py3-none-any.whl",
+      "size": 26577,
       "sha1": null,
-      "md5": "73208a6fc38d6904a1d7e793e8da1292",
-      "sha256": "2876f2051d7d6394a531d82dc6eb407faa0b01a0a0b3083817ccd7323b8d96bd",
+      "md5": "df9060a88bfb6ba3b9314783b16a0faa",
+      "sha256": "ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9",
       "sha512": null,
       "bug_tracking_url": null,
       "code_view_url": null,
@@ -931,9 +931,9 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/boolean-py/4.0/json",
+      "api_data_url": "https://pypi.org/pypi/boolean-py/5.0/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/boolean-py@4.0"
+      "purl": "pkg:pypi/boolean-py@5.0"
     },
     {
       "type": "pypi",
@@ -2820,7 +2820,7 @@
         {
           "key": "boolean-py",
           "package_name": "boolean-py",
-          "installed_version": "4.0",
+          "installed_version": "5.0",
           "dependencies": []
         },
         {
@@ -2856,7 +2856,7 @@
             {
               "key": "boolean-py",
               "package_name": "boolean-py",
-              "installed_version": "4.0",
+              "installed_version": "5.0",
               "dependencies": []
             }
           ]

tests/data/azure-devops.req-313-expected.json

@@ -879,12 +879,12 @@
       "type": "pypi",
       "namespace": null,
       "name": "boolean-py",
-      "version": "4.0",
+      "version": "5.0",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "Define boolean algebras, create and parse boolean expressions and create custom boolean DSL.\nThis library helps you deal with boolean expressions and algebra with variables\nand the boolean functions AND, OR, NOT.\n\nYou can parse expressions from strings and simplify and compare expressions.\nYou can also easily create your custom algreba and mini DSL and create custom\ntokenizers to handle custom expressions.\n\nFor extensive documentation look either into the docs directory or view it online, at\nhttps://booleanpy.readthedocs.org/en/latest/\n\nhttps://github.com/bastikr/boolean.py\n\nCopyright (c) 2009-2020 Sebastian Kraemer, [email protected] and others\nSPDX-License-Identifier: BSD-2-Clause",
-      "release_date": "2022-05-05T08:18:58",
+      "release_date": "2025-04-03T10:39:48",
       "parties": [
         {
           "type": "person",
@@ -910,11 +910,11 @@
         "Topic :: Utilities"
       ],
       "homepage_url": "https://github.com/bastikr/boolean.py",
-      "download_url": "https://files.pythonhosted.org/packages/3f/02/6389ef0529af6da0b913374dedb9bbde8eabfe45767ceec38cc37801b0bd/boolean.py-4.0-py3-none-any.whl",
-      "size": 25909,
+      "download_url": "https://files.pythonhosted.org/packages/e5/ca/78d423b324b8d77900030fa59c4aa9054261ef0925631cd2501dd015b7b7/boolean_py-5.0-py3-none-any.whl",
+      "size": 26577,
       "sha1": null,
-      "md5": "73208a6fc38d6904a1d7e793e8da1292",
-      "sha256": "2876f2051d7d6394a531d82dc6eb407faa0b01a0a0b3083817ccd7323b8d96bd",
+      "md5": "df9060a88bfb6ba3b9314783b16a0faa",
+      "sha256": "ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9",
       "sha512": null,
       "bug_tracking_url": null,
       "code_view_url": null,
@@ -931,9 +931,9 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/boolean-py/4.0/json",
+      "api_data_url": "https://pypi.org/pypi/boolean-py/5.0/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/boolean-py@4.0"
+      "purl": "pkg:pypi/boolean-py@5.0"
     },
     {
       "type": "pypi",
@@ -2810,7 +2810,7 @@
       "package": "pkg:pypi/[email protected]",
       "dependencies": [
         "pkg:pypi/[email protected]",
-        "pkg:pypi/boolean-py@4.0",
+        "pkg:pypi/boolean-py@5.0",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
@@ -2831,7 +2831,7 @@
       ]
     },
     {
-      "package": "pkg:pypi/boolean-py@4.0",
+      "package": "pkg:pypi/boolean-py@5.0",
       "dependencies": []
     },
     {
@@ -2897,7 +2897,7 @@
     {
       "package": "pkg:pypi/[email protected]",
       "dependencies": [
-        "pkg:pypi/boolean-py@4.0"
+        "pkg:pypi/boolean-py@5.0"
       ]
     },
     {

tests/data/azure-devops.req-38-expected.json

@@ -3727,12 +3727,12 @@
       "type": "pypi",
       "namespace": null,
       "name": "pip",
-      "version": "25.0.1",
+      "version": "25.1",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "The PyPA recommended tool for installing Python packages.\npip - The Python Package Installer\n==================================\n\n.. |pypi-version| image:: https://img.shields.io/pypi/v/pip.svg\n   :target: https://pypi.org/project/pip/\n   :alt: PyPI\n\n.. |python-versions| image:: https://img.shields.io/pypi/pyversions/pip\n   :target: https://pypi.org/project/pip\n   :alt: PyPI - Python Version\n\n.. |docs-badge| image:: https://readthedocs.org/projects/pip/badge/?version=latest\n   :target: https://pip.pypa.io/en/latest\n   :alt: Documentation\n\n|pypi-version| |python-versions| |docs-badge|\n\npip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes.\n\nPlease take a look at our documentation for how to install and use pip:\n\n* `Installation`_\n* `Usage`_\n\nWe release updates regularly, with a new version every 3 months. Find more details in our documentation:\n\n* `Release notes`_\n* `Release process`_\n\nIf you find bugs, need help, or want to talk to the developers, please use our mailing lists or chat rooms:\n\n* `Issue tracking`_\n* `Discourse channel`_\n* `User IRC`_\n\nIf you want to get involved head over to GitHub to get the source code, look at our development documentation and feel free to jump on the developer mailing lists and chat rooms:\n\n* `GitHub page`_\n* `Development documentation`_\n* `Development IRC`_\n\nCode of Conduct\n---------------\n\nEveryone interacting in the pip project's codebases, issue trackers, chat\nrooms, and mailing lists is expected to follow the `PSF Code of Conduct`_.\n\n.. _package installer: https://packaging.python.org/guides/tool-recommendations/\n.. _Python Package Index: https://pypi.org\n.. _Installation: https://pip.pypa.io/en/stable/installation/\n.. _Usage: https://pip.pypa.io/en/stable/\n.. _Release notes: https://pip.pypa.io/en/stable/news.html\n.. _Release process: https://pip.pypa.io/en/latest/development/release-process/\n.. _GitHub page: https://github.com/pypa/pip\n.. _Development documentation: https://pip.pypa.io/en/latest/development\n.. _Issue tracking: https://github.com/pypa/pip/issues\n.. _Discourse channel: https://discuss.python.org/c/packaging\n.. _User IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa\n.. _Development IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa-dev\n.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md",
-      "release_date": "2025-02-09T17:14:01",
+      "release_date": "2025-04-26T09:45:33",
       "parties": [
         {
           "type": "person",
@@ -3752,18 +3752,17 @@
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: 3.13",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Topic :: Software Development :: Build Tools"
       ],
       "homepage_url": null,
-      "download_url": "https://files.pythonhosted.org/packages/c9/bc/b7db44f5f39f9d0494071bddae6880eb645970366d0a200022a1a93d57f5/pip-25.0.1-py3-none-any.whl",
-      "size": 1841526,
+      "download_url": "https://files.pythonhosted.org/packages/e0/f0/8a2806114cd36e282823fd4d8e88e3b94dc943c2569c350d0c826a49db38/pip-25.1-py3-none-any.whl",
+      "size": 1824948,
       "sha1": null,
-      "md5": "99f43f22d5321305507b804a2be662c0",
-      "sha256": "c46efd13b6aa8279f33f2864459c8ce587ea6a1a59ee20de055868d8f7688f7f",
+      "md5": "4eb2baef792e6841cfcd4c24c808421a",
+      "sha256": "13b4aa0aaad055020a11bec8a1c2a70a2b2d080e12d89b962266029fff0a16ba",
       "sha512": null,
       "bug_tracking_url": null,
       "code_view_url": "https://github.com/pypa/pip",
@@ -3783,9 +3782,9 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/pip/25.0.1/json",
+      "api_data_url": "https://pypi.org/pypi/pip/25.1/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/pip@25.0.1"
+      "purl": "pkg:pypi/[email protected]"
     },
     {
       "type": "pypi",
@@ -6163,13 +6162,13 @@
       ]
     },
     {
-      "package": "pkg:pypi/pip@25.0.1",
+      "package": "pkg:pypi/[email protected]",
       "dependencies": []
     },
     {
       "package": "pkg:pypi/[email protected]",
       "dependencies": [
-        "pkg:pypi/pip@25.0.1"
+        "pkg:pypi/[email protected]"
       ]
     },
     {

tests/data/example-requirements-ignore-errors-expected.json

@@ -15,6 +15,7 @@
 from commoncode.testcase import FileDrivenTesting
 from test_cli import check_data_results
 
+from python_inspector.api import get_index_urls
 from python_inspector.api import resolver_api
 
 test_env = FileDrivenTesting()
@@ -128,3 +129,31 @@ def test_api_with_partial_setup_py():
         analyze_setup_py_insecurely=True,
     )
     check_data_results(results=results.to_dict(generic_paths=True), expected_file=expected_file)
+
+
+def test_get_index_urls():
+
+    # pass as a tuple
+    index_urls = get_index_urls(
+        index_urls=("https://pypi.org/simple",),
+        extra_data={"extra_index_urls": ["https://pypi.org/simple", "https://example.com/simple"]},
+    )
+
+    expected = ["https://example.com/simple", "https://pypi.org/simple"]
+
+    assert sorted(list(index_urls)) == expected
+
+    # pass as a string
+    index_urls = get_index_urls(
+        index_urls=("https://pypi.org/simple"),
+        extra_data={"extra_index_urls": ["https://pypi.org/simple", "https://example.com/simple"]},
+    )
+
+    assert sorted(list(index_urls)) == expected
+
+    index_urls = get_index_urls(
+        index_urls=("https://pypi.org/simple",),
+        extra_data={"index_url": "https://pypi.org/simple"},
+    )
+
+    assert index_urls == ("https://pypi.org/simple",)

tests/data/pinned-pdt-requirements.txt-expected.json

@@ -123,7 +123,7 @@ def test_get_resolved_dependencies_with_tilde_requirement_using_json_api():
     assert plist == [
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
-        "pkg:pypi/importlib-metadata@8.6.1",
+        "pkg:pypi/importlib-metadata@8.7.0",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
@@ -155,7 +155,7 @@ def test_get_resolved_dependencies_for_version_containing_local_version_identifi
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
-        "pkg:pypi/[email protected].1",
+        "pkg:pypi/[email protected].3",
         "pkg:pypi/[email protected]%2Bcpu",
         "pkg:pypi/[email protected]",
     ]