feat(pypi/parse_requirements): get dists by version when no hash provied (#2695)

This pull request modifies the SimpleAPI HTML parsing to add a new
field where we can get the `sha256` values by package version. This
allows us to very easily fallback to all packages of a particular
version when using `experimental_index_url` if the hashes are not
specified.

The code deciding which packages to query the SimpleAPI for has been
also modified to only omit queries for packages that are included via
direct URL references.

If we fail to get the data from the SimpleAPI, we will fallback to
`pip` and try to install it via the legacy behaviour.

Fixes #2023
Work towards #260
Work towards #1357
Work towards #2363

---------

Co-authored-by: Ignas Anikevicius <[email protected]>

Author: Yanpei-Wang

CHANGELOG.md

@@ -81,6 +81,14 @@ Unreleased changes template.
 
 {#v0-0-0-added}
 ### Added
+* (pypi) From now on `sha256` values in the `requirements.txt` is no longer
+  mandatory when enabling {attr}`pip.parse.experimental_index_url` feature.
+  This means that `rules_python` will attempt to fetch metadata for all
+  packages through SimpleAPI unless they are pulled through direct URL
+  references. Fixes [#2023](https://github.com/bazel-contrib/rules_python/issues/2023).
+  In case you see issues with `rules_python` being too eager to fetch the SimpleAPI
+  metadata, you can use the newly added {attr}`pip.parse.experimental_skip_sources`
+  to skip metadata fetching for those packages.
 * (uv) A {obj}`lock` rule that is the replacement for the
   {obj}`compile_pip_requirements`. This may still have rough corners
   so please report issues with it in the

docs/pypi-dependencies.md

@@ -386,11 +386,13 @@ This does not mean that `rules_python` is fetching the wheels eagerly, but it
 rather means that it is calling the PyPI server to get the Simple API response
 to get the list of all available source and wheel distributions. Once it has
 got all of the available distributions, it will select the right ones depending
-on the `sha256` values in your `requirements_lock.txt` file. The compatible
-distribution URLs will be then written to the `MODULE.bazel.lock` file. Currently
-users wishing to use the lock file with `rules_python` with this feature have
-to set an environment variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will
-become default in the next release.
+on the `sha256` values in your `requirements_lock.txt` file. If `sha256` hashes
+are not present in the requirements file, we will fallback to matching by version
+specified in the lock file. The compatible distribution URLs will be then
+written to the `MODULE.bazel.lock` file. Currently users wishing to use the
+lock file with `rules_python` with this feature have to set an environment
+variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will become default in the
+next release.
 
 Fetching the distribution information from the PyPI allows `rules_python` to
 know which `whl` should be used on which target platform and it will determine

python/private/pypi/extension.bzl

@@ -459,13 +459,21 @@ You cannot use both the additive_build_content and additive_build_content_file a
             get_index_urls = None
             if pip_attr.experimental_index_url:
                 is_reproducible = False
+                skip_sources = [
+                    normalize_name(s)
+                    for s in pip_attr.simpleapi_skip
+                ]
                 get_index_urls = lambda ctx, distributions: simpleapi_download(
                     ctx,
                     attr = struct(
                         index_url = pip_attr.experimental_index_url,
                         extra_index_urls = pip_attr.experimental_extra_index_urls or [],
                         index_url_overrides = pip_attr.experimental_index_url_overrides or {},
-                        sources = distributions,
+                        sources = [
+                            d
+                            for d in distributions
+                            if normalize_name(d) not in skip_sources
+                        ],
                         envsubst = pip_attr.envsubst,
                         # Auth related info
                         netrc = pip_attr.netrc,
@@ -682,6 +690,11 @@ This is equivalent to `--index-url` `pip` option.
 If {attr}`download_only` is set, then `sdist` archives will be discarded and `pip.parse` will
 operate in wheel-only mode.
 :::
+
+:::{versionchanged} VERSION_NEXT_FEATURE
+Index metadata will be used to deduct `sha256` values for packages even if the
+`sha256` values are not present in the requirements.txt lock file.
+:::
 """,
         ),
         "experimental_index_url_overrides": attr.string_dict(
@@ -749,6 +762,18 @@ The Python version the dependencies are targetting, in Major.Minor format
 If an interpreter isn't explicitly provided (using `python_interpreter` or
 `python_interpreter_target`), then the version specified here must have
 a corresponding `python.toolchain()` configured.
+""",
+        ),
+        "simpleapi_skip": attr.string_list(
+            doc = """\
+The list of packages to skip fetching metadata for from SimpleAPI index. You should
+normally not need this attribute, but in case you do, please report this as a bug
+to `rules_python` and use this attribute until the bug is fixed.
+
+EXPERIMENTAL: this may be removed without notice.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
 """,
         ),
         "whl_modifications": attr.label_keyed_string_dict(

python/private/pypi/parse_requirements.bzl

@@ -184,7 +184,7 @@ def parse_requirements(
                 req.distribution: None
                 for reqs in requirements_by_platform.values()
                 for req in reqs.values()
-                if req.srcs.shas
+                if not req.srcs.url
             }),
         )
 
@@ -315,10 +315,15 @@ def _add_dists(*, requirement, index_urls, logger = None):
     whls = []
     sdist = None
 
-    # TODO @aignas 2024-05-22: it is in theory possible to add all
-    # requirements by version instead of by sha256. This may be useful
-    # for some projects.
-    for sha256 in requirement.srcs.shas:
+    # First try to find distributions by SHA256 if provided
+    shas_to_use = requirement.srcs.shas
+    if not shas_to_use:
+        version = requirement.srcs.version
+        shas_to_use = index_urls.sha256s_by_version.get(version, [])
+        if logger:
+            logger.warn(lambda: "requirement file has been generated without hashes, will use all hashes for the given version {} that could find on the index:\n    {}".format(version, shas_to_use))
+
+    for sha256 in shas_to_use:
         # For now if the artifact is marked as yanked we just ignore it.
         #
         # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api

python/private/pypi/parse_simpleapi_html.bzl

@@ -26,6 +26,7 @@ def parse_simpleapi_html(*, url, content):
     Returns:
         A list of structs with:
         * filename: The filename of the artifact.
+        * version: The version of the artifact.
         * url: The URL to download the artifact.
         * sha256: The sha256 of the artifact.
         * metadata_sha256: The whl METADATA sha256 if we can download it. If this is
@@ -51,15 +52,20 @@ def parse_simpleapi_html(*, url, content):
 
     # Each line follows the following pattern
     # <a href="https://...#sha256=..." attribute1="foo" ... attributeN="bar">filename</a><br />
+    sha256_by_version = {}
     for line in lines[1:]:
         dist_url, _, tail = line.partition("#sha256=")
+        dist_url = _absolute_url(url, dist_url)
+
         sha256, _, tail = tail.partition("\"")
 
         # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
         yanked = "data-yanked" in line
 
         head, _, _ = tail.rpartition("</a>")
         maybe_metadata, _, filename = head.rpartition(">")
+        version = _version(filename)
+        sha256_by_version.setdefault(version, []).append(sha256)
 
         metadata_sha256 = ""
         metadata_url = ""
@@ -75,7 +81,8 @@ def parse_simpleapi_html(*, url, content):
         if filename.endswith(".whl"):
             whls[sha256] = struct(
                 filename = filename,
-                url = _absolute_url(url, dist_url),
+                version = version,
+                url = dist_url,
                 sha256 = sha256,
                 metadata_sha256 = metadata_sha256,
                 metadata_url = _absolute_url(url, metadata_url) if metadata_url else "",
@@ -84,7 +91,8 @@ def parse_simpleapi_html(*, url, content):
         else:
             sdists[sha256] = struct(
                 filename = filename,
-                url = _absolute_url(url, dist_url),
+                version = version,
+                url = dist_url,
                 sha256 = sha256,
                 metadata_sha256 = "",
                 metadata_url = "",
@@ -94,8 +102,31 @@ def parse_simpleapi_html(*, url, content):
     return struct(
         sdists = sdists,
         whls = whls,
+        sha256_by_version = sha256_by_version,
     )
 
+_SDIST_EXTS = [
+    ".tar",  # handles any compression
+    ".zip",
+]
+
+def _version(filename):
+    # See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#binary-distribution-format
+
+    _, _, tail = filename.partition("-")
+    version, _, _ = tail.partition("-")
+    if version != tail:
+        # The format is {name}-{version}-{whl_specifiers}.whl
+        return version
+
+    # NOTE @aignas 2025-03-29: most of the files are wheels, so this is not the common path
+
+    # {name}-{version}.{ext}
+    for ext in _SDIST_EXTS:
+        version, _, _ = version.partition(ext)  # build or name
+
+    return version
+
 def _get_root_directory(url):
     scheme_end = url.find("://")
     if scheme_end == -1:

python/private/pypi/simpleapi_download.bzl

@@ -127,10 +127,17 @@ def simpleapi_download(
 
     failed_sources = [pkg for pkg in attr.sources if pkg not in found_on_index]
     if failed_sources:
-        _fail("Failed to download metadata for {} for from urls: {}".format(
-            failed_sources,
-            index_urls,
-        ))
+        _fail(
+            "\n".join([
+                "Failed to download metadata for {} for from urls: {}.".format(
+                    failed_sources,
+                    index_urls,
+                ),
+                "If you would like to skip downloading metadata for these packages please add 'simpleapi_skip={}' to your 'pip.parse' call.".format(
+                    render.list(failed_sources),
+                ),
+            ]),
+        )
         return None
 
     if warn_overrides:

python/private/pypi/whl_library.bzl

@@ -270,6 +270,12 @@ def _whl_library_impl(rctx):
             sha256 = rctx.attr.sha256,
             auth = get_auth(rctx, urls),
         )
+        if not rctx.attr.sha256:
+            # this is only seen when there is a direct URL reference without sha256
+            logger.warn("Please update the requirement line to include the hash:\n{} \\\n    --hash=sha256:{}".format(
+                rctx.attr.requirement,
+                result.sha256,
+            ))
 
         if not result.success:
             fail("could not download the '{}' from {}:\n{}".format(filename, urls, result))

python/private/pypi/whl_repo_name.bzl

@@ -32,11 +32,19 @@ def whl_repo_name(filename, sha256):
 
     if not filename.endswith(".whl"):
         # Then the filename is basically foo-3.2.1.<ext>
-        parts.append(normalize_name(filename.rpartition("-")[0]))
-        parts.append("sdist")
+        name, _, tail = filename.rpartition("-")
+        parts.append(normalize_name(name))
+        if sha256:
+            parts.append("sdist")
+            version = ""
+        else:
+            for ext in [".tar", ".zip"]:
+                tail, _, _ = tail.partition(ext)
+            version = tail.replace(".", "_").replace("!", "_")
     else:
         parsed = parse_whl_name(filename)
         name = normalize_name(parsed.distribution)
+        version = parsed.version.replace(".", "_").replace("!", "_")
         python_tag, _, _ = parsed.python_tag.partition(".")
         abi_tag, _, _ = parsed.abi_tag.partition(".")
         platform_tag, _, _ = parsed.platform_tag.partition(".")
@@ -46,7 +54,10 @@ def whl_repo_name(filename, sha256):
         parts.append(abi_tag)
         parts.append(platform_tag)
 
-    parts.append(sha256[:8])
+    if sha256:
+        parts.append(sha256[:8])
+    elif version:
+        parts.insert(1, version)
 
     return "_".join(parts)