feat(bzlmod): introduce pypi_index for using bazel's downloader

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 

BUILD.bazel

@@ -35,6 +35,7 @@ filegroup(
         "BUILD.bazel",
         "MODULE.bazel",
         "WORKSPACE",
+        "WORKSPACE.bzlmod",
         "internal_deps.bzl",
         "internal_setup.bzl",
         "version.bzl",

CHANGELOG.md

@@ -19,6 +19,8 @@ A brief description of the categories of changes:
 
 ## Unreleased
 
+[0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
+
 ### Changed
 
 ### Fixed
@@ -32,8 +34,10 @@ A brief description of the categories of changes:
 * (gazelle) Added a new `python_default_visibility` directive to control the
   _default_ visibility of generated targets. See the [docs][python_default_visibility]
   for details.
+* (bzlmod) New **experimental** `pypi_index` extension that can be used to
+  instruct the `pip.parse` tag class to use the bazel downloader to fetch
+  wheels. Note, the API is very unstable and may be changed at any time.
 
-[0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
 [python_default_visibility]: gazelle/README.md#directive-python_default_visibility
 
 ### Changed

MODULE.bazel

@@ -4,15 +4,15 @@ module(
     compatibility_level = 1,
 )
 
-bazel_dep(name = "bazel_features", version = "1.1.1")
+bazel_dep(name = "bazel_features", version = "1.9.0")
 bazel_dep(name = "bazel_skylib", version = "1.3.0")
 bazel_dep(name = "platforms", version = "0.0.4")
 
 # Those are loaded only when using py_proto_library
 bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
 bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
 
-internal_deps = use_extension("@rules_python//python/private/bzlmod:internal_deps.bzl", "internal_deps")
+internal_deps = use_extension("//python/private/bzlmod:internal_deps.bzl", "internal_deps")
 internal_deps.install()
 use_repo(
     internal_deps,
@@ -38,7 +38,7 @@ use_repo(
 
 # We need to do another use_extension call to expose the "pythons_hub"
 # repo.
-python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python = use_extension("//python/extensions:python.bzl", "python")
 
 # The default toolchain to use if nobody configures a toolchain.
 # NOTE: This is not a stable version. It is provided for convenience, but will
@@ -53,9 +53,36 @@ use_repo(python, "pythons_hub")
 # This call registers the Python toolchains.
 register_toolchains("@pythons_hub//:all")
 
+# This call registers the `pypi_index` extension so that it can be used in the `pip` extension
+pypi_index = use_extension("//python/extensions:pypi_index.bzl", "pypi_index")
+use_repo(pypi_index, "pypi_index")
+
 # ===== DEV ONLY DEPS AND SETUP BELOW HERE =====
 bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
 bazel_dep(name = "rules_bazel_integration_test", version = "0.20.0", dev_dependency = True)
+bazel_dep(name = "rules_testing", version = "0.5.0", dev_dependency = True)
+bazel_dep(name = "rules_cc", version = "0.0.9", dev_dependency = True)
+
+# Extra gazelle deps
+bazel_dep(name = "rules_go", version = "0.41.0", dev_dependency = True, repo_name = "io_bazel_rules_go")
+bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True, repo_name = "bazel_gazelle")
+
+# This call additionally only adds items to the `pypi_index` if we are
+# not ignoring dev dependencies, making it no-op for the regular usage.
+dev_pypi_index = use_extension(
+    "//python/extensions:pypi_index.bzl",
+    "pypi_index",
+    dev_dependency = True,
+)
+dev_pypi_index.add_requirements(
+    srcs = [
+        # List all of the requirements files used by us
+        "//docs/sphinx:requirements.txt",
+        "//tools/publish:requirements_darwin.txt",
+        "//tools/publish:requirements.txt",
+        "//tools/publish:requirements_windows.txt",
+    ],
+)
 
 dev_pip = use_extension(
     "//python/extensions:pip.bzl",
@@ -77,6 +104,7 @@ dev_pip.parse(
     python_version = "3.11",
     requirements_lock = "//docs/sphinx:requirements.txt",
 )
+use_repo(dev_pip, "dev_pip")
 
 bazel_binaries = use_extension(
     "@rules_bazel_integration_test//:extensions.bzl",

WORKSPACE.bzlmod

@@ -0,0 +1,99 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file contains everything that is needed when using bzlmod
+workspace(name = "rules_python")
+
+load("//python:repositories.bzl", "python_register_multi_toolchains")
+load("//python:versions.bzl", "MINOR_MAPPING")
+
+python_register_multi_toolchains(
+    name = "python",
+    default_version = MINOR_MAPPING.values()[-2],
+    python_versions = MINOR_MAPPING.values(),
+)
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
+
+# Used for Bazel CI
+http_archive(
+    name = "bazelci_rules",
+    sha256 = "eca21884e6f66a88c358e580fd67a6b148d30ab57b1680f62a96c00f9bc6a07e",
+    strip_prefix = "bazelci_rules-1.0.0",
+    url = "https://github.com/bazelbuild/continuous-integration/releases/download/rules-1.0.0/bazelci_rules-1.0.0.tar.gz",
+)
+
+load("@bazelci_rules//:rbe_repo.bzl", "rbe_preconfig")
+
+# Creates a default toolchain config for RBE.
+# Use this as is if you are using the rbe_ubuntu16_04 container,
+# otherwise refer to RBE docs.
+rbe_preconfig(
+    name = "buildkite_config",
+    toolchain = "ubuntu1804-bazel-java11",
+)
+
+local_repository(
+    name = "rules_python_gazelle_plugin",
+    path = "gazelle",
+)
+
+# The rules_python gazelle extension has some third-party go dependencies
+# which we need to fetch in order to compile it.
+load("@rules_python_gazelle_plugin//:deps.bzl", _py_gazelle_deps = "gazelle_deps")
+
+# See: https://github.com/bazelbuild/rules_python/blob/main/gazelle/README.md
+# This rule loads and compiles various go dependencies that running gazelle
+# for python requirements.
+_py_gazelle_deps()
+
+# This interpreter is used for various rules_python dev-time tools
+load("@python//3.11.8:defs.bzl", "interpreter")
+
+#####################
+# Install twine for our own runfiles wheel publishing.
+# Eventually we might want to install twine automatically for users too, see:
+# https://github.com/bazelbuild/rules_python/issues/1016.
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+pip_parse(
+    name = "publish_deps",
+    python_interpreter_target = interpreter,
+    requirements_darwin = "//tools/publish:requirements_darwin.txt",
+    requirements_lock = "//tools/publish:requirements.txt",
+    requirements_windows = "//tools/publish:requirements_windows.txt",
+)
+
+load("@publish_deps//:requirements.bzl", "install_deps")
+
+install_deps()
+
+#####################
+
+# This wheel is purely here to validate the wheel extraction code. It's not
+# intended for anything else.
+http_file(
+    name = "wheel_for_testing",
+    downloaded_file_path = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+    sha256 = "0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2",
+    urls = [
+        "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
+    ],
+)
+
+# rules_proto expects //external:python_headers to point at the python headers.
+bind(
+    name = "python_headers",
+    actual = "//python/cc:current_py_cc_headers",
+)

docs/sphinx/BUILD.bazel

@@ -91,6 +91,7 @@ sphinx_stardocs(
     } if IS_BAZEL_7_OR_HIGHER else {}) | ({
         # This depends on @pythons_hub, which is only created under bzlmod,
         "api/extensions/pip.md": "//python/extensions:pip_bzl",
+        "api/extensions/pypi_index.md": "//python/extensions:pypi_index_bzl",
     } if IS_BAZEL_7_OR_HIGHER and BZLMOD_ENABLED else {}),
     footer = "_stardoc_footer.md",
     tags = ["docs"],

examples/bzlmod/MODULE.bazel

@@ -43,6 +43,30 @@ python.toolchain(
 # rules based on the `python_version` arg values.
 use_repo(python, "python_3_10", "python_3_9", "python_versions")
 
+# This extension allows rules_python to optimize downloading for packages by checking
+# for available artifacts on PyPI Simple API compatible mirrors.
+pypi_index = use_extension("@rules_python//python/extensions:pypi_index.bzl", "pypi_index")
+pypi_index.add_requirements(
+    srcs = [
+        "//:requirements_lock_3_10.txt",
+        "//:requirements_lock_3_9.txt",
+        "//:requirements_windows_3_10.txt",
+        "//:requirements_windows_3_9.txt",
+    ],
+)
+
+# We can also initialize the extension in dev mode.
+dev_pypi_index = use_extension(
+    "@rules_python//python/extensions:pypi_index.bzl",
+    "pypi_index",
+    dev_dependency = True,
+)
+dev_pypi_index.add_requirements(
+    srcs = [
+        "//tests/dupe_requirements:requirements.txt",
+    ],
+)
+
 # This extension allows a user to create modifications to how rules_python
 # creates different wheel repositories.  Different attributes allow the user
 # to modify the BUILD file, and copy files.

python/extensions/BUILD.bazel

@@ -31,6 +31,13 @@ bzl_library(
     deps = ["//python/private/bzlmod:pip_bzl"],
 )
 
+bzl_library(
+    name = "pypi_index_bzl",
+    srcs = ["pypi_index.bzl"],
+    visibility = ["//:__subpackages__"],
+    deps = ["//python/private/bzlmod:pypi_index_bzl"],
+)
+
 bzl_library(
     name = "python_bzl",
     srcs = ["python.bzl"],

python/extensions/pypi_index.bzl

@@ -0,0 +1,19 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""See the doc in the implementation file."""
+
+load("//python/private/bzlmod:pypi_index.bzl", _pypi_index = "pypi_index")
+
+pypi_index = _pypi_index

python/pip_install/pip_repository.bzl

@@ -765,18 +765,27 @@ def _whl_library_impl(rctx):
     # Manually construct the PYTHONPATH since we cannot use the toolchain here
     environment = _create_repository_execution_environment(rctx, python_interpreter)
 
-    repo_utils.execute_checked(
-        rctx,
-        op = "whl_library.ResolveRequirement({}, {})".format(rctx.attr.name, rctx.attr.requirement),
-        arguments = args,
-        environment = environment,
-        quiet = rctx.attr.quiet,
-        timeout = rctx.attr.timeout,
-    )
+    if rctx.attr.whl_file:
+        whl_path = rctx.path(rctx.attr.whl_file)
+        if not whl_path.exists:
+            fail("The given whl '{}' does not exist".format(rctx.attr.whl_file))
+
+        # Simulate the behaviour where the whl is present in the current directory.
+        rctx.symlink(whl_path, whl_path.basename)
+        whl_path = rctx.path(whl_path.basename)
+    else:
+        repo_utils.execute_checked(
+            rctx,
+            op = "whl_library.ResolveRequirement({}, {})".format(rctx.attr.name, rctx.attr.requirement),
+            arguments = args,
+            environment = environment,
+            quiet = rctx.attr.quiet,
+            timeout = rctx.attr.timeout,
+        )
 
-    whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
-    if not rctx.delete("whl_file.json"):
-        fail("failed to delete the whl_file.json file")
+        whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
+        if not rctx.delete("whl_file.json"):
+            fail("failed to delete the whl_file.json file")
 
     if rctx.attr.whl_patches:
         patches = {}
@@ -910,6 +919,12 @@ whl_library_attrs = {
         mandatory = True,
         doc = "Python requirement string describing the package to make available",
     ),
+    "whl_file": attr.label(
+        doc = """\
+The wheel file label to be used for this installation. This will not use pip to download the
+whl and instead use the supplied file. Note that the label needs to point to a single file.
+""",
+    ),
     "whl_patches": attr.label_keyed_string_dict(
         doc = """a label-keyed-string dict that has
             json.encode(struct([whl_file], patch_strip]) as values. This

python/private/BUILD.bazel

@@ -119,6 +119,17 @@ bzl_library(
     srcs = ["parse_whl_name.bzl"],
 )
 
+bzl_library(
+    name = "pypi_index_bzl",
+    srcs = ["pypi_index.bzl"],
+    deps = [
+        ":auth_bzl",
+        ":normalize_name_bzl",
+        ":text_util_bzl",
+        "//python/pip_install:requirements_parser_bzl",
+    ],
+)
+
 bzl_library(
     name = "py_cc_toolchain_bzl",
     srcs = [

python/private/auth.bzl

@@ -33,10 +33,13 @@ def get_auth(rctx, urls):
     Returns:
         dict: A map of authentication parameters by URL.
     """
-    if rctx.attr.netrc:
-        netrc = read_netrc(rctx, rctx.attr.netrc)
+    attr = getattr(rctx, "attr", None)
+
+    if getattr(attr, "netrc", None):
+        netrc = read_netrc(rctx, getattr(attr, "netrc"))
     elif "NETRC" in rctx.os.environ:
         netrc = read_netrc(rctx, rctx.os.environ["NETRC"])
     else:
         netrc = read_user_netrc(rctx)
-    return use_netrc(netrc, urls, rctx.attr.auth_patterns)
+
+    return use_netrc(netrc, urls, getattr(attr, "auth_patterns", ""))