wip: reimplement namespace pkg filepath calc in starlark

With this PR I would like to facilitate the implementation of the venv
layouts because we can in theory take the `srcs` and the `data` within
the `py_library` and then use the `expand_template` to write the extra
Python files if the namespace_pkgs flag is enabled.

This is WIP so that maintainers can hop in if they have time.

TODO:
- [ ] Do the ctx.actions.expand_template based on the output from the
  reimplementation
- [ ] Thread the namespace_pkgs attr
- [ ] Remove Python code? Put under pipstar?

Work towards #2156

Author: aignas

python/private/pypi/namespace_pkgs.bzl

@@ -0,0 +1,42 @@
+"""Utilities to get where we should write namespace pkg paths."""
+
+EXTS = [
+    ".py",
+    ".pyd",
+    ".so",
+    ".pyc",
+]
+
+def _get_files(files, ignored_dirnames = []):
+    dirs = {}
+    ignored = {i: None for i in ignored_dirnames}
+    for file in files:
+        dirname, _, filename = file.rpartition("/")
+
+        if filename == "__init__.py":
+            ignored[dirname] = None
+            dirname, _, _ = dirname.rpartition("/")
+        elif filename.endswith(EXTS[0]):
+            pass
+        elif filename.endswith(EXTS[1]):
+            pass
+        elif filename.endswith(EXTS[2]):
+            pass
+        elif filename.endswith(EXTS[3]):
+            pass
+        else:
+            continue
+
+        if dirname in dirs or not dirname:
+            continue
+
+        dir_path = "."
+        for dir_name in dirname.split("/"):
+            dir_path = "{}/{}".format(dir_path, dir_name)
+            dirs[dir_path[2:]] = None
+
+    return sorted([d for d in dirs if d not in ignored])
+
+namespace_pkgs = struct(
+    get_files = _get_files,
+)

tests/pypi/namespace_pkgs/BUILD.bazel

@@ -0,0 +1,5 @@
+load(":namespace_pkgs_tests.bzl", "namespace_pkgs_test_suite")
+
+namespace_pkgs_test_suite(
+    name = "namespace_pkgs_tests",
+)

tests/pypi/namespace_pkgs/namespace_pkgs_tests.bzl

@@ -0,0 +1,166 @@
+""
+
+load("@rules_testing//lib:analysis_test.bzl", "test_suite")
+load("//python/private/pypi:namespace_pkgs.bzl", "namespace_pkgs")  # buildifier: disable=bzl-visibility
+
+_tests = []
+
+def test_in_current_dir(env):
+    files = [
+        "foo/bar/biz.py",
+        "foo/bee/boo.py",
+        "foo/buu/__init__.py",
+        "foo/buu/bii.py",
+    ]
+    got = namespace_pkgs.get_files(files)
+    expected = [
+        "foo",
+        "foo/bar",
+        "foo/bee",
+    ]
+    env.expect.that_collection(got).contains_exactly(expected)
+
+_tests.append(test_in_current_dir)
+
+def test_find_correct_namespace_packages(env):
+    files = [
+        "foo/bar/biz.py",
+        "foo/bee/boo.py",
+        "foo/buu/__init__.py",
+        "foo/buu/bii.py",
+    ]
+
+    got = namespace_pkgs.get_files(files)
+    expected = [
+        "foo",
+        "foo/bar",
+        "foo/bee",
+    ]
+    env.expect.that_collection(got).contains_exactly(expected)
+
+_tests.append(test_find_correct_namespace_packages)
+
+def test_ignores_empty_directories(_):
+    # because globs do not add directories, this test is not needed
+    pass
+
+_tests.append(test_ignores_empty_directories)
+
+def test_empty_case(env):
+    files = [
+        "foo/__init__.py",
+        "foo/bar/__init__.py",
+        "foo/bar/biz.py",
+    ]
+
+    got = namespace_pkgs.get_files(files)
+    expected = []
+    env.expect.that_collection(got).contains_exactly(expected)
+
+_tests.append(test_empty_case)
+
+def test_ignores_non_module_files_in_directories(env):
+    files = [
+        "foo/__init__.pyi",
+        "foo/py.typed",
+    ]
+
+    got = namespace_pkgs.get_files(files)
+    expected = []
+    env.expect.that_collection(got).contains_exactly(expected)
+
+_tests.append(test_ignores_non_module_files_in_directories)
+
+def test_parent_child_relationship_of_namespace_pkgs(env):
+    files = [
+        "foo/bar/biff/my_module.py",
+        "foo/bar/biff/another_module.py",
+    ]
+
+    got = namespace_pkgs.get_files(files)
+    expected = [
+        "foo",
+        "foo/bar",
+        "foo/bar/biff",
+    ]
+    env.expect.that_collection(got).contains_exactly(expected)
+
+_tests.append(test_parent_child_relationship_of_namespace_pkgs)
+
+def test_parent_child_relationship_of_namespace_and_standard_pkgs(env):
+    files = [
+        "foo/bar/biff/__init__.py",
+        "foo/bar/biff/another_module.py",
+    ]
+
+    got = namespace_pkgs.get_files(files)
+    expected = [
+        "foo",
+        "foo/bar",
+    ]
+    env.expect.that_collection(got).contains_exactly(expected)
+
+_tests.append(test_parent_child_relationship_of_namespace_and_standard_pkgs)
+
+def test_parent_child_relationship_of_namespace_and_nested_standard_pkgs(env):
+    files = [
+        "foo/bar/__init__.py",
+        "foo/bar/biff/another_module.py",
+        "foo/bar/biff/__init__.py",
+        "foo/bar/boof/big_module.py",
+        "foo/bar/boof/__init__.py",
+        "fim/in_a_ns_pkg.py",
+    ]
+
+    got = namespace_pkgs.get_files(files)
+    expected = [
+        "foo",
+        "fim",
+    ]
+    env.expect.that_collection(got).contains_exactly(expected)
+
+_tests.append(test_parent_child_relationship_of_namespace_and_nested_standard_pkgs)
+
+def test_recognized_all_nonstandard_module_types(env):
+    files = [
+        "ayy/my_module.pyc",
+        "bee/ccc/dee/eee.so",
+        "eff/jee/aych.pyd",
+    ]
+
+    expected = [
+        "ayy",
+        "bee",
+        "bee/ccc",
+        "bee/ccc/dee",
+        "eff",
+        "eff/jee",
+    ]
+    got = namespace_pkgs.get_files(files)
+    env.expect.that_collection(got).contains_exactly(expected)
+
+_tests.append(test_recognized_all_nonstandard_module_types)
+
+def test_skips_ignored_directories(env):
+    files = [
+        "foo/boo/my_module.py",
+        "foo/bar/another_module.py",
+    ]
+
+    expected = [
+        "foo",
+        "foo/bar",
+    ]
+    got = namespace_pkgs.get_files(
+        files,
+        ignored_dirnames = ["foo/boo"],
+    )
+    env.expect.that_collection(got).contains_exactly(expected)
+
+_tests.append(test_skips_ignored_directories)
+
+def namespace_pkgs_test_suite(name):
+    test_suite(
+        name = name,
+        basic_tests = _tests,
+    )