Support Implicit Namespace Packages (PEP 420)

Author: alexey-pelykh

doc/user_guide/configuration/all-options.rst

@@ -174,6 +174,13 @@ Standard Checkers
 **Default:**  ``(3, 10)``
 
 
+--source-roots
+""""""""""""""
+*Add paths to the list of the source roots. The source root is an absolute path or a path relative to the current working directory used to determine a package namespace for modules located under the source root.*
+
+**Default:**  ``()``
+
+
 --recursive
 """""""""""
 *Discover python modules and packages in the file system subtree.*

doc/user_guide/usage/run.rst

@@ -45,6 +45,13 @@ directory is automatically added on top of the python path
 package (i.e. has an ``__init__.py`` file), an implicit namespace package
 or if ``directory`` is in the python path.
 
+With implicit namespace packages
+--------------------------------
+
+If the analyzed sources use the Implicit Namespace Packages (PEP 420), the source root(s) should
+be specified to Pylint using the ``--source-roots`` option. Otherwise, the package names are
+detected incorrectly, since the Implicit Namespace Packages don't contain the ``__init__.py``.
+
 Command line options
 --------------------
 

doc/whatsnew/fragments/8154.feature

@@ -0,0 +1,3 @@
+Support Implicit Namespace Packages (PEP 420).
+
+Closes #8154

examples/pylintrc

@@ -90,6 +90,11 @@ persistent=yes
 # the version used to run pylint.
 py-version=3.10
 
+# Add paths to the list of the source roots. The source root is an absolute
+# path or a path relative to the current working directory used to
+# determine a package namespace for modules located under the source root.
+source-roots=src,tests
+
 # Discover python modules and packages in the file system subtree.
 recursive=no
 

pylint/config/argument.py

@@ -88,6 +88,16 @@ def _path_transformer(value: str) -> str:
     return os.path.expandvars(os.path.expanduser(value))
 
 
+def _paths_csv_transformer(value: str) -> Sequence[str]:
+    """Transforms a comma separated list of paths while expanding user and
+    variables.
+    """
+    paths: list[str] = []
+    for path in _csv_transformer(value):
+        paths.append(os.path.expandvars(os.path.expanduser(path)))
+    return paths
+
+
 def _py_version_transformer(value: str) -> tuple[int, ...]:
     """Transforms a version string into a version tuple."""
     try:
@@ -138,6 +148,7 @@ def _regexp_paths_csv_transfomer(value: str) -> Sequence[Pattern[str]]:
     "confidence": _confidence_transformer,
     "non_empty_string": _non_empty_string_transformer,
     "path": _path_transformer,
+    "paths_csv": _paths_csv_transformer,
     "py_version": _py_version_transformer,
     "regexp": _regex_transformer,
     "regexp_csv": _regexp_csv_transfomer,

pylint/config/option.py

@@ -117,6 +117,7 @@ def _py_version_validator(_: Any, name: str, value: Any) -> tuple[int, int, int]
     "string": utils._unquote,
     "int": int,
     "float": float,
+    "paths_csv": _csv_validator,
     "regexp": lambda pattern: re.compile(pattern or ""),
     "regexp_csv": _regexp_csv_validator,
     "regexp_paths_csv": _regexp_paths_csv_validator,
@@ -163,6 +164,7 @@ def _validate(value: Any, optdict: Any, name: str = "") -> Any:
 # pylint: disable=no-member
 class Option(optparse.Option):
     TYPES = optparse.Option.TYPES + (
+        "paths_csv",
         "regexp",
         "regexp_csv",
         "regexp_paths_csv",
@@ -175,6 +177,7 @@ class Option(optparse.Option):
     )
     ATTRS = optparse.Option.ATTRS + ["hide", "level"]
     TYPE_CHECKER = copy.copy(optparse.Option.TYPE_CHECKER)
+    TYPE_CHECKER["paths_csv"] = _csv_validator
     TYPE_CHECKER["regexp"] = _regexp_validator
     TYPE_CHECKER["regexp_csv"] = _regexp_csv_validator
     TYPE_CHECKER["regexp_paths_csv"] = _regexp_paths_csv_validator

pylint/lint/__init__.py

@@ -18,6 +18,7 @@
 
 from pylint.config.exceptions import ArgumentPreprocessingError
 from pylint.lint.caching import load_results, save_results
+from pylint.lint.expand_modules import discover_package_path
 from pylint.lint.parallel import check_parallel
 from pylint.lint.pylinter import PyLinter
 from pylint.lint.report_functions import (
@@ -26,7 +27,12 @@
     report_total_messages_stats,
 )
 from pylint.lint.run import Run
-from pylint.lint.utils import _patch_sys_path, fix_import_path
+from pylint.lint.utils import (
+    _augment_sys_path,
+    _patch_sys_path,
+    augmented_sys_path,
+    fix_import_path,
+)
 
 __all__ = [
     "check_parallel",
@@ -38,6 +44,9 @@
     "ArgumentPreprocessingError",
     "_patch_sys_path",
     "fix_import_path",
+    "_augment_sys_path",
+    "augmented_sys_path",
+    "discover_package_path",
     "save_results",
     "load_results",
 ]

pylint/lint/base_options.py

@@ -343,6 +343,17 @@ def _make_linter_options(linter: PyLinter) -> Options:
                 ),
             },
         ),
+        (
+            "source-roots",
+            {
+                "type": "paths_csv",
+                "metavar": "<path>[,<path>...]",
+                "default": (),
+                "help": "Add paths to the list of the source roots. The source root is an absolute "
+                "path or a path relative to the current working directory used to "
+                "determine a package namespace for modules located under the source root.",
+            },
+        ),
         (
             "recursive",
             {

pylint/lint/expand_modules.py

@@ -6,6 +6,7 @@
 
 import os
 import sys
+import warnings
 from collections.abc import Sequence
 from re import Pattern
 
@@ -24,14 +25,31 @@ def _is_package_cb(inner_path: str, parts: list[str]) -> bool:
 
 
 def get_python_path(filepath: str) -> str:
-    """TODO This get the python path with the (bad) assumption that there is always
-    an __init__.py.
+    # TODO: Remove deprecated function
+    warnings.warn(
+        "get_python_path has been deprecated because assumption that there's always an __init__.py "
+        "is not true since python 3.3 and is causing problems, particularly with PEP 420."
+        "Use discover_package_path and pass source root(s).",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return discover_package_path(filepath, [])
 
-    This is not true since python 3.3 and is causing problem.
-    """
-    dirname = os.path.realpath(os.path.expanduser(filepath))
+
+def discover_package_path(modulepath: str, source_roots: Sequence[str]) -> str:
+    """Discover package path from one its modules and source roots."""
+    dirname = os.path.realpath(os.path.expanduser(modulepath))
     if not os.path.isdir(dirname):
         dirname = os.path.dirname(dirname)
+
+    # Look for a source root that contains the module directory
+    for source_root in source_roots:
+        source_root = os.path.realpath(os.path.expanduser(source_root))
+        if os.path.commonpath([source_root, dirname]) == source_root:
+            return source_root
+
+    # Fall back to legacy discovery by looking for __init__.py upwards as
+    # it's the only way given that source root was not found or was not provided
     while True:
         if not os.path.exists(os.path.join(dirname, "__init__.py")):
             return dirname
@@ -64,6 +82,7 @@ def _is_ignored_file(
 # pylint: disable = too-many-locals, too-many-statements
 def expand_modules(
     files_or_modules: Sequence[str],
+    source_roots: Sequence[str],
     ignore_list: list[str],
     ignore_list_re: list[Pattern[str]],
     ignore_list_paths_re: list[Pattern[str]],
@@ -81,8 +100,8 @@ def expand_modules(
             something, ignore_list, ignore_list_re, ignore_list_paths_re
         ):
             continue
-        module_path = get_python_path(something)
-        additional_search_path = [".", module_path] + path
+        module_package_path = discover_package_path(something, source_roots)
+        additional_search_path = [".", module_package_path] + path
         if os.path.exists(something):
             # this is a file or a directory
             try:

pylint/lint/parallel.py

@@ -13,7 +13,7 @@
 import dill
 
 from pylint import reporters
-from pylint.lint.utils import _patch_sys_path
+from pylint.lint.utils import _augment_sys_path
 from pylint.message import Message
 from pylint.typing import FileItem
 from pylint.utils import LinterStats, merge_stats
@@ -37,12 +37,12 @@
 
 
 def _worker_initialize(
-    linter: bytes, arguments: None | str | Sequence[str] = None
+    linter: bytes, extra_packages_paths: Sequence[str] | None = None
 ) -> None:
     """Function called to initialize a worker for a Process within a concurrent Pool.
 
     :param linter: A linter-class (PyLinter) instance pickled with dill
-    :param arguments: File or module name(s) to lint and to be added to sys.path
+    :param extra_packages_paths: Extra entries to be added to sys.path
     """
     global _worker_linter  # pylint: disable=global-statement
     _worker_linter = dill.loads(linter)
@@ -53,8 +53,8 @@ def _worker_initialize(
     _worker_linter.set_reporter(reporters.CollectingReporter())
     _worker_linter.open()
 
-    # Patch sys.path so that each argument is importable just like in single job mode
-    _patch_sys_path(arguments or ())
+    if extra_packages_paths:
+        _augment_sys_path(extra_packages_paths)
 
 
 def _worker_check_single_file(
@@ -130,7 +130,7 @@ def check_parallel(
     linter: PyLinter,
     jobs: int,
     files: Iterable[FileItem],
-    arguments: None | str | Sequence[str] = None,
+    extra_packages_paths: Sequence[str] | None = None,
 ) -> None:
     """Use the given linter to lint the files with given amount of workers (jobs).
 
@@ -140,7 +140,9 @@ def check_parallel(
     # The linter is inherited by all the pool's workers, i.e. the linter
     # is identical to the linter object here. This is required so that
     # a custom PyLinter object can be used.
-    initializer = functools.partial(_worker_initialize, arguments=arguments)
+    initializer = functools.partial(
+        _worker_initialize, extra_packages_paths=extra_packages_paths
+    )
     with ProcessPoolExecutor(
         max_workers=jobs, initializer=initializer, initargs=(dill.dumps(linter),)
     ) as executor:

pylint/lint/pylinter.py

@@ -36,7 +36,11 @@
 from pylint.interfaces import HIGH
 from pylint.lint.base_options import _make_linter_options
 from pylint.lint.caching import load_results, save_results
-from pylint.lint.expand_modules import _is_ignored_file, expand_modules
+from pylint.lint.expand_modules import (
+    _is_ignored_file,
+    discover_package_path,
+    expand_modules,
+)
 from pylint.lint.message_state_handler import _MessageStateHandler
 from pylint.lint.parallel import check_parallel
 from pylint.lint.report_functions import (
@@ -46,7 +50,7 @@
 )
 from pylint.lint.utils import (
     _is_relative_to,
-    fix_import_path,
+    augmented_sys_path,
     get_fatal_error_message,
     prepare_crash_report,
 )
@@ -675,20 +679,27 @@ def check(self, files_or_modules: Sequence[str] | str) -> None:
                     "Missing filename required for --from-stdin"
                 )
 
+        extra_packages_paths = list(
+            {
+                discover_package_path(file_or_module, self.config.source_roots)
+                for file_or_module in files_or_modules
+            }
+        )
+
         # TODO: Move the parallel invocation into step 5 of the checking process
         if not self.config.from_stdin and self.config.jobs > 1:
             original_sys_path = sys.path[:]
             check_parallel(
                 self,
                 self.config.jobs,
                 self._iterate_file_descrs(files_or_modules),
-                files_or_modules,  # this argument patches sys.path
+                extra_packages_paths,
             )
             sys.path = original_sys_path
             return
 
         # 3) Get all FileItems
-        with fix_import_path(files_or_modules):
+        with augmented_sys_path(extra_packages_paths):
             if self.config.from_stdin:
                 fileitems = self._get_file_descr_from_stdin(files_or_modules[0])
                 data: str | None = _read_stdin()
@@ -697,7 +708,7 @@ def check(self, files_or_modules: Sequence[str] | str) -> None:
                 data = None
 
         # The contextmanager also opens all checkers and sets up the PyLinter class
-        with fix_import_path(files_or_modules):
+        with augmented_sys_path(extra_packages_paths):
             with self._astroid_module_checker() as check_astroid_module:
                 # 4) Get the AST for each FileItem
                 ast_per_fileitem = self._get_asts(fileitems, data)
@@ -884,10 +895,13 @@ def _iterate_file_descrs(
             if self.should_analyze_file(name, filepath, is_argument=is_arg):
                 yield FileItem(name, filepath, descr["basename"])
 
-    def _expand_files(self, modules: Sequence[str]) -> dict[str, ModuleDescriptionDict]:
+    def _expand_files(
+        self, files_or_modules: Sequence[str]
+    ) -> dict[str, ModuleDescriptionDict]:
         """Get modules and errors from a list of modules and handle errors."""
         result, errors = expand_modules(
-            modules,
+            files_or_modules,
+            self.config.source_roots,
             self.config.ignore,
             self.config.ignore_patterns,
             self._ignore_paths,