Analyze directory tree and archives (#6)

Add APIs to analyze a directory tree or an archive (zip, tar). By
default, all regular files with executable bit are analyzed except for
files that have a `.py`, `.sh`, or docs extension (md, rst, txt). In
addition, files with a pattern like `lib*.so`, `lib*.so.1.2.3`, and
`ld-*.so` are analyzed, too.

Signed-off-by: Christian Heimes <[email protected]>

Author: tiran

README.md

@@ -73,7 +73,10 @@ Complete!
 * exception `elfdeps.ELFError`
 * dataclass `elfdeps.ELFInfo`
 * dataclass `elfdeps.SOInfo`
+* `elfdeps.analyze_dirtree(dirname, settings=None) -> Generator[ELFInfo, None, None]`
 * `elfdeps.analyze_elffile(elffile, *, filename, is_exec, settings=None) -> ELFInfo`
 * `elfdeps.analyze_file(filename, *, settings=None) -> ELFInfo`
+* `elfdeps.analyze_tarfile(tfile, *, settings=None) -> Generator[ELFInfo, None, None]`
 * `elfdeps.analyze_tarmember(tfile, tarinfo, *, settings=None) -> ELFInfo`
+* `elfdeps.analyze_zipfile(zfile, *, settings=None) -> Generator[ELFInfo, None, None]`
 * `elfdeps.analyze_zipmember(zfile, zipinfo, *, settings=None) -> ELFInfo`

pyproject.toml

@@ -59,7 +59,7 @@ source = [
    "src/elfdeps",
    ".tox/py*/**/site-packages/elfdeps",
 ]
-tests =[
+tests = [
    "tests/",
 ]
 

src/elfdeps/__init__.py

@@ -5,16 +5,22 @@
     "ELFError",
     "ELFInfo",
     "SOInfo",
+    "analyze_dirtree",
     "analyze_elffile",
     "analyze_file",
+    "analyze_tarfile",
     "analyze_tarmember",
+    "analyze_zipfile",
     "analyze_zipmember",
 )
 
 from elftools.common.exceptions import ELFError
 
 from ._archives import (
+    analyze_dirtree,
+    analyze_tarfile,
     analyze_tarmember,
+    analyze_zipfile,
     analyze_zipmember,
 )
 from ._elfdeps import (

src/elfdeps/__main__.py

@@ -1,10 +1,11 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import argparse
+import logging
 import pathlib
 import pprint
+import stat
 import tarfile
-import typing
 import zipfile
 
 from . import _archives, _elfdeps
@@ -14,6 +15,13 @@
 
 parser = argparse.ArgumentParser("elfdeps")
 parser.add_argument("filename", type=pathlib.Path)
+parser.add_argument(
+    "-d",
+    "--debug",
+    action="store_true",
+    dest="debug",
+    help="debug logging",
+)
 parser.add_argument(
     "-P",
     "--provides",
@@ -61,24 +69,6 @@
 )
 
 
-def zip_requires(
-    filename: pathlib.Path, settings: _elfdeps.ELFAnalyzeSettings
-) -> typing.Iterable[_elfdeps.ELFInfo]:
-    with zipfile.ZipFile(filename) as zf:
-        for zipinfo in zf.infolist():
-            if zipinfo.filename.endswith(".so"):
-                yield _archives.analyze_zipmember(zf, zipinfo, settings=settings)
-
-
-def tar_requires(
-    filename: pathlib.Path, settings: _elfdeps.ELFAnalyzeSettings
-) -> typing.Iterable[_elfdeps.ELFInfo]:
-    with tarfile.TarFile.open(filename, mode="r:*") as tf:
-        for tarinfo in tf.getmembers():
-            if tarinfo.name.endswith(".so"):
-                yield _archives.analyze_tarmember(tf, tarinfo, settings=settings)
-
-
 def main(argv: list[str] | None = None) -> None:
     args = parser.parse_args(argv)
     settings = _elfdeps.ELFAnalyzeSettings(
@@ -88,13 +78,20 @@ def main(argv: list[str] | None = None) -> None:
         require_interp=args.require_interp,
         unique=args.unique,
     )
+    logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO)
     filename: pathlib.Path = args.filename
+    st = filename.stat()
     if filename.name.endswith(ZIPEXT):
-        infos = list(zip_requires(filename, settings=settings))
+        with zipfile.ZipFile(filename, mode="r") as zfile:
+            infos = list(_archives.analyze_zipfile(zfile=zfile, settings=settings))
     elif filename.name.endswith(TAREXT):
-        infos = list(tar_requires(filename, settings=settings))
+        with tarfile.TarFile.open(filename, mode="r:*") as tfile:
+            infos = list(_archives.analyze_tarfile(tfile=tfile, settings=settings))
+    elif stat.S_ISDIR(st.st_mode):
+        infos = list(_archives.analyze_dirtree(filename, settings=settings))
     else:
         infos = [_elfdeps.analyze_file(filename, settings=settings)]
+
     if args.provides:
         provides = set()
         for info in infos:
@@ -108,7 +105,7 @@ def main(argv: list[str] | None = None) -> None:
         for r in sorted(requires):
             print(r)
     else:
-        for info in infos:
+        for info in sorted(infos):
             pprint.pprint(info)
 
 

src/elfdeps/_archives.py

@@ -1,15 +1,35 @@
 # SPDX-License-Identifier: Apache-2.0
 """Analyze archive members"""
 
+import logging
+import os
 import pathlib
 import stat
 import tarfile
 import typing
 import zipfile
 
+from elftools.common.exceptions import ELFError
 from elftools.elf.elffile import ELFFile
 
 from ._elfdeps import ELFAnalyzeSettings, ELFInfo, analyze_elffile
+from ._fileinfo import is_executable_file
+
+logger = logging.getLogger(__name__)
+
+
+def _zipinfo_mode(zipinfo: zipfile.ZipInfo) -> int:
+    """Full mode for zipinfo object"""
+    # mode may not contain reg file info
+    mode = zipinfo.external_attr >> 16
+    if stat.S_IFMT(mode) == 0:
+        lo = zipinfo.external_attr & 0xFFFF
+        if lo & 0x10:
+            # MS-DOS directory
+            mode |= stat.S_IFDIR
+        else:
+            mode |= stat.S_IFREG
+    return mode
 
 
 def analyze_zipmember(
@@ -19,8 +39,8 @@ def analyze_zipmember(
     settings: ELFAnalyzeSettings | None = None,
 ) -> ELFInfo:
     """Analyze a zipfile member"""
-    mode = zipinfo.external_attr >> 16
-    is_exec = bool(mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
+    mode = _zipinfo_mode(zipinfo)
+    is_exec = is_executable_file(mode)
     filename = pathlib.Path(zipinfo.filename)
     with zfile.open(zipinfo, mode="r") as f:
         elffile = ELFFile(f)
@@ -29,15 +49,51 @@ def analyze_zipmember(
         )
 
 
+def analyze_zipfile(
+    zfile: zipfile.ZipFile, *, settings: ELFAnalyzeSettings | None = None
+) -> typing.Generator[ELFInfo, None, None]:
+    """Analyze a zip file"""
+    if settings is None:
+        settings = ELFAnalyzeSettings()
+    for zipinfo in zfile.infolist():
+        filename = pathlib.Path(zipinfo.filename)
+        mode = _zipinfo_mode(zipinfo)
+        if settings.is_candidate(filename, mode):
+            try:
+                yield analyze_zipmember(zfile, zipinfo, settings=settings)
+            except ELFError as err:
+                # not an ELF file (e.g. a script or linker script)
+                logger.debug("%s is not a ELF file: %s", filename, err)
+
+
+def _tarinfo_mode(tarinfo: tarfile.TarInfo) -> int:
+    """Full mode for tarinfo"""
+    # tarinfo.mode contains only permission bits
+    mode = tarinfo.mode
+    if tarinfo.isreg():
+        mode |= stat.S_IFREG
+    elif tarinfo.isdir():
+        mode |= stat.S_IFDIR
+    elif tarinfo.issym():
+        mode |= stat.S_IFLNK
+    elif tarinfo.isblk():
+        mode |= stat.S_IFBLK
+    elif tarinfo.ischr():
+        mode |= stat.S_IFCHR
+    elif tarinfo.isfifo():
+        mode |= stat.S_IFIFO
+    return mode
+
+
 def analyze_tarmember(
     tfile: tarfile.TarFile,
     tarinfo: tarfile.TarInfo,
     *,
     settings: ELFAnalyzeSettings | None = None,
 ) -> ELFInfo:
     """Analze a tarfile member"""
-    mode = tarinfo.mode
-    is_exec = bool(mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
+    mode = _tarinfo_mode(tarinfo)
+    is_exec = is_executable_file(mode)
     filename = pathlib.Path(tarinfo.name)
     f = tfile.extractfile(tarinfo)
     if typing.TYPE_CHECKING:
@@ -47,3 +103,82 @@ def analyze_tarmember(
         return analyze_elffile(
             elffile, filename=filename, is_exec=is_exec, settings=settings
         )
+
+
+def analyze_tarfile(
+    tfile: tarfile.TarFile, *, settings: ELFAnalyzeSettings | None = None
+) -> typing.Generator[ELFInfo, None, None]:
+    """Analyze a tar ball"""
+    if settings is None:
+        settings = ELFAnalyzeSettings()
+    for tarinfo in tfile:
+        filename = pathlib.Path(tarinfo.name)
+        mode = _tarinfo_mode(tarinfo)
+        if settings.is_candidate(filename, mode):
+            try:
+                yield analyze_tarmember(tfile, tarinfo, settings=settings)
+            except ELFError as err:
+                # not an ELF file (e.g. a script or linker script)
+                logger.debug("%s is not a ELF file: %s", filename, err)
+
+
+OnError = typing.Callable[[pathlib.Path, OSError | ELFError], None] | None
+
+
+def _scanwalk(
+    dirname: pathlib.Path, onerror: OnError = None
+) -> typing.Generator[os.DirEntry, None, None]:
+    """Recursive scandir"""
+    try:
+        it = os.scandir(dirname)
+    except OSError as err:
+        if onerror is not None:
+            onerror(dirname, err)
+        return
+
+    with it:
+        while True:
+            try:
+                entry = next(it)
+            except StopIteration:
+                break
+            except OSError as err:
+                if onerror is not None:
+                    onerror(dirname, err)
+                return
+            try:
+                is_dir = entry.is_dir(follow_symlinks=False)
+            except OSError:
+                is_dir = False
+            if is_dir:
+                yield from _scanwalk(pathlib.Path(entry.path), onerror=onerror)
+            else:
+                yield entry
+
+
+def analyze_dirtree(
+    dirname: pathlib.Path,
+    *,
+    settings: ELFAnalyzeSettings | None = None,
+    onerror: OnError = None,
+) -> typing.Generator[ELFInfo, None, None]:
+    """Recursively analyze dirctory tree"""
+    if settings is None:
+        settings = ELFAnalyzeSettings()
+    for entry in _scanwalk(dirname):
+        filename = pathlib.Path(entry.path)
+        try:
+            mode = entry.stat(follow_symlinks=False).st_mode
+            if settings.is_candidate(filename, mode):
+                with filename.open("rb") as f:
+                    elffile = ELFFile(f)
+                    yield analyze_elffile(
+                        elffile,
+                        filename=filename,
+                        is_exec=is_executable_file(mode),
+                        settings=settings,
+                    )
+        except (OSError, ELFError) as err:
+            logger.debug("%s is not a ELF file or is not accessible: %s", filename, err)
+            if onerror is not None:
+                onerror(filename, err)