Scan package files and extract for packages

For rootfs pipelines (rootfs, docker, docker-windows) all package files
which were a part of system packages had their status updated and
consequently were not being scanned for licenses, copyrights, emails and
urls. We were also not scanning package metadata files tagged as application
packages in scan_codebase and the rootfs pipelines. This commit scans all
package files and package metadata files to make sure we are not missing
any information.

Reference: #762
Reference: #1194
Reference: #83
Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

scanpipe/models.py

@@ -1909,6 +1909,17 @@ def no_status(self, status=None):
             return self.filter(~Q(status=status))
         return self.filter(status="")
 
+    def package_files(self):
+        """
+        Filter for CodebaseResources which are part of either an application
+        package or a system package.
+        """
+        from scanpipe.pipes import flag
+
+        return self.filter(
+            Q(status=flag.APPLICATION_PACKAGE) | Q(status=flag.SYSTEM_PACKAGE)
+        )
+
     def empty(self):
         return self.filter(Q(size__isnull=True) | Q(size=0))
 

scanpipe/pipelines/docker.py

@@ -33,6 +33,7 @@ def steps(cls):
         return (
             cls.extract_images,
             cls.extract_layers,
+            cls.extract_archives,
             cls.find_images_os_and_distro,
             cls.collect_images_information,
             cls.collect_and_create_codebase_resources,
@@ -42,6 +43,7 @@ def steps(cls):
             cls.flag_ignored_resources,
             cls.scan_for_application_packages,
             cls.scan_for_files,
+            cls.scan_package_files,
             cls.analyze_scanned_files,
             cls.flag_not_analyzed_codebase_resources,
         )

scanpipe/pipelines/docker_windows.py

@@ -34,6 +34,7 @@ def steps(cls):
         return (
             cls.extract_images,
             cls.extract_layers,
+            cls.extract_archives,
             cls.find_images_os_and_distro,
             cls.collect_images_information,
             cls.collect_and_create_codebase_resources,
@@ -45,6 +46,7 @@ def steps(cls):
             cls.flag_ignored_resources,
             cls.scan_for_application_packages,
             cls.scan_for_files,
+            cls.scan_package_files,
             cls.analyze_scanned_files,
             cls.flag_data_files_with_no_clues,
             cls.flag_not_analyzed_codebase_resources,

scanpipe/pipelines/root_filesystem.py

@@ -35,6 +35,7 @@ class RootFS(Pipeline):
     def steps(cls):
         return (
             cls.extract_input_files_to_codebase_directory,
+            cls.extract_archives,
             cls.find_root_filesystems,
             cls.collect_rootfs_information,
             cls.collect_and_create_codebase_resources,
@@ -45,6 +46,7 @@ def steps(cls):
             cls.scan_for_application_packages,
             cls.match_not_analyzed_to_system_packages,
             cls.scan_for_files,
+            cls.scan_package_files,
             cls.analyze_scanned_files,
             cls.flag_not_analyzed_codebase_resources,
         )
@@ -89,7 +91,7 @@ def collect_and_create_system_packages(self):
                 rootfs.scan_rootfs_for_system_packages(self.project, rfs)
 
     def flag_uninteresting_codebase_resources(self):
-        """Flag files—not worth tracking—that don’t belong to any system packages."""
+        """Flag files—not worth tracking—that do not belong to any system packages."""
         rootfs.flag_uninteresting_codebase_resources(self.project)
 
     def scan_for_application_packages(self):
@@ -123,6 +125,13 @@ def scan_for_files(self):
         """Scan unknown resources for copyrights, licenses, emails, and urls."""
         scancode.scan_for_files(self.project, progress_logger=self.log)
 
+    def scan_package_files(self):
+        """
+        Scan files which are part of a package, for copyright, license, email
+        and urls.
+        """
+        scancode.scan_package_files(self.project, progress_logger=self.log)
+
     def analyze_scanned_files(self):
         """Analyze single file scan results for completeness."""
         flag.analyze_scanned_files(self.project)

scanpipe/pipelines/scan_codebase.py

@@ -45,6 +45,7 @@ def steps(cls):
             cls.flag_ignored_resources,
             cls.scan_for_application_packages,
             cls.scan_for_files,
+            cls.scan_package_files,
         )
 
     def copy_inputs_to_codebase_directory(self):
@@ -65,3 +66,10 @@ def scan_for_application_packages(self):
     def scan_for_files(self):
         """Scan unknown resources for copyrights, licenses, emails, and urls."""
         scancode.scan_for_files(self.project, progress_logger=self.log)
+
+    def scan_package_files(self):
+        """
+        Scan files which are manifests for detected application packages, for copyright,
+        license, email and urls.
+        """
+        scancode.scan_package_files(self.project, progress_logger=self.log)

scanpipe/pipes/scancode.py

@@ -235,7 +235,9 @@ def scan_for_package_data(location, with_threading=True, package_only=False, **k
     return _scan_resource(location, scanners, with_threading=with_threading)
 
 
-def save_scan_file_results(codebase_resource, scan_results, scan_errors):
+def save_scan_file_results(
+    codebase_resource, scan_results, scan_errors, update_status=True, **kwargs
+):
     """
     Save the resource scan file results in the database.
     Create project errors if any occurred during the scan.
@@ -246,6 +248,9 @@ def save_scan_file_results(codebase_resource, scan_results, scan_errors):
         codebase_resource.add_errors(scan_errors)
         status = flag.SCANNED_WITH_ERROR
 
+    if not update_status:
+        status = None
+
     codebase_resource.set_scan_results(scan_results, status)
 
 
@@ -266,7 +271,12 @@ def save_scan_package_results(codebase_resource, scan_results, scan_errors):
 
 
 def scan_resources(
-    resource_qs, scan_func, save_func, scan_func_kwargs=None, progress_logger=None
+    resource_qs,
+    scan_func,
+    save_func,
+    scan_func_kwargs=None,
+    save_func_kwargs=None,
+    progress_logger=None,
 ):
     """
     Run the `scan_func` on the codebase resources of the provided `resource_qs`.
@@ -286,6 +296,9 @@ def scan_resources(
     if not scan_func_kwargs:
         scan_func_kwargs = {}
 
+    if not save_func_kwargs:
+        save_func_kwargs = {}
+
     resource_count = resource_qs.count()
     logger.info(f"Scan {resource_count} codebase resources with {scan_func.__name__}")
     resource_iterator = resource_qs.iterator(chunk_size=2000)
@@ -300,7 +313,7 @@ def scan_resources(
             scan_results, scan_errors = scan_func(
                 resource.location, with_threading, **scan_func_kwargs
             )
-            save_func(resource, scan_results, scan_errors)
+            save_func(resource, scan_results, scan_errors, **save_func_kwargs)
         return
 
     logger.info(f"Starting ProcessPoolExecutor with {max_workers} max_workers")
@@ -319,10 +332,10 @@ def scan_resources(
             progress.log_progress()
             logger.debug(f"{scan_func.__name__} pk={resource.pk}")
             scan_results, scan_errors = future.result()
-            save_func(resource, scan_results, scan_errors)
+            save_func(resource, scan_results, scan_errors, **save_func_kwargs)
 
 
-def scan_for_files(project, resource_qs=None, progress_logger=None):
+def scan_for_files(project, resource_qs=None, progress_logger=None, update_status=True):
     """
     Run a license, copyright, email, and url scan on files without a status for
     a `project`.
@@ -338,12 +351,39 @@ def scan_for_files(project, resource_qs=None, progress_logger=None):
     if license_score := project.get_env("scancode_license_score"):
         scan_func_kwargs["min_license_score"] = license_score
 
+    save_func_kwargs = {
+        "update_status": update_status,
+    }
+
     scan_resources(
         resource_qs=resource_qs,
         scan_func=scan_file,
         save_func=save_scan_file_results,
         scan_func_kwargs=scan_func_kwargs,
+        save_func_kwargs=save_func_kwargs,
+        progress_logger=progress_logger,
+    )
+
+
+def scan_package_files(
+    project,
+    progress_logger=None,
+    update_status=False,
+):
+    """
+    Scan files which are part of a package, for copyright, license, email
+    and urls.
+
+    If `update_status` is False, the status field of codebase resources is not
+    updated to `scanned` (which is a side-effect of scanning files), but rather
+    keep the old status intact.
+    """
+    package_files = project.codebaseresources.package_files()
+    scan_for_files(
+        project=project,
+        resource_qs=package_files,
         progress_logger=progress_logger,
+        update_status=update_status,
     )