Merge pull request #553 from aboutcode-org/551-resources-filter-by-checksums

551 resources filter by checksums

Author: JonoYang

Dockerfile

@@ -1,7 +1,7 @@
 # Copyright (c) nexB Inc. and others. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-FROM --platform=linux/amd64 python:3.12-slim
+FROM python:3.12-slim
 
 LABEL org.opencontainers.image.source="https://github.com/aboutcode-org/purldb"
 LABEL org.opencontainers.image.description="PurlDB"

minecode/ls.py

@@ -182,10 +182,7 @@ def parse_directory_listing(dir_listing, from_find=False):
             # this is likely a directory line from an ls -LR listing. Strip
             # trailing colon and keep track of the base directory
             if not line.endswith(":"):
-                raise Exception(
-                    "Unknown directory listing line format: #%(ln)d: %(line)r"
-                    % locals()
-                )
+                raise Exception(f"Unknown directory listing line format: #{ln}: {line}")
             base_dir = line.strip(":")
             continue
 

packagedb/api.py

@@ -258,12 +258,18 @@ def filter_by_checksums(self, request, *args, **kwargs):
             d = {f"{field}__in": value}
             lookups |= Q(**d)
 
-        qs = Resource.objects.filter(lookups).prefetch_related("package")
+        qs = (
+            Resource.objects.filter(lookups)
+            .select_related("package")
+            .only("package__uuid")
+            .order_by()
+        )
         paginated_qs = self.paginate_queryset(qs)
         serializer = ResourceAPISerializer(
             paginated_qs, many=True, context={"request": request}
         )
-        return self.get_paginated_response(serializer.data)
+        paginated_response = self.get_paginated_response(serializer.data)
+        return paginated_response
 
 
 class MultiplePackageURLFilter(MultipleCharFilter):
@@ -485,7 +491,7 @@ def filter_by_checksums(self, request, *args, **kwargs):
             )
 
         # Query to get the full Package objects with the earliest release_date for each sha1
-        qs = Package.objects.filter(lookups)
+        qs = Package.objects.filter(lookups).order_by()
         paginated_qs = self.paginate_queryset(qs)
         if enhance_package_data:
             serialized_package_data = [

packagedb/migrations/0088_alter_package_md5_alter_package_sha1_and_more.py

@@ -0,0 +1,109 @@
+# Generated by Django 5.1.2 on 2024-12-05 00:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("packagedb", "0087_rename_discovered_dependency_attribute"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="package",
+            name="md5",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="MD5 checksum hex-encoded, as in md5sum.",
+                max_length=32,
+                null=True,
+                verbose_name="MD5",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="package",
+            name="sha1",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="SHA1 checksum hex-encoded, as in sha1sum.",
+                max_length=40,
+                null=True,
+                verbose_name="SHA1",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="package",
+            name="sha256",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="SHA256 checksum hex-encoded, as in sha256sum.",
+                max_length=64,
+                null=True,
+                verbose_name="SHA256",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="package",
+            name="sha512",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="SHA512 checksum hex-encoded, as in sha512sum.",
+                max_length=128,
+                null=True,
+                verbose_name="SHA512",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="resource",
+            name="md5",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="MD5 checksum hex-encoded, as in md5sum.",
+                max_length=32,
+                null=True,
+                verbose_name="MD5",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="resource",
+            name="sha1",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="SHA1 checksum hex-encoded, as in sha1sum.",
+                max_length=40,
+                null=True,
+                verbose_name="SHA1",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="resource",
+            name="sha256",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="SHA256 checksum hex-encoded, as in sha256sum.",
+                max_length=64,
+                null=True,
+                verbose_name="SHA256",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="resource",
+            name="sha512",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="SHA512 checksum hex-encoded, as in sha512sum.",
+                max_length=128,
+                null=True,
+                verbose_name="SHA512",
+            ),
+        ),
+    ]

packagedb/models.py

@@ -174,27 +174,31 @@ class Meta:
         max_length=32,
         blank=True,
         null=True,
+        db_index=True,
         help_text=_("MD5 checksum hex-encoded, as in md5sum."),
     )
     sha1 = models.CharField(
         _("SHA1"),
         max_length=40,
         blank=True,
         null=True,
+        db_index=True,
         help_text=_("SHA1 checksum hex-encoded, as in sha1sum."),
     )
     sha256 = models.CharField(
         _("SHA256"),
         max_length=64,
         blank=True,
         null=True,
+        db_index=True,
         help_text=_("SHA256 checksum hex-encoded, as in sha256sum."),
     )
     sha512 = models.CharField(
         _("SHA512"),
         max_length=128,
         blank=True,
         null=True,
+        db_index=True,
         help_text=_("SHA512 checksum hex-encoded, as in sha512sum."),
     )