Add source fields to datasets and allow filter datasets by cluster name

Author: willemarcel

vbos/datasets/admin.py

@@ -9,6 +9,7 @@
 from django.urls import path
 
 from .models import (
+    Cluster,
     RasterDataset,
     RasterFile,
     TabularDataset,
@@ -19,19 +20,24 @@
 from .forms import CSVUploadForm, GeoJSONUploadForm
 
 
+@admin.register(Cluster)
+class ClusterAdmin(admin.ModelAdmin):
+    list_display = ["id", "name"]
+
+
 @admin.register(RasterFile)
 class RasterFileAdmin(admin.ModelAdmin):
     list_display = ["id", "name", "created", "file"]
 
 
 @admin.register(RasterDataset)
 class RasterDatasetAdmin(admin.ModelAdmin):
-    list_display = ["id", "name", "created", "updated", "file"]
+    list_display = ["id", "cluster", "name", "created", "updated", "file"]
 
 
 @admin.register(VectorDataset)
 class VectorDatasetAdmin(admin.ModelAdmin):
-    list_display = ["id", "name", "created", "updated"]
+    list_display = ["id", "cluster", "name", "created", "updated"]
 
 
 @admin.register(VectorItem)
@@ -111,7 +117,7 @@ def import_file(self, request):
 
 @admin.register(TabularDataset)
 class TabularDatasetAdmin(admin.ModelAdmin):
-    list_display = ["id", "name", "created", "updated"]
+    list_display = ["id", "cluster", "name", "created", "updated"]
 
 
 @admin.register(TabularItem)

vbos/datasets/filters.py

@@ -7,11 +7,17 @@
     ModelChoiceFilter,
 )
 
-from .models import RasterDataset, VectorDataset, TabularDataset
+from .models import RasterDataset, VectorDataset, TabularDataset, Cluster
 
 
 class DatasetFilter(FilterSet):
     name = CharFilter(field_name="name", lookup_expr="icontains")
+    source = CharFilter(field_name="source", lookup_expr="icontains")
+    cluster = ModelChoiceFilter(
+        field_name="cluster__name",
+        to_field_name="name__iexact",
+        queryset=Cluster.objects.all(),
+    )
     created = DateFromToRangeFilter()
     updated = DateFromToRangeFilter()
     order_by = OrderingFilter(
@@ -22,16 +28,16 @@ class DatasetFilter(FilterSet):
 class RasterDatasetFilter(DatasetFilter):
     class Meta:
         model = RasterDataset
-        fields = ["name", "created", "updated"]
+        fields = ["name", "source", "cluster", "created", "updated"]
 
 
 class VectorDatasetFilter(DatasetFilter):
     class Meta:
         model = VectorDataset
-        fields = ["name", "created", "updated"]
+        fields = ["name", "source", "cluster", "created", "updated"]
 
 
 class TabularDatasetFilter(DatasetFilter):
     class Meta:
         model = TabularDataset
-        fields = ["name", "created", "updated"]
+        fields = ["name", "source", "cluster", "created", "updated"]

vbos/datasets/migrations/0005_cluster_rasterdataset_source_tabulardataset_source_and_more.py

@@ -0,0 +1,46 @@
+# Generated by Django 5.2.5 on 2025-09-11 13:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("datasets", "0004_alter_rasterdataset_name_alter_rasterfile_file_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Cluster",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(max_length=100, unique=True)),
+            ],
+            options={
+                "ordering": ["name"],
+            },
+        ),
+        migrations.AddField(
+            model_name="rasterdataset",
+            name="source",
+            field=models.CharField(blank=True, max_length=155, null=True),
+        ),
+        migrations.AddField(
+            model_name="tabulardataset",
+            name="source",
+            field=models.CharField(blank=True, max_length=155, null=True),
+        ),
+        migrations.AddField(
+            model_name="vectordataset",
+            name="source",
+            field=models.CharField(blank=True, max_length=155, null=True),
+        ),
+    ]

vbos/datasets/migrations/0006_rasterdataset_cluster_tabulardataset_cluster_and_more.py

@@ -0,0 +1,44 @@
+# Generated by Django 5.2.5 on 2025-09-11 13:59
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "datasets",
+            "0005_cluster_rasterdataset_source_tabulardataset_source_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="rasterdataset",
+            name="cluster",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="datasets.cluster",
+            ),
+        ),
+        migrations.AddField(
+            model_name="tabulardataset",
+            name="cluster",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="datasets.cluster",
+            ),
+        ),
+        migrations.AddField(
+            model_name="vectordataset",
+            name="cluster",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="datasets.cluster",
+            ),
+        ),
+    ]

vbos/datasets/migrations/0007_alter_rasterdataset_cluster_and_more.py

@@ -0,0 +1,35 @@
+# Generated by Django 5.2.5 on 2025-09-11 14:09
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("datasets", "0006_rasterdataset_cluster_tabulardataset_cluster_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="rasterdataset",
+            name="cluster",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="datasets.cluster"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="tabulardataset",
+            name="cluster",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="datasets.cluster"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="vectordataset",
+            name="cluster",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="datasets.cluster"
+            ),
+        ),
+    ]

vbos/datasets/models.py

@@ -8,6 +8,16 @@
 UPLOAD_TO = "staging/raster/" if settings.DEBUG else "production/raster/"
 
 
+class Cluster(models.Model):
+    name = models.CharField(max_length=100, unique=True)
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        ordering = ["name"]
+
+
 class RasterFile(models.Model):
     name = models.CharField(max_length=155, unique=True)
     created = models.DateTimeField(auto_now_add=True)
@@ -43,6 +53,11 @@ class RasterDataset(models.Model):
     name = models.CharField(max_length=155, unique=True)
     created = models.DateTimeField(auto_now_add=True)
     updated = models.DateTimeField(auto_now=True)
+    cluster = models.ForeignKey(
+        Cluster,
+        on_delete=models.PROTECT,
+    )
+    source = models.CharField(max_length=155, blank=True, null=True)
     file = models.ForeignKey(RasterFile, on_delete=models.PROTECT)
 
     def __str__(self):
@@ -56,6 +71,11 @@ class VectorDataset(models.Model):
     name = models.CharField(max_length=155, unique=True)
     created = models.DateTimeField(auto_now_add=True)
     updated = models.DateTimeField(auto_now=True)
+    cluster = models.ForeignKey(
+        Cluster,
+        on_delete=models.PROTECT,
+    )
+    source = models.CharField(max_length=155, blank=True, null=True)
 
     def __str__(self):
         return self.name
@@ -80,6 +100,11 @@ class TabularDataset(models.Model):
     name = models.CharField(max_length=155, unique=True)
     created = models.DateTimeField(auto_now_add=True)
     updated = models.DateTimeField(auto_now=True)
+    cluster = models.ForeignKey(
+        Cluster,
+        on_delete=models.PROTECT,
+    )
+    source = models.CharField(max_length=155, blank=True, null=True)
 
     def __str__(self):
         return self.name

vbos/datasets/serializers.py

@@ -2,6 +2,7 @@
 from rest_framework_gis.serializers import GeoFeatureModelSerializer
 
 from .models import (
+    Cluster,
     RasterDataset,
     TabularDataset,
     TabularItem,
@@ -10,18 +11,27 @@
 )
 
 
+class ClusterSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Cluster
+        fields = ["id", "name"]
+
+
 class RasterDatasetSerializer(serializers.ModelSerializer):
     file = serializers.ReadOnlyField(source="file.file.url")
+    cluster = serializers.ReadOnlyField(source="cluster.name")
 
     class Meta:
         model = RasterDataset
-        fields = ["id", "name", "created", "updated", "file"]
+        fields = ["id", "name", "created", "updated", "cluster", "source", "file"]
 
 
 class VectorDatasetSerializer(serializers.ModelSerializer):
+    cluster = serializers.ReadOnlyField(source="cluster.name")
+
     class Meta:
         model = VectorDataset
-        fields = "__all__"
+        fields = ["id", "name", "created", "updated", "cluster", "source"]
 
 
 class VectorItemSerializer(GeoFeatureModelSerializer):
@@ -47,9 +57,11 @@ def unformat_geojson(self, feature):
 
 
 class TabularDatasetSerializer(serializers.ModelSerializer):
+    cluster = serializers.ReadOnlyField(source="cluster.name")
+
     class Meta:
         model = TabularDataset
-        fields = "__all__"
+        fields = ["id", "name", "created", "updated", "cluster", "source"]
 
 
 class TabularItemSerializer(serializers.ModelSerializer):

vbos/datasets/test/test_admin.py

@@ -4,7 +4,13 @@
 from django.test import TestCase, Client
 from django.urls import reverse
 
-from vbos.datasets.models import TabularDataset, TabularItem, VectorDataset, VectorItem
+from vbos.datasets.models import (
+    Cluster,
+    TabularDataset,
+    TabularItem,
+    VectorDataset,
+    VectorItem,
+)
 
 
 class TabularItemAdminImportFileTests(TestCase):
@@ -14,7 +20,10 @@ def setUp(self):
             username="admin", password="password", email="[email protected]"
         )
         self.client.login(username="admin", password="password")
-        self.dataset = TabularDataset.objects.create(name="Test Dataset")
+        self.cluster = Cluster.objects.create(name="Other")
+        self.dataset = TabularDataset.objects.create(
+            name="Test Dataset", cluster=self.cluster
+        )
         self.upload_url = reverse("admin:datasets_tabularitem_import_file")
 
     def test_change_list_has_link_to_import_file(self):
@@ -68,7 +77,10 @@ def setUp(self):
             username="admin", password="password", email="[email protected]"
         )
         self.client.login(username="admin", password="password")
-        self.dataset = VectorDataset.objects.create(name="Test Dataset")
+        self.cluster = Cluster.objects.create(name="Other")
+        self.dataset = VectorDataset.objects.create(
+            name="Test Dataset", cluster=self.cluster
+        )
         self.upload_url = reverse("admin:datasets_vectoritem_import_file")
 
     def test_change_list_has_link_to_import_file(self):

vbos/datasets/test/test_cluster_views.py

@@ -0,0 +1,25 @@
+from django.urls.base import reverse
+from rest_framework import status
+from rest_framework.test import APITestCase
+
+from vbos.datasets.models import Cluster
+
+
+class TestClusterListView(APITestCase):
+    def setUp(self):
+        self.c1 = Cluster.objects.create(name="Administrative")
+        self.c2 = Cluster.objects.create(name="Transportation")
+        self.c3 = Cluster.objects.create(name="Other")
+        self.url = reverse("datasets:cluster-list")
+
+    def test_cluster_list_view(self):
+        req = self.client.get(self.url)
+        assert req.status_code == status.HTTP_200_OK
+        assert req.data.get("count") == 3
+        # It's ordered alphabetically by the name
+        assert req.data.get("results")[0]["name"] == "Administrative"
+        assert req.data.get("results")[1]["name"] == "Other"
+        assert req.data.get("results")[2]["name"] == "Transportation"
+        assert req.data.get("results")[0]["id"] == self.c1.id
+        assert req.data.get("results")[1]["id"] == self.c3.id
+        assert req.data.get("results")[2]["id"] == self.c2.id