Use django-storages to upload raster file to S3-like bucket + set names

as unique

Author: willemarcel

.env.example

@@ -0,0 +1,4 @@
+DJANGO_SECRET_KEY="local"
+DJANGO_AWS_ACCESS_KEY_ID=""
+DJANGO_AWS_SECRET_ACCESS_KEY=""
+DJANGO_AWS_STORAGE_BUCKET_NAME=""

README.md

@@ -7,7 +7,7 @@ VBOS Django application and data services. Check out the project's [documentatio
 
 # Prerequisites
 
-- [Docker](https://docs.docker.com/docker-for-mac/install/)  
+- [Docker](https://docs.docker.com/docker-for-mac/install/)
 
 # Local Development
 
@@ -21,3 +21,7 @@ Run a command inside the docker container:
 ```bash
 docker-compose run --rm web [command]
 ```
+
+# Configuration
+
+Copy the `.env.example` file to `.env` and edit the variables you need to configure the access to the DigitalOcean Spaces (S3 compatible).

docker-compose.yml

@@ -16,7 +16,10 @@ services:
   web:
     restart: always
     environment:
-      - DJANGO_SECRET_KEY=local
+      - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY}
+      - DJANGO_AWS_ACCESS_KEY_ID=${DJANGO_AWS_ACCESS_KEY_ID}
+      - DJANGO_AWS_SECRET_ACCESS_KEY=${DJANGO_AWS_SECRET_ACCESS_KEY}
+      - DJANGO_AWS_STORAGE_BUCKET_NAME=${DJANGO_AWS_STORAGE_BUCKET_NAME}
     build: ./
     command: >
       bash -c "python3 wait_for_postgres.py &&
@@ -54,5 +57,5 @@ services:
     ports:
       - "8002:8000"
     volumes:
-      - ./data:/data  # Optional: mount local directory with your raster files
+      - ./data:/data # Optional: mount local directory with your raster files
     restart: unless-stopped

requirements.txt

@@ -20,6 +20,11 @@ drf_spectacular==0.28.0
 django-cors-headers==4.7.0
 drf-excel==2.5.3
 
+# Storage
+django-storages==1.14.6
+boto3==1.40.26
+whitenoise
+
 # Developer Tools
 ipdb==0.13.13
 ipython==8.30.0

vbos/config/production.py

@@ -3,6 +3,7 @@
 
 
 class Production(Common):
+    DEBUG = False
     INSTALLED_APPS = Common.INSTALLED_APPS
     SECRET_KEY = os.getenv("DJANGO_SECRET_KEY")
     # Site
@@ -13,13 +14,26 @@ class Production(Common):
     # Static files (CSS, JavaScript, Images)
     # https://docs.djangoproject.com/en/2.0/howto/static-files/
     # http://django-storages.readthedocs.org/en/latest/index.html
+    INSTALLED_APPS += ("storages",)
+    STORAGES = {
+        "default": {"BACKEND": "storages.backends.s3.S3Storage"},
+        "staticfiles": {
+            "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"
+        },
+    }
+    AWS_S3_OBJECT_PARAMETERS = {
+        "CacheControl": "max-age=2592000",
+    }
+    AWS_S3_ENDPOINT_URL = os.getenv(
+        "DJANGO_AWS_S3_ENDPOINT_URL", "https://syd1.digitaloceanspaces.com"
+    )
     AWS_ACCESS_KEY_ID = os.getenv("DJANGO_AWS_ACCESS_KEY_ID")
     AWS_SECRET_ACCESS_KEY = os.getenv("DJANGO_AWS_SECRET_ACCESS_KEY")
     AWS_STORAGE_BUCKET_NAME = os.getenv("DJANGO_AWS_STORAGE_BUCKET_NAME")
     AWS_DEFAULT_ACL = "public-read"
     AWS_AUTO_CREATE_BUCKET = True
     AWS_QUERYSTRING_AUTH = False
-    MEDIA_URL = f"https://s3.amazonaws.com/{AWS_STORAGE_BUCKET_NAME}/"
+    MEDIA_URL = f"https://{AWS_STORAGE_BUCKET_NAME}.syd1.digitaloceanspaces.com/"
 
     # https://developers.google.com/web/fundamentals/performance/optimizing-content-efficiency/http-caching#cache-control
     # Response can be cached by browser and any intermediary caches (i.e. it is "public") for up to 1 day

vbos/datasets/migrations/0004_alter_rasterdataset_name_alter_rasterfile_file_and_more.py

@@ -0,0 +1,47 @@
+# Generated by Django 5.2.5 on 2025-09-11 11:04
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("datasets", "0003_rasterfile_rasterdataset"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="rasterdataset",
+            name="name",
+            field=models.CharField(max_length=155, unique=True),
+        ),
+        migrations.AlterField(
+            model_name="rasterfile",
+            name="file",
+            field=models.FileField(
+                unique=True,
+                upload_to="staging/raster/",
+                validators=[
+                    django.core.validators.FileExtensionValidator(
+                        allowed_extensions=["tiff", "tif", "geotiff", "gtiff"]
+                    )
+                ],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="rasterfile",
+            name="name",
+            field=models.CharField(max_length=155, unique=True),
+        ),
+        migrations.AlterField(
+            model_name="tabulardataset",
+            name="name",
+            field=models.CharField(max_length=155, unique=True),
+        ),
+        migrations.AlterField(
+            model_name="vectordataset",
+            name="name",
+            field=models.CharField(max_length=155, unique=True),
+        ),
+    ]

vbos/datasets/models.py

@@ -1,15 +1,19 @@
 from django.contrib.gis.db import models
+from django.conf import settings
 from django.core.validators import FileExtensionValidator
 from django.db.models.fields.files import default_storage
 from django.db.models.signals import pre_delete
 from django.dispatch import receiver
 
+UPLOAD_TO = "staging/raster/" if settings.DEBUG else "production/raster/"
+
 
 class RasterFile(models.Model):
-    name = models.CharField(max_length=155)
+    name = models.CharField(max_length=155, unique=True)
     created = models.DateTimeField(auto_now_add=True)
     file = models.FileField(
-        upload_to="raster/",
+        upload_to=UPLOAD_TO,
+        unique=True,
         validators=[
             FileExtensionValidator(
                 allowed_extensions=["tiff", "tif", "geotiff", "gtiff"]
@@ -36,7 +40,7 @@ def delete_raster_file(sender, instance, **kwargs):
 
 
 class RasterDataset(models.Model):
-    name = models.CharField(max_length=155)
+    name = models.CharField(max_length=155, unique=True)
     created = models.DateTimeField(auto_now_add=True)
     updated = models.DateTimeField(auto_now=True)
     file = models.ForeignKey(RasterFile, on_delete=models.PROTECT)
@@ -49,7 +53,7 @@ class Meta:
 
 
 class VectorDataset(models.Model):
-    name = models.CharField(max_length=155)
+    name = models.CharField(max_length=155, unique=True)
     created = models.DateTimeField(auto_now_add=True)
     updated = models.DateTimeField(auto_now=True)
 
@@ -73,7 +77,7 @@ class Meta:
 
 
 class TabularDataset(models.Model):
-    name = models.CharField(max_length=155)
+    name = models.CharField(max_length=155, unique=True)
     created = models.DateTimeField(auto_now_add=True)
     updated = models.DateTimeField(auto_now=True)
 

vbos/datasets/test/test_raster_models.py

@@ -4,13 +4,12 @@
 from django.core.exceptions import ValidationError
 
 from vbos.datasets.models import RasterDataset, RasterFile
-from genericpath import exists
 
 
 class TestRasterModels(TestCase):
     def setUp(self):
         self.valid_file = SimpleUploadedFile(
-            "rainfall.tif", b"file_content", content_type="image/tiff"
+            "rainfall.tiff", b"file_content", content_type="image/tiff"
         )
         self.r_1 = RasterFile.objects.create(name="Rainfall COG", file=self.valid_file)
         self.r_2 = RasterFile.objects.create(
@@ -22,6 +21,17 @@ def test_deletion(self):
         # RasterFile can't be deleted if it's associates with a dataset
         with self.assertRaises(ProtectedError):
             self.r_1.delete()
+
+        # name should be unique
+        raster = RasterFile(name="Rainfall COG 2", file="raster/coastline.tiff")
+        with self.assertRaises(ValidationError):
+            raster.full_clean()
+
+        # file path should be unique
+        raster = RasterFile(name="Rainfall COG", file="newfile.tif")
+        with self.assertRaises(ValidationError):
+            raster.full_clean()
+
         # modify dataset
         self.dataset.file = self.r_2
         self.dataset.save()
@@ -42,3 +52,6 @@ def test_deletion(self):
         raster = RasterFile(name="Test", file=invalid_file)
         with self.assertRaises(ValidationError):
             raster.full_clean()
+
+    def tearDown(self):
+        RasterFile.objects.all().delete()