TG-940 Add local MinIO service

niccolomineo · niccolomineo · commit f62a4661eea5 · 2024-01-20T00:20:31.000+01:00
diff --git a/README.md b/README.md
@@ -33,6 +33,8 @@ python3 -m pip install -r requirements/common.txt
 
 The `terraform` cli package is required, unless you want to generate a project only locally. To install it we suggest to use the official [install guide](https://learn.hashicorp.com/tutorials/terraform/install-cli).
 
+Should you opt for an S3-like object storage, you must install and launch MinIO Server's `mc` package as outlined in this [install guide](https://min.io/docs/minio/linux/index.html).
+
 ## 🔑 Credentials (optional)
 
 ### 🦊 GitLab
@@ -202,6 +204,18 @@ If you don't want DigitalOcean DNS configuration the following args are required
 | local           | Docker Volume are used to store media       | `--media-storage=local`           |
 | none            | Project have no media                       | `--media-storage=none`            |
 
+#### Local S3 storage
+
+For enabling a local S3-like object storage the following argument is needed:
+
+For enabling redis integration the following arguments are needed:
+
+`--local-s3-storage`
+
+Disabled arg:
+
+`--no-local-s3-storage`
+
 #### Redis
 
 For enabling redis integration the following arguments are needed:
diff --git a/bootstrap/collector.py b/bootstrap/collector.py
@@ -61,6 +61,7 @@ class Collector:
     sentry_org: str | None = None
     sentry_url: str | None = None
     media_storage: str | None = None
+    local_s3_storage: bool | None = None
     gitlab_url: str | None = None
     gitlab_token: str | None = None
     gitlab_namespace_path: str | None = None
@@ -89,6 +90,7 @@ def collect(self):
         self.set_sentry()
         self.set_gitlab()
         self.set_media_storage()
+        self.set_local_s3_storage()
 
     def set_project_slug(self):
         """Set the project slug option."""
@@ -286,6 +288,13 @@ def set_media_storage(self):
                 type=click.Choice(MEDIA_STORAGE_CHOICES, case_sensitive=False),
             ).lower()
 
+    def set_local_s3_storage(self):
+        """Set the local S3 storage option."""
+        if "s3" in self.media_storage and self.local_s3_storage is None:
+            self.local_s3_storage = click.confirm(
+                warning("Do you want to use the local S3 storage?"), default=False
+            )
+
     def get_runner(self):
         """Get the bootstrap runner instance."""
         return Runner(
@@ -315,6 +324,7 @@ def get_runner(self):
             sentry_org=self.sentry_org,
             sentry_url=self.sentry_url,
             media_storage=self.media_storage,
+            local_s3_storage=self.local_s3_storage,
             use_redis=self.use_redis,
             gitlab_url=self.gitlab_url,
             gitlab_token=self.gitlab_token,
diff --git a/bootstrap/runner.py b/bootstrap/runner.py
@@ -73,6 +73,7 @@ class Runner:
     sentry_url: str | None = None
     media_storage: str
     use_redis: bool = False
+    local_s3_storage: bool = False
     gitlab_url: str | None = None
     gitlab_namespace_path: str | None = None
     gitlab_token: str | None = None
@@ -251,6 +252,7 @@ def init_service(self):
                 "terraform_cloud_organization": self.terraform_cloud_organization,
                 "tfvars": self.tfvars,
                 "use_redis": self.use_redis and "true" or "false",
+                "local_s3_storage": self.local_s3_storage and "true" or "false",
                 "use_vault": self.vault_url and "true" or "false",
             },
             output_dir=self.output_dir,
diff --git a/cookiecutter.json b/cookiecutter.json
@@ -10,6 +10,7 @@
   "terraform_cloud_organization": "",
   "media_storage": ["digitalocean-s3", "other-s3", "local", "none"],
   "use_redis": "false",
+  "local_s3_storage": "false",
   "use_vault": "false",
   "environments_distribution": "1",
   "resources": {
diff --git a/start.py b/start.py
@@ -62,6 +62,7 @@
     "--media-storage",
     type=click.Choice(MEDIA_STORAGE_CHOICES, case_sensitive=False),
 )
+@click.option("--local-s3-storage/--no-local-s3-storage", is_flag=True, default=None)
 @click.option("--use-redis/--no-redis", is_flag=True, default=None)
 @click.option("--gitlab-url")
 @click.option("--gitlab-token", envvar=GITLAB_TOKEN_ENV_VAR)
diff --git a/{{cookiecutter.project_dirname}}/.env_template b/{{cookiecutter.project_dirname}}/.env_template
@@ -3,7 +3,8 @@ CACHE_URL=locmem://
 COMPOSE_FILE=docker-compose.yaml
 DATABASE_URL=postgres://postgres:postgres@postgres:5432/{{ cookiecutter.project_slug }}
 DJANGO_ADMINS=admin,errors@example.org
-DJANGO_ALLOWED_HOSTS=localhost,{{ cookiecutter.service_slug }}
+DJANGO_ALLOWED_HOSTS=localhost,{{ cookiecutter.service_slug }}{% if cookiecutter.local_s3_storage == "true" %}
+DJANGO_AWS_S3_URL=http://minio-admin:minio-admin@172.20.0.11:9000/{{ cookiecutter.project_slug }}{% endif %}
 DJANGO_CONFIGURATION=Local
 DJANGO_DEBUG=True
 DJANGO_DEFAULT_FROM_EMAIL=info@example.org
diff --git a/{{cookiecutter.project_dirname}}/Dockerfile b/{{cookiecutter.project_dirname}}/Dockerfile
@@ -69,10 +69,11 @@ RUN apt-get update \
         make \
         openssh-client \
         postgresql-client
+RUN curl https://dl.min.io/client/mc/release/linux-amd64/mc > /usr/bin/minio-client
 USER $APPUSER
 COPY --chown=$APPUSER ./requirements/local.txt requirements/local.txt
 RUN python3 -m pip install --user --no-cache-dir -r requirements/local.txt
 COPY --chown=$APPUSER . .
 RUN DJANGO_SECRET_KEY=build python3 -m manage collectstatic --clear --link --noinput
-ENTRYPOINT ["./scripts/entrypoint.sh"]
+ENTRYPOINT ["./scripts/entrypoint_local.sh"]
 CMD python3 -m manage runserver 0.0.0.0:${INTERNAL_SERVICE_PORT}
diff --git a/{{cookiecutter.project_dirname}}/Makefile b/{{cookiecutter.project_dirname}}/Makefile
@@ -20,6 +20,11 @@ compilemessages:  ## Django compilemessages
 coverage:  ## Run coverage
 	./scripts/coverage.sh
 
+.PHONY: createbucket
+createbucket:  ## Create MinIO bucket
+	minio-client mb --quiet minio/{{ cookiecutter.project_slug }};
+	minio-client anonymous set public minio/{{ cookiecutter.project_slug }};
+
 .PHONY: createsuperuser
 createsuperuser:  ## Django createsuperuser
 	python3 -m manage createsuperuser --noinput
diff --git a/{{cookiecutter.project_dirname}}/docker-compose.yaml b/{{cookiecutter.project_dirname}}/docker-compose.yaml
@@ -10,12 +10,15 @@ services:
     image: ${{ "{" }}{{ cookiecutter.service_slug|upper }}_IMAGE_NAME:-{{ cookiecutter.project_slug }}_{{ cookiecutter.service_slug }}}:${{ "{" }}{{ cookiecutter.service_slug|upper }}_IMAGE_TAG:-latest}
     depends_on:
       postgres:
-        condition: service_healthy
+        condition: service_healthy{% if cookiecutter.local_s3_storage == "true" %}
+      minio:
+        condition: service_healthy{% endif %}
     environment:
       - CACHE_URL
       - DATABASE_URL=${DATABASE_URL:-postgres://postgres:postgres@postgres:5432/{{ cookiecutter.project_slug }}}
       - DJANGO_ADMINS
       - DJANGO_ALLOWED_HOSTS
+      - DJANGO_AWS_S3_URL
       - DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION:-Testing}
       - DJANGO_DEBUG
       - DJANGO_DEFAULT_FROM_EMAIL
@@ -32,7 +35,10 @@ services:
       - PYTHONBREAKPOINT
     ports:
       - "${{ '{' }}{{ cookiecutter.service_slug|upper }}_PORT:-{{ cookiecutter.internal_service_port }}{{ '}' }}:${INTERNAL_SERVICE_PORT:-{{ cookiecutter.internal_service_port }}{{ '}' }}"
-    user: ${USER:-appuser}
+    user: ${USER:-appuser}{% if "s3" in cookiecutter.media_storage %}
+    networks:
+      custom:
+        ipv4_address: 172.20.0.10{% endif %}
 
   postgres:
     environment:
@@ -46,7 +52,39 @@ services:
       retries: 30
     image: postgres:14-bullseye
     volumes:
-      - pg_data:/var/lib/postgresql/data
+      - pg_data:/var/lib/postgresql/data{% if cookiecutter.local_s3_storage == "true" %}
+    networks:
+      custom:
+        ipv4_address: 172.20.0.11
+
+  minio:
+    command: minio server /var/lib/minio/data --console-address ":9001"
+    environment:
+      - MINIO_ENDPOINT=http://minio:9000
+      - MINIO_BUCKET_NAME={{ cookiecutter.project_slug }}
+      - MINIO_ROOT_USER=minio-admin
+      - MINIO_ROOT_PASSWORD=minio-admin
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 3s
+      timeout: 3s
+      retries: 30
+    image: minio/minio:latest
+    ports:
+      - 9000:9000
+      - 9001:9001
+    volumes:
+      - minio_data:/var/lib/minio/data
+    networks:
+      custom:
+        ipv4_address: 172.20.0.13{% endif %}
 
 volumes:
-  pg_data: {}
+  pg_data: {}{% if cookiecutter.local_s3_storage == "true" %}
+  minio_data: {}
+
+networks:
+  custom:
+    ipam:
+      config:
+      - subnet: 172.20.0.0/16{% endif %}
diff --git a/{{cookiecutter.project_dirname}}/requirements/common.in b/{{cookiecutter.project_dirname}}/requirements/common.in
@@ -1,3 +1,4 @@
 -r base.in
 django-configurations[cache,database,email]~=2.5.0
-django~=5.0.0
+{% if "s3" in cookiecutter.media_storage %}django-storages[boto3]~=1.14.0
+{% endif %}django~=5.0.0
diff --git a/{{cookiecutter.project_dirname}}/requirements/remote.in b/{{cookiecutter.project_dirname}}/requirements/remote.in
@@ -1,7 +1,6 @@
 -r common.in
 argon2-cffi~=23.1.0
-{% if "s3" in cookiecutter.media_storage %}django-storages[boto3]~=1.14.0
-{% endif %}gunicorn~=21.2.0
+gunicorn~=21.2.0
 {% if cookiecutter.use_redis == "true" %}redis~=5.0.0
 {% endif %}sentry-sdk~=1.39.0
 uvicorn[standard]~=0.25.0
diff --git a/{{cookiecutter.project_dirname}}/scripts/entrypoint_local.sh b/{{cookiecutter.project_dirname}}/scripts/entrypoint_local.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+./entrypoint.sh
+minio-client alias set minio http://minio:9000 minio-admin minio-admin;
+minio-client mb --quiet --ignore-existing minio/{{ cookiecutter.project_slug }};
+minio-client anonymous set public minio/{{ cookiecutter.project_slug }};
+exec "${@}"
diff --git a/{{cookiecutter.project_dirname}}/{{cookiecutter.django_settings_dirname}}/settings.py b/{{cookiecutter.project_dirname}}/{{cookiecutter.django_settings_dirname}}/settings.py
@@ -10,6 +10,7 @@
 https://docs.djangoproject.com/en/stable/ref/settings/
 """
 
+import re
 import string
 from copy import deepcopy
 from pathlib import Path
@@ -142,6 +143,60 @@ class ProjectDefault(Configuration):
 
     # MEDIA_ROOT = BASE_DIR / "media"{% endif %}
 
+    {% if cookiecutter.local_s3_storage == "true" %}# Django Storages
+    # https://django-storages.readthedocs.io/en/latest/
+
+    AWS_S3_URL = values.Value()
+
+    @property
+    def AWS_S3_URL_PARTS(self):
+        """Return the AWS S3 URL parts."""
+        pattern = re.compile(
+            r"(?P<scheme>https?):\/\/(?P<access_key_id>.+?):(?P<secret_access_key>.+?)"
+            r"@(?P<host>.+?)(?::(?P<port>\d*?))?\/(?P<bucket_name>.+)"
+        )
+        return pattern.match(self.AWS_S3_URL).groupdict()
+
+    @property
+    def AWS_ACCESS_KEY_ID(self):
+        """Return the AWS access key id."""
+        return self.AWS_URL_PARTS["access_key_id"]
+
+    @property
+    def AWS_SECRET_ACCESS_KEY(self):
+        """Return the AWS secret access key."""
+        return self.AWS_S3_URL_PARTS["secret_access_key"]
+
+    @property
+    def AWS_STORAGE_BUCKET_NAME(self):
+        """Return the AWS storage bucket name."""
+        return self.AWS_S3_URL_PARTS["bucket_name"]
+
+    @property
+    def AWS_S3_ENDPOINT_URL(self):
+        """Return the AWS S3 endpoint URL."""
+        return (
+            f'{self.AWS_S3_URL_PARTS["scheme"]}://'
+            f'{self.AWS_S3_URL_PARTS["host"]}'
+            f':{self.AWS_S3_URL_PARTS["port"]}'
+        ){% endif %}
+
+    AWS_LOCATION = values.Value("")
+
+    AWS_S3_FILE_OVERWRITE = values.BooleanValue(False)
+
+    # Storage
+    # https://docs.djangoproject.com/en/stable/ref/files/storage/
+
+    STORAGES = {
+        "default": {
+            "BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
+        },
+        "staticfiles": {
+            "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+        },
+    }{% endif %}
+
     # Email Settings
     # https://docs.djangoproject.com/en/stable/topics/email/
 
@@ -285,15 +340,16 @@ class Testing(ProjectDefault):
 
     # Storages
     # https://docs.djangoproject.com/en/stable/ref/settings/#std-setting-STORAGES
-
+    {% if "s3" not in cookiecutter.media_storage %}
     STORAGES = {
         "default": {
             "BACKEND": "django.core.files.storage.InMemoryStorage",
         },
         "staticfiles": {
             "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
         },
-    }
+    }{% endif %}
+
 
     # The MD5 based password hasher is much less secure but faster
     # https://docs.djangoproject.com/en/stable/topics/auth/passwords/
@@ -403,12 +459,7 @@ def DATABASES(self):  # pragma: no cover
     @property
     def STORAGES(self):  # pragma: no cover
         """Return the storage settings."""
-        storages = deepcopy(
-            ProjectDefault.STORAGES
-        )  # noqa{% if "s3" in cookiecutter.media_storage %}
-        storages["default"][
-            "BACKEND"
-        ] = "storages.backends.s3boto3.S3Boto3Storage"  # noqa{% endif %}
+        storages = deepcopy(ProjectDefault.STORAGES)
         try:
             # WhiteNoise
             # http://whitenoise.evans.io/en/stable/django.html
@@ -420,7 +471,20 @@ def STORAGES(self):  # pragma: no cover
             storages["staticfiles"][
                 "BACKEND"
             ] = "whitenoise.storage.CompressedManifestStaticFilesStorage"
-        return storages
+        return storages{% if "s3" in cookiecutter.media_storage %}
+
+    # Django Storages
+    # https://django-storages.readthedocs.io/en/latest/{% if cookiecutter.local_s3_storage == "true" %}
+
+    AWS_S3_ENDPOINT_URL = None{% endif %}{% else %}
+
+    AWS_LOCATION = values.Value("")
+
+    AWS_S3_ENDPOINT_URL = values.Value()
+
+    AWS_S3_FILE_OVERWRITE = values.BooleanValue(False)
+
+    AWS_STORAGE_BUCKET_NAME = values.Value(){% endif %}{% endif %}
 
     # Sentry
     # https://sentry.io/for/django/
@@ -436,21 +500,10 @@ def STORAGES(self):  # pragma: no cover
         sentry_sdk.init(
             integrations=[DjangoIntegration(), RedisIntegration()],
             send_default_pii=True,
-        )  # noqa{% else %}
+        ){% else %}
         from sentry_sdk.integrations.django import DjangoIntegration
 
         sentry_sdk.init(
             integrations=[DjangoIntegration()],
             send_default_pii=True,
-        )  # noqa{% endif %}{% if "s3" in cookiecutter.media_storage %}
-
-    # Django Storages
-    # https://django-storages.readthedocs.io/en/stable/
-
-    AWS_LOCATION = values.Value("")
-
-    AWS_S3_ENDPOINT_URL = values.Value()
-
-    AWS_S3_FILE_OVERWRITE = values.BooleanValue(False)
-
-    AWS_STORAGE_BUCKET_NAME = values.Value()  # noqa{% endif %}
+        ){% endif %}

Original file line number	Diff line number	Diff line change
`@@ -62,6 +62,7 @@`
`62`	`62`	`"--media-storage",`
`63`	`63`	`type=click.Choice(MEDIA_STORAGE_CHOICES, case_sensitive=False),`
`64`	`64`	`)`
	`65`	`+@click.option("--local-s3-storage/--no-local-s3-storage", is_flag=True, default=None)`
`65`	`66`	`@click.option("--use-redis/--no-redis", is_flag=True, default=None)`
`66`	`67`	`@click.option("--gitlab-url")`
`67`	`68`	`@click.option("--gitlab-token", envvar=GITLAB_TOKEN_ENV_VAR)`