Feat: support for optional sample data and additional data migrations (#827)

Author: angela-tran

.gitignore

@@ -1,8 +1,8 @@
 *.db
 *.env
 *.mo
-fixtures/*.json
-!fixtures/??_*.json
+benefits/core/migrations/0002_*.py
+!benefits/core/migrations/0002_sample_data.py
 static/
 !benefits/static
 benefits/static/sha.txt

benefits/core/migrations/0002_sample_data.py

@@ -1,8 +1,16 @@
+"""Data migration which loads sample data.
+Set environment variable DJANGO_LOAD_SAMPLE_DATA to False to skip loading sample data.
+"""
+from django.conf import settings
 from django.db import migrations
 from django.utils.translation import gettext_lazy as _
 
 
-def load_initial_data(app, *args, **kwargs):
+def load_sample_data(app, *args, **kwargs):
+    if not settings.LOAD_SAMPLE_DATA:
+        print("  LOAD_SAMPLE_DATA is set to False, skipping sample data")
+        return
+
     EligibilityType = app.get_model("core", "EligibilityType")
 
     type1 = EligibilityType.objects.create(name="type1", label="Eligibility Type 1", group_id="group1")
@@ -220,5 +228,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(load_initial_data),
+        migrations.RunPython(load_sample_data),
     ]

benefits/core/migrations/0003_data_migration_order.py

@@ -0,0 +1,15 @@
+"""This migration simply ensures that data migrations (assumed to be files that start with 0002) will run sequentially.
+It does so by looking for all migration files starting with "0002" and declaring them in its dependency list.
+"""
+import os
+
+from django.conf import settings
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    migrations_directory = os.path.join(settings.BASE_DIR, "benefits", "core", "migrations")
+    dependencies = [("core", file.replace(".py", "")) for file in os.listdir(migrations_directory) if file.startswith("0002")]
+
+    operations = []

benefits/settings.py

@@ -172,6 +172,8 @@ def _filter_empty(ls):
     }
 }
 
+LOAD_SAMPLE_DATA = os.environ.get("DJANGO_LOAD_SAMPLE_DATA", "true").lower() != "false"
+
 # Password validation
 
 AUTH_PASSWORD_VALIDATORS = []

bin/init.sh

@@ -8,6 +8,15 @@ rm -f django.db
 
 # run database migrations
 
+if [[ ${DJANGO_LOAD_SAMPLE_DATA:-true} = false ]]; then
+    if [[ -d ${DJANGO_MIGRATIONS_DIR:-false} ]]; then
+        echo "Copying migrations from ${DJANGO_MIGRATIONS_DIR}"
+        cp ${DJANGO_MIGRATIONS_DIR}/0002_*.py ./benefits/core/migrations/
+    else
+        echo "DJANGO_MIGRATIONS_DIR is either unset or not a directory"
+    fi
+fi
+
 python manage.py migrate
 
 # create a superuser account for backend admin access

docs/configuration/data.md

@@ -4,6 +4,12 @@
 
     [`benefits/core/migrations/0002_sample_data.py`][data-sample]
 
+!!! example "Helper migration file to define data migration order"
+
+    [`benefits/core/migrations/0003_data_migration_order.py`][helper-migration]
+
+    Since our initial data migration files are assumed to start with `0002` and depend only on `0001_initial`, this migration file ensures the migration graph has a defined order (which is required for the migration to run). It looks in the `benefits/core/migrations` directory for migration files starting with `0002` and declares them in its dependencies list.
+
 !!! tldr "Django docs"
 
     [How to provide initial data for models][django-load-initial-data]
@@ -62,8 +68,38 @@ Run these commands from within the repository root, inside the devcontainer:
 bin/init.sh
 ```
 
+## Loading new data for different environment
+
+Django will run all migration files found in an app's `migrations` module.
+
+To load new data for a different environment:
+
+1. (Optional) Set an environment variable `DJANGO_LOAD_SAMPLE_DATA` to `false` if you don't want the `core` app's sample data to be loaded.
+1. Create a data migration file, and make sure the name is prefixed with `0002`. (The migration process for Benefits expects data migration files to named as such). The basic structure for the contents of this file is:
+```python
+from django.db import migrations
+
+
+def load_data(app, *args, **kwargs):
+    pass
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.RunPython(load_data),
+    ]
+```
+1. Put this file under `benefits/core/migrations`
+1. If `DJANGO_LOAD_SAMPLE_DATA` is `false`, you can also set [`DJANGO_MIGRATIONS_DIR`](../environment-variables/#django_migrations_dir) to a directory path, and put your data migration there.
+
+
 [core-models]: https://github.com/cal-itp/benefits/blob/dev/benefits/core/models.py
 [django-load-initial-data]: https://docs.djangoproject.com/en/4.0/howto/initial-data/
 [eligibility-server]: https://docs.calitp.org/eligibility-server
 [data-sample]: https://github.com/cal-itp/benefits/tree/dev/benefits/core/migrations/0002_sample_data.py
+[helper-migration]: https://github.com/cal-itp/benefits/tree/dev/benefits/core/migrations/0003_data_migration_order.py
 [init]: https://github.com/cal-itp/benefits/blob/dev/bin/init.sh

docs/configuration/environment-variables.md

@@ -73,6 +73,19 @@ Boolean:
   files are served
 - `False` (default): the application is launched with debug mode turned off, similar to how it runs in production
 
+### `DJANGO_LOAD_SAMPLE_DATA`
+
+!!! warning "Deployment configuration"
+
+    You may change this setting when deploying the app to a non-localhost domain
+
+Boolean:
+
+- `True` (default): The sample data in `benefits/core/migrations/0002_sample_data.py` is used to initialize the Django configuration database.
+- `False`: The sample data from `benefits/core/migrations/0002_sample_data.py` will not be loaded.
+
+See [Configuration data](data.md) for more.
+
 ### `DJANGO_LOCAL_PORT`
 
 !!! info "Local configuration"
@@ -100,6 +113,15 @@ The log level used in the application's logging configuration.
 
 By default the application sends logs to `stdout`.
 
+
+### `DJANGO_MIGRATIONS_DIR`
+
+!!! warning "Deployment configuration"
+
+    You may change this setting when deploying the app to a non-localhost domain
+
+If [`DJANGO_LOAD_SAMPLE_DATA`](#django_load_sample_data) is `false`, then you can set `DJANGO_MIGRATIONS_DIR` to the path of a directory containing data migrations that you want to be run. Those data migration files need to be prefixed with `0002` so that the [helper migration file](data.md)) can find it. See [Configuration data](./data.md) for more on loading data for different environments.
+
 ### `DJANGO_SECRET_KEY`
 
 !!! warning "Deployment configuration"