[#2903] Support import of Zaaktype configs without library

Author: pi-sigma

src/open_inwoner/openzaak/admin.py

@@ -17,7 +17,7 @@
 from solo.admin import SingletonModelAdmin
 
 from open_inwoner.ckeditor5.widgets import CKEditorWidget
-from open_inwoner.openzaak.import_export import CatalogusConfigImport, ZGWConfigExport
+from open_inwoner.openzaak.import_export import ZGWConfigExport, ZGWConfigImport
 from open_inwoner.utils.forms import LimitedUploadFileField
 
 from .models import (
@@ -131,7 +131,7 @@ def get_urls(self):
             path(
                 "import-catalogus-dump/",
                 self.admin_site.admin_view(self.process_file_view),
-                name="upload_zgw_import_file",
+                name="upload_catalogus_import_file",
             ),
         ]
         return custom_urls + urls
@@ -161,7 +161,7 @@ def process_file_view(self, request):
 
                 try:
                     import_result = (
-                        CatalogusConfigImport.import_from_jsonl_file_in_django_storage(
+                        ZGWConfigImport.import_from_jsonl_file_in_django_storage(
                             target_file_name,
                             storage,
                         )
@@ -371,6 +371,7 @@ def has_delete_permission(self, request, obj=None):
 
 @admin.register(ZaakTypeConfig)
 class ZaakTypeConfigAdmin(admin.ModelAdmin):
+    change_list_template = "admin/zaaktypeconfig_change_list.html"
     inlines = [
         ZaakTypeInformatieObjectTypeConfigInline,
         ZaakTypeStatusTypeConfigInline,
@@ -434,6 +435,17 @@ class ZaakTypeConfigAdmin(admin.ModelAdmin):
     ]
     ordering = ("identificatie", "catalogus__domein")
 
+    def get_urls(self):
+        urls = super().get_urls()
+        custom_urls = [
+            path(
+                "import-zaaktype-dump/",
+                self.admin_site.admin_view(self.process_file_view),
+                name="upload_zaaktype_import_file",
+            ),
+        ]
+        return custom_urls + urls
+
     @admin.action(description=_("Export to file"))
     def export_zaaktype_configs(modeladmin, request, queryset):
         export = ZGWConfigExport.from_zaaktype_configs(queryset)
@@ -446,6 +458,71 @@ def export_zaaktype_configs(modeladmin, request, queryset):
         ] = 'attachment; filename="zgw-zaaktype-export.json"'
         return response
 
+    def process_file_view(self, request):
+        form = ImportZGWExportFileForm()
+
+        if request.method == "POST":
+            form = ImportZGWExportFileForm(request.POST, request.FILES)
+            if form.is_valid():
+                storage = PrivateMediaFileSystemStorage()
+                timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+                target_file_name = f"zgw_import_dump_{timestamp}.json"
+                storage.save(target_file_name, request.FILES["zgw_export_file"])
+
+                try:
+                    import_result = (
+                        ZGWConfigImport.import_from_jsonl_file_in_django_storage(
+                            target_file_name,
+                            storage,
+                        )
+                    )
+                    self.message_user(
+                        request,
+                        _(
+                            "%(num_rows)d item(s) processed in total, with %(error_rows)d failing row(s)."
+                            % {
+                                "num_rows": import_result.total_rows_processed,
+                                "error_rows": len(import_result.import_errors),
+                            }
+                        ),
+                        messages.SUCCESS
+                        if not import_result.import_errors
+                        else messages.WARNING,
+                    )
+                    if errors := import_result.import_errors:
+                        msgs_deduped = set(error.__str__() for error in errors)
+                        error_msg_iterator = ([msg] for msg in msgs_deduped)
+
+                        error_msg_html = format_html_join(
+                            "\n", "<p> - {}</p>", error_msg_iterator
+                        )
+                        error_msg_html = format_html(
+                            _("It was not possible to import the following items:")
+                            + f"<div>{error_msg_html}</div>"
+                        )
+                        self.message_user(request, error_msg_html, messages.ERROR)
+
+                    return HttpResponseRedirect(
+                        reverse(
+                            "admin:openzaak_zaaktypeconfig_changelist",
+                        )
+                    )
+                except Exception:
+                    logger.exception("Unable to process ZGW import")
+                    self.message_user(
+                        request,
+                        _(
+                            "We were unable to process your upload. Please regenerate the file and try again."
+                        ),
+                        messages.ERROR,
+                    )
+                finally:
+                    storage.delete(target_file_name)
+
+        return TemplateResponse(
+            request, "admin/import_zgw_export_form.html", {"form": form}
+        )
+
     def has_add_permission(self, request):
         return False
 

src/open_inwoner/openzaak/import_export.py

@@ -263,7 +263,7 @@ def from_zaaktype_configs(cls, zaaktype_configs: QuerySet) -> Self:
 
 
 @dataclasses.dataclass(frozen=True)
-class CatalogusConfigImport:
+class ZGWConfigImport:
     """Import CatalogusConfig(s) and all associated relations."""
 
     total_rows_processed: int = 0

src/open_inwoner/openzaak/tests/test_admin.py

@@ -191,7 +191,7 @@ def test_import_flow_reports_success(self) -> None:
 
         form = self.app.get(
             reverse(
-                "admin:upload_zgw_import_file",
+                "admin:upload_catalogus_import_file",
             ),
             user=self.user,
         ).form
@@ -218,13 +218,13 @@ def test_import_flow_reports_success(self) -> None:
         )
 
     @mock.patch(
-        "open_inwoner.openzaak.import_export.CatalogusConfigImport.import_from_jsonl_file_in_django_storage"
+        "open_inwoner.openzaak.import_export.ZGWConfigImport.import_from_jsonl_file_in_django_storage"
     )
     def test_import_flow_errors_reports_failure_to_user(self, m) -> None:
         m.side_effect = Exception("something went wrong")
         form = self.app.get(
             reverse(
-                "admin:upload_zgw_import_file",
+                "admin:upload_catalogus_import_file",
             ),
             user=self.user,
         ).form
@@ -254,7 +254,7 @@ def test_import_flow_errors_reports_failure_to_user(self, m) -> None:
         self.assertEqual(
             response.request.path,
             reverse(
-                "admin:upload_zgw_import_file",
+                "admin:upload_catalogus_import_file",
             ),
         )
 
@@ -272,7 +272,7 @@ def test_import_flow_reports_errors(self) -> None:
 
         form = self.app.get(
             reverse(
-                "admin:upload_zgw_import_file",
+                "admin:upload_catalogus_import_file",
             ),
             user=self.user,
         ).form
@@ -333,7 +333,7 @@ def test_import_flow_reports_partial_errors(self) -> None:
 
         form = self.app.get(
             reverse(
-                "admin:upload_zgw_import_file",
+                "admin:upload_catalogus_import_file",
             ),
             user=self.user,
         ).form

src/open_inwoner/openzaak/tests/test_import_export.py

@@ -5,7 +5,7 @@
 from django.core.files.storage.memory import InMemoryStorage
 from django.test import TestCase
 
-from open_inwoner.openzaak.import_export import CatalogusConfigImport, ZGWConfigExport
+from open_inwoner.openzaak.import_export import ZGWConfigExport, ZGWConfigImport
 from open_inwoner.openzaak.models import (
     CatalogusConfig,
     ZaakTypeConfig,
@@ -342,20 +342,24 @@ def setUp(self):
             '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://bar.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "statustekst nieuw", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}',
             '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "resultaattype_url": "https://bar.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[]", "description": "description new"}}',
         ]
+        self.json_dupes = [
+            '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://bar.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "statustekst nieuw", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}',
+        ]
         self.jsonl = "\n".join(self.json_lines)
+        self.jsonl_with_dupes = "\n".join(self.json_lines + self.json_dupes)
 
     def test_import_jsonl_update_success(self):
         mocks = ZGWExportImportMockData()
         self.storage.save("import.jsonl", io.StringIO(self.jsonl))
 
-        import_result = CatalogusConfigImport.import_from_jsonl_file_in_django_storage(
+        import_result = ZGWConfigImport.import_from_jsonl_file_in_django_storage(
             "import.jsonl", self.storage
         )
 
         # check import
         self.assertEqual(
             import_result,
-            CatalogusConfigImport(
+            ZGWConfigImport(
                 total_rows_processed=5,
                 catalogus_configs_imported=1,
                 zaaktype_configs_imported=1,
@@ -435,16 +439,16 @@ def test_import_jsonl_missing_statustype_config(self):
         # we use `asdict` and replace the Exceptions with string representations
         # because for Exceptions raised from within dataclasses, equality ==/is identity
         import_result = dataclasses.asdict(
-            CatalogusConfigImport.import_from_jsonl_file_in_django_storage(
+            ZGWConfigImport.import_from_jsonl_file_in_django_storage(
                 "import.jsonl", self.storage
             )
         )
         expected_error = ZGWImportError(
-            "ZaakTypeStatusTypeConfig not found in target environment: omschrijving = bogus, "
-            "ZaakTypeConfig identificatie = ztc-id-a-0, Catalogus domein = DM-0, Catalogus rsin = 123456789"
+            "ZaakTypeStatusTypeConfig not found in target environment: omschrijving = 'bogus', "
+            "ZaakTypeConfig identificatie = 'ztc-id-a-0'"
         )
         import_expected = dataclasses.asdict(
-            CatalogusConfigImport(
+            ZGWConfigImport(
                 total_rows_processed=6,
                 catalogus_configs_imported=1,
                 zaaktype_configs_imported=1,
@@ -481,16 +485,16 @@ def test_import_jsonl_update_statustype_config_missing_zt_config(self):
         # we use `asdict` and replace the Exceptions with string representations
         # because for Exceptions raised from within dataclasses, equality ==/is identity
         import_result = dataclasses.asdict(
-            CatalogusConfigImport.import_from_jsonl_file_in_django_storage(
+            ZGWConfigImport.import_from_jsonl_file_in_django_storage(
                 "import.jsonl", self.storage
             )
         )
         expected_error = ZGWImportError(
-            "ZaakTypeStatusTypeConfig not found in target environment: omschrijving = status omschrijving, "
-            "ZaakTypeConfig identificatie = bogus, Catalogus domein = DM-1, Catalogus rsin = 666666666"
+            "ZaakTypeStatusTypeConfig not found in target environment: omschrijving = 'status omschrijving', "
+            "ZaakTypeConfig identificatie = 'bogus'"
         )
         import_expected = dataclasses.asdict(
-            CatalogusConfigImport(
+            ZGWConfigImport(
                 total_rows_processed=6,
                 catalogus_configs_imported=1,
                 zaaktype_configs_imported=1,
@@ -513,7 +517,7 @@ def test_import_jsonl_update_statustype_config_missing_zt_config(self):
         self.assertEqual(ZaakTypeStatusTypeConfig.objects.count(), 1)
         self.assertEqual(ZaakTypeResultaatTypeConfig.objects.count(), 1)
 
-    def test_import_jsonl_update_reports_duplicates(self):
+    def test_import_jsonl_update_reports_duplicate_db_records(self):
         mocks = ZGWExportImportMockData()
 
         ZaakTypeResultaatTypeConfigFactory(
@@ -527,16 +531,16 @@ def test_import_jsonl_update_reports_duplicates(self):
         # we use `asdict` and replace the Exceptions with string representations
         # because for Exceptions raised from within dataclasses, equality ==/is identity
         import_result = dataclasses.asdict(
-            CatalogusConfigImport.import_from_jsonl_file_in_django_storage(
+            ZGWConfigImport.import_from_jsonl_file_in_django_storage(
                 "import.jsonl", self.storage
             )
         )
         expected_error = ZGWImportError(
-            "Got multiple results for ZaakTypeResultaatTypeConfig: omschrijving = resultaat, "
-            "ZaakTypeConfig identificatie = ztc-id-a-0, Catalogus domein = DM-0, Catalogus rsin = 123456789"
+            "Got multiple results for ZaakTypeResultaatTypeConfig: omschrijving = 'resultaat', "
+            "ZaakTypeConfig identificatie = 'ztc-id-a-0'"
         )
         import_expected = dataclasses.asdict(
-            CatalogusConfigImport(
+            ZGWConfigImport(
                 total_rows_processed=5,
                 catalogus_configs_imported=1,
                 zaaktype_configs_imported=1,
@@ -552,6 +556,39 @@ def test_import_jsonl_update_reports_duplicates(self):
         # check import
         self.assertEqual(import_result, import_expected)
 
+    def test_import_jsonl_update_reports_duplicate_natural_keys_in_upload_file(self):
+        mocks = ZGWExportImportMockData()
+
+        self.storage.save("import.jsonl", io.StringIO(self.jsonl_with_dupes))
+
+        # we use `asdict` and replace the Exceptions with string representations
+        # because for Exceptions raised from within dataclasses, equality ==/is identity
+        import_result = dataclasses.asdict(
+            ZGWConfigImport.import_from_jsonl_file_in_django_storage(
+                "import.jsonl", self.storage
+            )
+        )
+        expected_error = ZGWImportError(
+            "ZaakTypeStatusTypeConfig was processed multiple times because it contains duplicate "
+            "natural keys: omschrijving = 'status omschrijving', ZaakTypeConfig identificatie = 'ztc-id-a-0'"
+        )
+        import_expected = dataclasses.asdict(
+            ZGWConfigImport(
+                total_rows_processed=6,
+                catalogus_configs_imported=1,
+                zaaktype_configs_imported=1,
+                zaak_informatie_object_type_configs_imported=1,
+                zaak_status_type_configs_imported=1,
+                zaak_resultaat_type_configs_imported=1,
+                import_errors=[expected_error],
+            ),
+        )
+        import_result["import_errors"][0] = str(import_result["import_errors"][0])
+        import_expected["import_errors"][0] = str(import_expected["import_errors"][0])
+
+        # check import
+        self.assertEqual(import_result, import_expected)
+
     def test_import_jsonl_fails_with_catalogus_domein_rsin_mismatch(self):
         service = ServiceFactory(slug="service-0")
         CatalogusConfigFactory(
@@ -569,18 +606,16 @@ def test_import_jsonl_fails_with_catalogus_domein_rsin_mismatch(self):
         with self.assertLogs(
             logger="open_inwoner.openzaak.import_export", level="ERROR"
         ) as cm:
-            import_result = CatalogusConfigImport.from_jsonl_stream_or_string(
-                import_line
-            )
+            import_result = ZGWConfigImport.from_jsonl_stream_or_string(import_line)
             self.assertEqual(
                 cm.output,
                 [
                     # error from trying to load existing CatalogusConfig
                     "ERROR:open_inwoner.openzaak.import_export:"
-                    "CatalogusConfig not found in target environment: Domein = BAR, Rsin = 987654321",
+                    "CatalogusConfig not found in target environment: Domein = 'BAR', Rsin = '987654321'",
                     # error from deserializing nested ZGW objects
                     "ERROR:open_inwoner.openzaak.import_export:"
-                    "ZaakTypeConfig not found in target environment: Identificatie = ztc-id-a-0, Catalogus domein = DM-0, Catalogus rsin = 123456789",
+                    "ZaakTypeConfig not found in target environment: Identificatie = 'ztc-id-a-0', Catalogus domein = 'DM-0', Catalogus rsin = '123456789'",
                 ],
             )
 
@@ -598,7 +633,7 @@ def test_import_jsonl_fails_with_catalogus_domein_rsin_mismatch(self):
     def test_bad_import_types(self):
         for bad_type in (set(), list(), b""):
             with self.assertRaises(ValueError):
-                CatalogusConfigImport.from_jsonl_stream_or_string(bad_type)
+                ZGWConfigImport.from_jsonl_stream_or_string(bad_type)
 
     def test_valid_input_types_are_accepted(self):
         ZGWExportImportMockData()
@@ -609,10 +644,10 @@ def test_valid_input_types_are_accepted(self):
             self.jsonl,
         ):
             with self.subTest(f"Input type {type(input)}"):
-                import_result = CatalogusConfigImport.from_jsonl_stream_or_string(input)
+                import_result = ZGWConfigImport.from_jsonl_stream_or_string(input)
                 self.assertEqual(
                     import_result,
-                    CatalogusConfigImport(
+                    ZGWConfigImport(
                         total_rows_processed=5,
                         catalogus_configs_imported=1,
                         zaaktype_configs_imported=1,
@@ -628,9 +663,7 @@ def test_import_is_atomic(self):
         bad_jsonl = self.jsonl + "\n" + bad_line
 
         with self.assertRaises(KeyError):
-            CatalogusConfigImport.from_jsonl_stream_or_string(
-                stream_or_string=bad_jsonl
-            )
+            ZGWConfigImport.from_jsonl_stream_or_string(stream_or_string=bad_jsonl)
 
         counts = (
             CatalogusConfig.objects.count(),
@@ -654,8 +687,6 @@ def setUp(self):
 
     def test_exports_can_be_imported(self):
         export = ZGWConfigExport.from_catalogus_configs(CatalogusConfig.objects.all())
-        import_result = CatalogusConfigImport.from_jsonl_stream_or_string(
-            export.as_jsonl()
-        )
+        import_result = ZGWConfigImport.from_jsonl_stream_or_string(export.as_jsonl())
 
         self.assertEqual(import_result.total_rows_processed, 5)

src/open_inwoner/templates/admin/catalogusconfig_change_list.html

@@ -4,7 +4,7 @@
 {% block object-tools-items %}
 {{ block.super }}
 <li>
-    <a href="{% url 'admin:upload_zgw_import_file' %}" class="addlink">
+    <a href="{% url 'admin:upload_catalogus_import_file' %}" class="addlink">
         {% trans "Import from file" %}
     </a>
 </li>

src/open_inwoner/templates/admin/zaaktypeconfig_change_list.html

@@ -0,0 +1,12 @@
+{% extends "admin/change_list.html" %}
+{% load i18n admin_urls %}
+
+{% block object-tools-items %}
+{{ block.super }}
+<li>
+    <a href="{% url 'admin:upload_zaaktype_import_file' %}" class="addlink">
+        {% trans "Import from file" %}
+    </a>
+</li>
+{% endblock %}
+