Skip to content

Commit a5c818e

Browse files
pi-sigmapi-sigma
committed
[#2871] Refactor ZGW import logic
 - overwrite only editable fields when importing ZGW entities, skip read-only fields (url, domein, rsin)
1 parent 00dcc94 commit a5c818e

File tree

2 files changed

+323
-247
lines changed

2 files changed

+323
-247
lines changed

‎src/open_inwoner/openzaak/import_export.py

+188-71
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
from django.core import serializers
1010
from django.core.files.storage import Storage
11+
from django.core.serializers.base import DeserializationError
1112
from django.db import transaction
1213
from django.db.models import QuerySet
1314

@@ -22,6 +23,146 @@
2223
logger = logging.getLogger(__name__)
2324

2425

26+
class ZGWImportError(Exception):
27+
pass
28+
29+
30+
def check_catalogus_config_exists(source_config):
31+
try:
32+
CatalogusConfig.objects.get_by_natural_key(
33+
domein=source_config.domein, rsin=source_config.rsin
34+
)
35+
except CatalogusConfig.MultipleObjectsReturned:
36+
raise ZGWImportError(
37+
"Got multiple results for CatalogusConfig with domain={domein} and rsin={rsin}".format(
38+
domein=source_config.domein,
39+
rsin=source_config.rsin,
40+
)
41+
)
42+
except CatalogusConfig.DoesNotExist:
43+
raise ZGWImportError(
44+
"CatalogusConfig not found in target environment: domein={domein} and rsin={rsin}".format(
45+
domein=source_config.domein, rsin=source_config.rsin
46+
),
47+
)
48+
49+
50+
def update_zaaktype_config(source_config):
51+
catalogus_domein = source_config.catalogus.domein
52+
catalogus_rsin = source_config.catalogus.rsin
53+
54+
try:
55+
target = ZaakTypeConfig.objects.get_by_natural_key(
56+
identificatie=source_config.identificatie,
57+
catalogus_domein=catalogus_domein,
58+
catalogus_rsin=source_config.catalogus.rsin,
59+
)
60+
except ZaakTypeConfig.MultipleObjectsReturned:
61+
raise ZGWImportError(
62+
"Got multiple results for ZaakTypeConfig with identificatie={identificatie}, "
63+
"catalogus domein={domein} and catalogus_rsin={rsin}".format(
64+
identificatie=source_config.identificatie,
65+
domein=catalogus_domein,
66+
rsin=catalogus_rsin,
67+
)
68+
)
69+
except (CatalogusConfig.DoesNotExist, ZaakTypeConfig.DoesNotExist):
70+
raise ZGWImportError(
71+
"ZaakTypeConfig not found in target environment: identificatie={identificatie}, "
72+
"catalogus domein={domein}, catalogus rsin={rsin}".format(
73+
identificatie=source_config.identificatie,
74+
domein=source_config.domein,
75+
rsin=source_config.rsin,
76+
),
77+
)
78+
else:
79+
update_fields = [
80+
"notify_status_changes",
81+
"external_document_upload_url",
82+
"document_upload_enabled",
83+
"contact_form_enabled",
84+
"contact_subject_code",
85+
"relevante_zaakperiode",
86+
]
87+
for field in update_fields:
88+
val = getattr(source_config, field, None)
89+
setattr(target, field, val)
90+
target.save()
91+
92+
93+
def _update_nested_zgw_config(source_config: type, update_fields: list[str]):
94+
zaaktype_config_identificatie = source_config.zaaktype_config.identificatie
95+
catalogus_domein = source_config.zaaktype_config.catalogus.domein
96+
catalogus_rsin = source_config.zaaktype_config.catalogus.rsin
97+
98+
try:
99+
target = source_config.__class__.objects.get_by_natural_key(
100+
omschrijving=source_config.omschrijving,
101+
zaak_type_config_identificatie=zaaktype_config_identificatie,
102+
catalogus_domein=catalogus_domein,
103+
catalogus_rsin=catalogus_rsin,
104+
)
105+
except ZaakTypeInformatieObjectTypeConfig.MultipleObjectsReturned:
106+
raise ZGWImportError(
107+
f"Got multiple results for {source_config.__class__.__name__} with: "
108+
"zaaktype config={zaaktype_config_identificatie}, catalogus_domein={catalogus_domein}, "
109+
"catalogus_rsin={catalogus_rsin}".format(
110+
zaaktype_config_identificatie=zaaktype_config_identificatie,
111+
catalogus_domein=catalogus_domein,
112+
catalogus_rsin=catalogus_rsin,
113+
),
114+
)
115+
except ZaakTypeInformatieObjectTypeConfig.DoesNotExist:
116+
raise ZGWImportError(
117+
f"{source_config.__class__.__name__} not found in target environment: "
118+
"omschrijving={omschrijving}, catalogus domein={domein}, catalogus rsin={rsin}".format(
119+
omschrijving=source_config.omschrijving,
120+
domein=catalogus_domein,
121+
rsin=catalogus_rsin,
122+
),
123+
)
124+
else:
125+
for field in update_fields:
126+
val = getattr(source_config, field, None)
127+
setattr(target, field, val)
128+
target.save()
129+
130+
131+
def update_zaaktype_informatie_objecttype_config(source_config):
132+
update_fields = [
133+
"zaaktype_uuids",
134+
"document_upload_enabled",
135+
"document_notification_enabled",
136+
]
137+
_update_nested_zgw_config(source_config, update_fields)
138+
139+
140+
def update_zaaktype_statustype_config(source_config):
141+
update_fields = [
142+
"statustekst",
143+
"zaaktype_uuids",
144+
"status_indicator",
145+
"status_indicator_text",
146+
"document_upload_description",
147+
"desciption",
148+
"notify_status_change",
149+
"action_required",
150+
"document_upload_enabled",
151+
"call_to_action_url",
152+
"call_to_action_text",
153+
"case_link_text",
154+
]
155+
_update_nested_zgw_config(source_config, update_fields)
156+
157+
158+
def update_zaaktype_resultaattype_config(source_config):
159+
update_fields = [
160+
"zaaktype_uuids",
161+
"description",
162+
]
163+
_update_nested_zgw_config(source_config, update_fields)
164+
165+
25166
@dataclasses.dataclass(frozen=True)
26167
class CatalogusConfigExport:
27168
"""Gather and export CatalogusConfig(s) and all associated relations."""
@@ -113,9 +254,10 @@ class CatalogusConfigImport:
113254
total_rows_processed: int = 0
114255
catalogus_configs_imported: int = 0
115256
zaaktype_configs_imported: int = 0
116-
zaak_inormatie_object_type_configs_imported: int = 0
257+
zaak_informatie_object_type_configs_imported: int = 0
117258
zaak_status_type_configs_imported: int = 0
118259
zaak_resultaat_type_configs_imported: int = 0
260+
import_errors: list | None = None
119261

120262
@staticmethod
121263
def _get_url_root(url: str) -> str:
@@ -149,90 +291,65 @@ def _lines_iter_from_jsonl_stream_or_string(
149291
# Reset the stream in case it gets re-used
150292
lines.seek(0)
151293

152-
@classmethod
153-
def _rewrite_jsonl_url_references(
154-
cls, stream_or_string: IO | str
155-
) -> Generator[str, Any, None]:
156-
# The assumption is that the exporting and importing instance both have
157-
# a `Service` with the same slug as the `Service` referenced in the
158-
# `configued_from` attribute of the imported CatalogusConfig. The
159-
# assumption is further that all URLs in the imported objects are
160-
# prefixed by an URL that matches the API root in the service. Because
161-
# of this, the import file will contain URLs with a base URL pointing to
162-
# the `api_root`` of the `configured_from` Service on the _source_
163-
# instance, and has to be re-written to match the `api_root` of the
164-
# `configured_from` Service on the _target_ instance. Put differently,
165-
# we assume that we are migrating ZGW objects that _do not differ_ as
166-
# far as the ZGW objects themselves are concerned (apart from the URL,
167-
# they essentially point to the same ZGW backend), but that they _do_
168-
# differ in terms of additional model fields that do not have their
169-
# source of truth in the ZGW backends.
170-
#
171-
# This expectation is also encoded in our API clients: you can only
172-
# fetch ZGW objects using the ApePie clients if the root of those
173-
# objects matches the configured API root.
174-
175-
base_url_mapping = {}
176-
for deserialized_object in serializers.deserialize(
177-
"jsonl",
178-
filter(
179-
lambda row: ('"model": "openzaak.catalogusconfig"' in row),
180-
cls._lines_iter_from_jsonl_stream_or_string(stream_or_string),
181-
),
182-
use_natural_foreign_keys=True,
183-
use_natural_primary_keys=True,
184-
):
185-
object_type: str = deserialized_object.object.__class__.__name__
186-
187-
if object_type == "CatalogusConfig":
188-
target_base_url = cls._get_url_root(
189-
deserialized_object.object.service.api_root
190-
)
191-
source_base_url = cls._get_url_root(deserialized_object.object.url)
192-
base_url_mapping[source_base_url] = target_base_url
193-
else:
194-
# https://www.xkcd.com/2200/
195-
logger.error(
196-
"Tried to filter for catalogus config objects, but also got: %s",
197-
object_type,
198-
)
199-
200-
for line in cls._lines_iter_from_jsonl_stream_or_string(stream_or_string):
201-
source_url_found = False
202-
for source, target in base_url_mapping.items():
203-
line = line.replace(source, target)
204-
source_url_found = True
205-
206-
if not source_url_found:
207-
raise ValueError("Unable to rewrite ZGW urls")
208-
209-
yield line
210-
211294
@classmethod
212295
@transaction.atomic()
213296
def from_jsonl_stream_or_string(cls, stream_or_string: IO | str) -> Self:
214297
model_to_counter_mapping = {
215298
"CatalogusConfig": "catalogus_configs_imported",
216299
"ZaakTypeConfig": "zaaktype_configs_imported",
217-
"ZaakTypeInformatieObjectTypeConfig": "zaak_inormatie_object_type_configs_imported",
300+
"ZaakTypeInformatieObjectTypeConfig": "zaak_informatie_object_type_configs_imported",
218301
"ZaakTypeStatusTypeConfig": "zaak_status_type_configs_imported",
219302
"ZaakTypeResultaatTypeConfig": "zaak_resultaat_type_configs_imported",
220303
}
221-
222304
object_type_counts = defaultdict(int)
223305

224-
for deserialized_object in serializers.deserialize(
225-
"jsonl",
226-
cls._rewrite_jsonl_url_references(stream_or_string),
227-
use_natural_foreign_keys=True,
228-
use_natural_primary_keys=True,
229-
):
230-
deserialized_object.save()
231-
object_type = deserialized_object.object.__class__.__name__
232-
object_type_counts[object_type] += 1
306+
import_errors = []
307+
for line in cls._lines_iter_from_jsonl_stream_or_string(stream_or_string):
308+
try:
309+
(deserialized_object,) = serializers.deserialize(
310+
"jsonl",
311+
line,
312+
use_natural_primary_keys=True,
313+
use_natural_foreign_keys=True,
314+
)
315+
except DeserializationError as exc:
316+
exc_source = type(exc.__context__)
317+
if (
318+
exc_source is CatalogusConfig.DoesNotExist
319+
or ZaakTypeConfig.DoesNotExist
320+
):
321+
logger.error(exc)
322+
import_errors.append(exc)
323+
else:
324+
source_config = deserialized_object.object
325+
try:
326+
match source_config:
327+
case CatalogusConfig():
328+
check_catalogus_config_exists(source_config=source_config)
329+
case ZaakTypeConfig():
330+
update_zaaktype_config(source_config=source_config)
331+
case ZaakTypeInformatieObjectTypeConfig():
332+
update_zaaktype_informatie_objecttype_config(
333+
source_config=source_config
334+
)
335+
case ZaakTypeStatusTypeConfig():
336+
update_zaaktype_statustype_config(
337+
source_config=source_config
338+
)
339+
case ZaakTypeResultaatTypeConfig():
340+
update_zaaktype_resultaattype_config(
341+
source_config=source_config
342+
)
343+
except ZGWImportError as exc:
344+
logger.error(exc)
345+
import_errors.append(exc)
346+
else:
347+
object_type = deserialized_object.object.__class__.__name__
348+
object_type_counts[object_type] += 1
233349

234350
creation_kwargs = {
235351
"total_rows_processed": sum(object_type_counts.values()),
352+
"import_errors": import_errors,
236353
}
237354

238355
for model_name, counter_field in model_to_counter_mapping.items():

0 commit comments

Comments
 (0)