⚡ [#469] Reduce DB and HTTPs connections

- Avoid retrieving the selectielijst service for each zaak
- Take advantage of connection pooling by sharing client between zaken

Author: SilviaAmAm

backend/src/openarchiefbeheer/zaken/tasks.py

@@ -10,13 +10,14 @@
 from zgw_consumers.models import Service
 
 from openarchiefbeheer.celery import app
+from openarchiefbeheer.config.models import APIConfig
 from openarchiefbeheer.destruction.utils import resync_items_and_zaken
 from openarchiefbeheer.logging import logevent
 
 from .api.serializers import ZaakSerializer
 from .decorators import log_errors
 from .models import Zaak
-from .utils import pagination_helper, process_expanded_data
+from .utils import NoClient, pagination_helper, process_expanded_data
 
 logger = logging.getLogger(__name__)
 
@@ -26,6 +27,10 @@ def retrieve_and_cache_zaken_from_openzaak() -> None:
     zrc_service = Service.objects.get(api_type=APITypes.zrc)
     zrc_client = build_client(zrc_service)
 
+    config = APIConfig.get_solo()
+    service = config.selectielijst_api_service
+    selectielijst_api_client = build_client(service) if service else None
+
     today = datetime.date.today()
     query_params = {
         "expand": "resultaat,resultaat.resultaattype,zaaktype,rollen",
@@ -53,24 +58,27 @@ def retrieve_and_cache_zaken_from_openzaak() -> None:
             timeout=settings.REQUESTS_DEFAULT_TIMEOUT,
         )
 
-    for index, data in enumerate(data_iterator):
-        logger.info("Retrieved page %s.", index + 1)
+    with selectielijst_api_client or NoClient():
+        for index, data in enumerate(data_iterator):
+            logger.info("Retrieved page %s.", index + 1)
 
-        zaken = process_expanded_data(data["results"])
+            zaken = data["results"]
+            if selectielijst_api_client:
+                zaken = process_expanded_data(zaken, selectielijst_api_client)
 
-        # Since before we indexed also zaken without einddatum, prevent
-        # crash when retrieving zaken.
-        if zaken_in_db:
-            new_zaken = {zaak["url"]: zaak for zaak in zaken}
-            duplicates = Zaak.objects.filter(url__in=new_zaken.keys())
-            if duplicates.exists():
-                for duplicate in duplicates:
-                    del new_zaken[duplicate.url]
-            zaken = [zaak for zaak in new_zaken.values()]
+            # Since before we indexed also zaken without einddatum, prevent
+            # crash when retrieving zaken.
+            if zaken_in_db:
+                new_zaken = {zaak["url"]: zaak for zaak in zaken}
+                duplicates = Zaak.objects.filter(url__in=new_zaken.keys())
+                if duplicates.exists():
+                    for duplicate in duplicates:
+                        del new_zaken[duplicate.url]
+                zaken = [zaak for zaak in new_zaken.values()]
 
-        serializer = ZaakSerializer(data=zaken, many=True)
-        serializer.is_valid(raise_exception=True)
-        serializer.save()
+            serializer = ZaakSerializer(data=zaken, many=True)
+            serializer.is_valid(raise_exception=True)
+            serializer.save()
 
 
 @app.task
@@ -87,8 +95,12 @@ def resync_zaken():
         "einddatum__lt": today.isoformat(),
     }
 
+    config = APIConfig.get_solo()
+    service = config.selectielijst_api_service
+    selectielijst_api_client = build_client(service) if service else None
+
     logevent.resync_started()
-    with transaction.atomic(), zrc_client:
+    with transaction.atomic(), zrc_client, selectielijst_api_client or NoClient():
         Zaak.objects.all().delete()
 
         response = zrc_client.get(
@@ -109,7 +121,9 @@ def resync_zaken():
         for index, data in enumerate(data_iterator):
             logger.info("Retrieved page %s.", index + 1)
 
-            zaken = process_expanded_data(data["results"])
+            zaken = data["results"]
+            if selectielijst_api_client:
+                zaken = process_expanded_data(zaken, selectielijst_api_client)
 
             serializer = ZaakSerializer(data=zaken, many=True)
             serializer.is_valid(raise_exception=True)

backend/src/openarchiefbeheer/zaken/utils.py

@@ -61,14 +61,26 @@ def get_resource(url: str) -> dict | None:
     return data
 
 
-def process_expanded_data(zaken: list[dict]) -> list[dict]:
+def get_resource_with_prebuilt_client(url: str, client: APIClient) -> dict | None:
+    response = client.get(url)
+    response.raise_for_status()
+    data = response.json()
+
+    return data
+
+
+def process_expanded_data(
+    zaken: list[dict], selectielijst_api_client: APIClient
+) -> list[dict]:
     def expand_procestype(zaak: dict) -> dict:
         if "_expand" not in zaak:
             return zaak
 
         extra_data = zaak["_expand"]
         if procestype_url := extra_data["zaaktype"].get("selectielijst_procestype"):
-            expanded_procestype = get_resource(procestype_url)
+            expanded_procestype = get_resource_with_prebuilt_client(
+                procestype_url, selectielijst_api_client
+            )
             if expanded_procestype is not None:
                 extra_data["zaaktype"]["selectielijst_procestype"] = expanded_procestype
 
@@ -391,3 +403,11 @@ def retrieve_zaaktypen(query_params: HashableDict | None = None) -> list[dict]:
         results += page["results"]
 
     return results
+
+
+class NoClient:
+    def __enter__(self):
+        return None
+
+    def __exit__(self, *args):
+        return None