Merge pull request #869 from mapswipe/fix/footprint-calculation-fix

Fix footprint time threshold calculation logic

Author: thenav56

django/apps/aggregated/management/commands/update_aggregated_data.py

@@ -19,7 +19,62 @@
 # |1|00:00:00.208768|00:00:01.398161|00:00:28.951521|
 # |2|00:00:01.330297|00:00:06.076814|00:00:03.481192|
 # |3|00:00:02.092967|00:00:11.271081|00:00:06.045881|
-UPDATE_PROJECT_GROUP_DATA = f"""
+
+UPDATE_PROJECT_GROUP_DATA_USING_PROJECT_ID = f"""
+    WITH to_calculate_groups AS (
+        SELECT
+            project_id,
+            group_id
+        FROM groups
+            WHERE
+                (project_id, group_id) in (
+                    SELECT
+                        MS.project_id,
+                        MS.group_id
+                    FROM mapping_sessions MS
+                    WHERE
+                        project_id = %(project_id)s
+                    GROUP BY MS.project_id, MS.group_id
+                ) AND
+                (
+                    total_area is NULL OR time_spent_max_allowed is NULL
+                )
+    ),
+    groups_data AS (
+        SELECT
+          T.project_id,
+          T.group_id,
+          SUM( -- sqkm
+            ST_Area(T.geom::geography(GEOMETRY,4326)) / 1000000
+          ) as total_task_group_area,
+          (
+            CASE
+              -- Using 95_percent value of existing data for each project_type
+              WHEN P.project_type = {Project.Type.BUILD_AREA.value} THEN 1.4
+              WHEN P.project_type = {Project.Type.COMPLETENESS.value} THEN 1.4
+              WHEN P.project_type = {Project.Type.CHANGE_DETECTION.value} THEN 11.2
+              -- FOOTPRINT: Not calculated right now
+              WHEN P.project_type = {Project.Type.FOOTPRINT.value} THEN 6.1
+              ELSE 1
+            END
+          ) * COUNT(*) as time_spent_max_allowed
+        FROM tasks T
+          INNER JOIN to_calculate_groups G USING (project_id, group_id)
+          INNER JOIN projects P USING (project_id)
+        GROUP BY project_id, P.project_type, group_id
+    )
+    UPDATE groups G
+    SET
+        total_area = GD.total_task_group_area,
+        time_spent_max_allowed = GD.time_spent_max_allowed
+    FROM groups_data GD
+    WHERE
+        G.project_id = GD.project_id AND
+        G.group_id = GD.group_id;
+"""
+
+
+UPDATE_PROJECT_GROUP_DATA_USING_TIME_RANGE = f"""
     WITH to_calculate_groups AS (
         SELECT
             project_id,
@@ -73,11 +128,17 @@
         G.group_id = GD.group_id;
 """
 
-TASK_GROUP_METADATA_QUERY = """
+TASK_GROUP_METADATA_QUERY = f"""
         SELECT
             G.project_id,
             G.group_id,
-            G.total_area as total_task_group_area,
+            (
+                CASE
+                    -- Hide area for Footprint
+                    WHEN P.project_type = {Project.Type.FOOTPRINT.value} THEN 0
+                    ELSE G.total_area
+                END
+            ) as total_task_group_area,
             G.time_spent_max_allowed
         FROM groups G
             INNER JOIN used_task_groups UG USING (project_id, group_id)
@@ -106,9 +167,7 @@
             FROM mapping_sessions MS
                 INNER JOIN projects P USING (project_id)
             WHERE
-                -- Skip for footprint type missions
-                P.project_type != {Project.Type.FOOTPRINT.value}
-                AND MS.start_time >= %(from_date)s
+                MS.start_time >= %(from_date)s
                 AND MS.start_time < %(until_date)s
             GROUP BY project_id, project_type, group_id -- To get unique
         ),
@@ -185,9 +244,7 @@
                 INNER JOIN mapping_sessions MS USING (mapping_session_id)
                 INNER JOIN projects P USING (project_id)
             WHERE
-                -- Skip for footprint type missions
-                P.project_type != {Project.Type.FOOTPRINT.value}
-                AND MS.start_time >= %(from_date)s
+                MS.start_time >= %(from_date)s
                 AND MS.start_time < %(until_date)s
             GROUP BY project_id, project_type, group_id -- To get unique
         ),
@@ -288,7 +345,7 @@ def _track(self, tracker_type, label, sql):
             )
             with transaction.atomic():
                 with connection.cursor() as cursor:
-                    cursor.execute(UPDATE_PROJECT_GROUP_DATA, params)
+                    cursor.execute(UPDATE_PROJECT_GROUP_DATA_USING_TIME_RANGE, params)
                 self.stdout.write(
                     self.style.SUCCESS(
                         f"Successfull. Runtime: {time.time() - start_time} seconds"

django/apps/aggregated/management/commands/update_project_groups_data.py

@@ -0,0 +1,34 @@
+import time
+
+from apps.existing_database.models import Project
+from django.core.management.base import BaseCommand
+from django.db import connection, transaction
+
+from .update_aggregated_data import UPDATE_PROJECT_GROUP_DATA_USING_PROJECT_ID
+
+
+class Command(BaseCommand):
+    def handle(self, **_):
+        project_qs = Project.objects.all()
+        total_projects = project_qs.count()
+        self.stdout.write(f"Total projects: {total_projects}")
+        for index, project_id in enumerate(
+            project_qs.values_list("project_id", flat=True),
+            start=1,
+        ):
+            self.stdout.write(
+                "Running calculation for project ID "
+                f"({index}/{total_projects}): {project_id}"
+            )
+            with transaction.atomic():
+                start_time = time.time()
+                with connection.cursor() as cursor:
+                    cursor.execute(
+                        UPDATE_PROJECT_GROUP_DATA_USING_PROJECT_ID,
+                        dict(project_id=project_id),
+                    )
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"- Successfull. Runtime: {time.time() - start_time} seconds"
+                    )
+                )

docker-compose.yaml

@@ -30,7 +30,7 @@ x-django: &base_django
       DJANGO_DB_NAME: '${POSTGRES_DB}'
       DJANGO_DB_USER: '${POSTGRES_USER}'
       DJANGO_DB_PWD: '${POSTGRES_PASSWORD}'
-      DJANGO_DB_HOST: 'postgres'
+      DJANGO_DB_HOST: '${POSTGRES_HOST:-postgres}'
       DJANGO_DB_PORT: 5432
       DJANGO_STATIC_ROOT: '/assets/static/'
       DJANGO_MEDIA_ROOT: '/assets/media/'
@@ -62,7 +62,7 @@ x-mapswipe-workers: &base_mapswipe_workers
         POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
         POSTGRES_USER: '${POSTGRES_USER}'
         POSTGRES_DB: '${POSTGRES_DB}'
-        POSTGRES_HOST: 'postgres'
+        POSTGRES_HOST: '${POSTGRES_HOST:-postgres}'
         POSTGRES_PORT: 5432
         PGDATA: '/var/lib/postgresql/mapswipe'
         IMAGE_BING_API_KEY: '${IMAGE_BING_API_KEY}'