Improve Intervention Loaders

CHANGES

@@ -1,4 +1,13 @@
-4.6.1
+4.7
+----
+* Improvements on data.intervention
+* Page size increase for speed up
+* Improvement on data.partner by preetching related T2FTravelactivity 
+* Improvement on data.agreement (partners) 
+* Improvement on data.interventionbudget (partners)
+* Improvement on data.engagement 
+
+4.6.1 
 ----
 * Fix loaders delta logic
 * Increased PDIndicator.section_name field length 

Dockerfile-dev

@@ -20,7 +20,7 @@ ENV PYTHONUNBUFFERED 1
 COPY Pipfile .
 COPY Pipfile.lock .
 
-RUN pipenv install --ignore-pipfile  --system
+RUN pipenv install --ignore-pipfile --dev --system
 
 EXPOSE 8000
 

docker-compose-dev.yml

@@ -7,13 +7,13 @@ x-devdatamart-defaults: &devdatamart_defaults
     - db
     - redis
   volumes:
-    - "$PWD:/code"
+    - "./:/code"
   environment:
       DEBUG: 0
       DATABASE_URL: postgis://postgres:@db:5432/etools_datamart
       #NOTE: Please update ETOOLS DB server IP/name and port specific to your enviroment.
-      DATABASE_URL_ETOOLS: postgis://postgres:@192.168.24.151:5442/etools
-      DATABASE_URL_PRP: postgis://postgres:@db-prp:5432/prp
+      DATABASE_URL_ETOOLS: postgis://postgres:@192.168.0.106:25432/etools
+      DATABASE_URL_PRP: postgis://postgres:@192.168.0.106:35432/prp
       AUTOCREATE_USERS: "admin,123"
       CACHE_URL: "redis://redis:6379/1"
       CACHE_URL_LOCK: "redis://redis:6379/1"
@@ -55,22 +55,22 @@ services:
       - traefik.http.services.datamart.loadBalancer.server.port=8080
       - traefik.enable=true
     volumes:
-      - $PWD:/code
+      - ./:/code
   beat:
     container_name: datamart_beat
     << : *devdatamart_defaults
     working_dir: /code/src
     command: celery -A etools_datamart beat --loglevel=debug
     volumes:
-      - $PWD:/code
+      - ./:/code
 
   worker:
     container_name: datamart_worker
     << : *devdatamart_defaults
     working_dir: /code/src
     command: celery -A etools_datamart worker --loglevel=debug
     volumes:
-      - $PWD:/code
+      - ./:/code
 
   db:
     image: postgis/postgis:12-3.2
@@ -84,31 +84,7 @@ services:
       POSTGRES_DB: etools_datamart
     volumes:
       - "$PWD/build/db:/var/lib/postgresql/data"
-
-  # Rely on etools and prp database instances running locally
-  #db-etools:
-  #  image: postgis/postgis:12-3.2
-  #  container_name: datamart_etools
-  #  shm_size: '1gb'
-  #  environment:
-  #    POSTGRES_USER: postgres
-  #    POSTGRES_PASSWORD:
-  #    POSTGRES_DB: etools
-  #  volumes:
-  #    - "$PWD/build/etools:/var/lib/postgresql/data"
-
-  db-prp:
-    image: postgis/postgis:12-3.2
-    shm_size: '1gb'
-    container_name: datamart_prp
-    environment:
-      POSTGRES_HOST_AUTH_METHOD: trust
-      #POSTGRES_USER: postgres
-      #POSTGRES_PASSWORD:
-      POSTGRES_DB: prp
-    volumes:
-      - "$PWD/build/prp:/var/lib/postgresql/data"
-
+
   redis:
     image: redis:alpine
     container_name: datamart_redis

src/etools_datamart/apps/mart/data/models/audit_engagement.py

@@ -1,7 +1,7 @@
 from django.contrib.postgres.fields import ArrayField
 from django.core.paginator import Paginator
 from django.db import models
-from django.db.models import Count, JSONField
+from django.db.models import Count, JSONField, Prefetch
 from django.utils.translation import gettext as _
 
 from model_utils import Choices
@@ -15,6 +15,7 @@
     AuditAudit,
     AuditEngagement,
     AuditEngagementActivePd,
+    AuditEngagementSections,
     AuditMicroassessment,
     AuditRisk,
     AuditSpecialaudit,
@@ -214,7 +215,7 @@ def get_authorized_officers(self, record: AuditEngagement, values: dict, **kwarg
 
     def get_active_pd(self, record: AuditEngagement, values: dict, **kwargs):
         ret = []
-        for o in AuditEngagementActivePd.objects.select_related("intervention").filter(engagement=record):
+        for o in record.prefetched_AuditEngagementActivePds:
             ret.append(
                 {
                     "title": o.intervention.title,
@@ -253,15 +254,17 @@ def get_action_points(self, record: AuditEngagement, values: dict, **kwargs):
         from etools_datamart.api.endpoints.datamart.actionpoint import ActionPointSimpleSerializer
 
         ret = []
-        for r in ActionPointsActionpoint.objects.filter(engagement=record).all():
+        for r in record.prefetched_ActionPoints:
             ret.append(ActionPointSimpleSerializer(r).data)
         return ret
 
     def get_key_internal_control_weaknesses(self, record: AuditEngagement, values: dict, **kwargs):
-        qs = AuditRisk.objects.filter(
-            blueprint__category__header="Key Internal Controls Weaknesses", engagement_id=record.id
-        )
-        return [risk.blueprint.header for risk in qs]
+        risks = []
+
+        for risk in record.prefetched_AuditRisks:
+            risks.append(risk.blueprint.header)
+
+        return risks
 
     def process_country(self):
         # TODO: Analyze more before batch processing
@@ -271,12 +274,31 @@ def process_country(self):
                 "engagement_ptr__partner",
                 "engagement_ptr__agreement",
                 "engagement_ptr__agreement__auditor_firm",
+                "engagement_ptr__agreement__auditor_firm__organization",
                 "engagement_ptr__po_item",
             ).prefetch_related(
                 "engagement_ptr__authorized_officers",
                 "engagement_ptr__staff_members",
-                "engagement_ptr__AuditEngagementSections_engagement",
-                "engagement_ptr__AuditEngagementOffices_engagement",
+                Prefetch(
+                    "engagement_ptr__AuditEngagementSections_engagement",
+                    AuditEngagementSections.objects.all(),
+                    to_attr="prefetched_AuditEngagementSections",
+                ),
+                Prefetch(
+                    "engagement_ptr__ActionPointsActionpoint_engagement",
+                    ActionPointsActionpoint.objects.all(),
+                    to_attr="prefetched_ActionPoints",
+                ),
+                Prefetch(
+                    "engagement_ptr__AuditEngagementActivePd_engagement",
+                    AuditEngagementActivePd.objects.all(),
+                    to_attr="prefetched_AuditEngagementActivePds",
+                ),
+                Prefetch(
+                    "engagement_ptr__AuditRisk_engagement",
+                    AuditRisk.objects.filter(blueprint__category__header="Key Internal Controls Weaknesses"),
+                    to_attr="prefetched_AuditRisks",
+                ),
             ):
                 record.id = record.engagement_ptr_id
                 record.sub_type = m