Merge pull request #898 from kobotoolbox/fix-monthly-counters-deletion

Restore missing monthly submission counters data

onadata/apps/logger/management/commands/populate_submission_counters.py

@@ -51,7 +51,7 @@ def add_arguments(self, parser):
         )
 
         parser.add_argument(
-            '--skip_monthly',
+            '--skip-monthly',
             action='store_true',
             default=False,
             help='Skip updating monthly counters. Default is False',
@@ -182,6 +182,9 @@ def clean_old_data(self, user: 'auth.User'):
             xform__user_id=user.pk, date__gte=self._date_threshold
         ).delete()
 
+        if self._skip_monthly:
+            return
+
         # Because we don't have a real date field on `MonthlyXFormSubmissionCounter`
         # but we need to cast `year` and `month` as a date field to
         # compare it with `self._date_threshold`

onadata/apps/logger/migrations/0029_populate_daily_xform_counters_for_year.py

@@ -10,7 +10,7 @@ def populate_daily_counts_for_year(apps, schema_editor):
             !!! ATTENTION !!!
             If you have existing projects, you need to run this management command:
 
-               > python manage.py populate_submission_counters -f --skip_monthly
+               > python manage.py populate_submission_counters -f --skip-monthly
 
             Until you do, total usage counts from the KPI endpoints
             /api/v2/service_usage and /api/v2/asset_usage will be incorrect
@@ -22,7 +22,7 @@ def populate_daily_counts_for_year(apps, schema_editor):
             This might take a while. If it is too slow, you may want to re-run the
             migration with SKIP_HEAVY_MIGRATIONS=True and run the following management command:
 
-                > python manage.py populate_submission_counters -f --skip_monthly
+                > python manage.py populate_submission_counters -f --skip-monthly
             """
         )
         call_command('populate_submission_counters', force=True, skip_monthly=True)

onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py

@@ -0,0 +1,74 @@
+from django.db import migrations
+from django.db.migrations.recorder import MigrationRecorder
+from django.db.models import Sum
+from django.db.models import Value, F, DateField
+from django.db.models.functions import Cast, Concat
+from django.db.models.functions import ExtractYear, ExtractMonth
+from django.utils.timezone import now
+
+
+def populate_missing_monthly_counters(apps, schema_editor):
+
+    DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter')  # noqa
+    MonthlyXFormSubmissionCounter = apps.get_model('logger', 'MonthlyXFormSubmissionCounter')  # noqa
+
+    if not DailyXFormSubmissionCounter.objects.all().exists():
+        return
+
+    previous_migration = MigrationRecorder.Migration.objects.filter(
+        app='logger', name='0029_populate_daily_xform_counters_for_year'
+    ).first()
+
+    # Delete monthly counters in the range if any (to avoid conflicts in bulk_create below)
+    MonthlyXFormSubmissionCounter.objects.annotate(
+        date=Cast(
+            Concat(
+                F('year'), Value('-'), F('month'), Value('-'), 1
+            ),
+            DateField(),
+        )
+    ).filter(date__gte=previous_migration.applied.date().replace(day=1)).delete()
+
+    records = (
+        DailyXFormSubmissionCounter.objects.filter(
+            date__range=[
+                previous_migration.applied.date().replace(day=1),
+                now().date()
+            ]
+        )
+        .annotate(year=ExtractYear('date'), month=ExtractMonth('date'))
+        .values('month', 'year')
+        .annotate(total=Sum('counter'))
+        .values('user_id', 'xform_id', 'month', 'year', 'total')
+    ).order_by('year', 'month', 'user_id')
+
+    # Do not use `ignore_conflicts=True` to ensure all counters are successfully
+    # create.
+    # TODO use `update_conflicts` with Django 4.2 and avoid `.delete()` above
+    MonthlyXFormSubmissionCounter.objects.bulk_create(
+        [
+            MonthlyXFormSubmissionCounter(
+                year=r['year'],
+                month=r['month'],
+                user_id=r['user_id'],
+                xform_id=r['xform_id'],
+                counter=r['total'],
+            )
+            for r in records
+        ],
+        batch_size=5000
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('logger', '0029_populate_daily_xform_counters_for_year'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            populate_missing_monthly_counters,
+            migrations.RunPython.noop,
+        ),
+    ]