From 9e5b706ad80801ceaef0410714bbd0a12c832b28 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 5 Oct 2023 11:24:57 -0400 Subject: [PATCH 1/7] Skip monthly counters deletion with option --skip-monthly --- .../commands/populate_submission_counters.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/onadata/apps/logger/management/commands/populate_submission_counters.py b/onadata/apps/logger/management/commands/populate_submission_counters.py index 3be1cf68f..c0a420296 100644 --- a/onadata/apps/logger/management/commands/populate_submission_counters.py +++ b/onadata/apps/logger/management/commands/populate_submission_counters.py @@ -51,7 +51,7 @@ def add_arguments(self, parser): ) parser.add_argument( - '--skip_monthly', + '--skip-monthly', action='store_true', default=False, help='Skip updating monthly counters. Default is False', @@ -185,14 +185,15 @@ def clean_old_data(self, user: 'auth.User'): # Because we don't have a real date field on `MonthlyXFormSubmissionCounter` # but we need to cast `year` and `month` as a date field to # compare it with `self._date_threshold` - MonthlyXFormSubmissionCounter.objects.annotate( - date=Cast( - Concat( - F('year'), Value('-'), F('month'), Value('-'), 1 - ), - DateField(), - ) - ).filter(user_id=user.pk, date__gte=self._date_threshold).delete() + if not self._skip_monthly: + MonthlyXFormSubmissionCounter.objects.annotate( + date=Cast( + Concat( + F('year'), Value('-'), F('month'), Value('-'), 1 + ), + DateField(), + ) + ).filter(user_id=user.pk, date__gte=self._date_threshold).delete() def suspend_submissions_for_user(self, user: 'auth.User'): # Retrieve or create user's profile. From 154466c00a0760691924296a721e39ab511858dd Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 5 Oct 2023 11:25:10 -0400 Subject: [PATCH 2/7] Add migration to backfill missing data --- .../0030_backfill_lost_monthly_counter.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 onadata/apps/logger/migrations/0030_backfill_lost_monthly_counter.py diff --git a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counter.py b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counter.py new file mode 100644 index 000000000..79879bb26 --- /dev/null +++ b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counter.py @@ -0,0 +1,65 @@ +from django.db import migrations +from django.db.models import Sum +from django.db.models import Value, F, DateField +from django.db.models.functions import Cast, Concat +from django.db.models.functions import ExtractYear, ExtractMonth +from django.utils.timezone import now + + +def populate_missing_monthly_counters(apps, schema_editor): + + DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter') # noqa + MonthlyXFormSubmissionCounter = apps.get_model('logger', 'MonthlyXFormSubmissionCounter') # noqa + + first_daily_counter = DailyXFormSubmissionCounter.objects.order_by( + 'date' + ).first() + MonthlyXFormSubmissionCounter.objects.annotate( + date=Cast( + Concat( + F('year'), Value('-'), F('month'), Value('-'), 1 + ), + DateField(), + ) + ).filter(date__gte=first_daily_counter.date.replace(day=1)).delete() + + records = ( + DailyXFormSubmissionCounter.objects.filter( + date__range=[first_daily_counter.date.replace(day=1), now().date()] + ) + .annotate(year=ExtractYear('date'), month=ExtractMonth('date')) + .values('month', 'year') + .annotate(total=Sum('counter')) + .values('user_id', 'xform_id', 'month', 'year', 'total') + ).order_by('year', 'month', 'user_id') + + # Do not use `ignore_conflicts=True` to ensure all counters are successfully + # create. + # TODO use `update_conflicts` with Django 4.2 + MonthlyXFormSubmissionCounter.objects.bulk_create( + [ + MonthlyXFormSubmissionCounter( + year=r['year'], + month=r['month'], + user_id=r['user_id'], + xform_id=r['xform_id'], + counter=r['total'], + ) + for r in records + ], + batch_size=5000 + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('logger', '0029_populate_daily_xform_counters_for_year'), + ] + + operations = [ + migrations.RunPython( + populate_missing_monthly_counters, + migrations.RunPython.noop, + ), + ] From 6a3794425c0bec883c84d667bbf061f4531e011c Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 5 Oct 2023 12:13:38 -0400 Subject: [PATCH 3/7] Keep indent --- .../commands/populate_submission_counters.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/onadata/apps/logger/management/commands/populate_submission_counters.py b/onadata/apps/logger/management/commands/populate_submission_counters.py index c0a420296..708a1f3ab 100644 --- a/onadata/apps/logger/management/commands/populate_submission_counters.py +++ b/onadata/apps/logger/management/commands/populate_submission_counters.py @@ -182,18 +182,20 @@ def clean_old_data(self, user: 'auth.User'): xform__user_id=user.pk, date__gte=self._date_threshold ).delete() + if self._skip_monthly: + return + # Because we don't have a real date field on `MonthlyXFormSubmissionCounter` # but we need to cast `year` and `month` as a date field to # compare it with `self._date_threshold` - if not self._skip_monthly: - MonthlyXFormSubmissionCounter.objects.annotate( - date=Cast( - Concat( - F('year'), Value('-'), F('month'), Value('-'), 1 - ), - DateField(), - ) - ).filter(user_id=user.pk, date__gte=self._date_threshold).delete() + MonthlyXFormSubmissionCounter.objects.annotate( + date=Cast( + Concat( + F('year'), Value('-'), F('month'), Value('-'), 1 + ), + DateField(), + ) + ).filter(user_id=user.pk, date__gte=self._date_threshold).delete() def suspend_submissions_for_user(self, user: 'auth.User'): # Retrieve or create user's profile. From cf3e44d3065e307afe7641f5136c1fee2930dee9 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 5 Oct 2023 12:16:26 -0400 Subject: [PATCH 4/7] Fix instructions --- .../migrations/0029_populate_daily_xform_counters_for_year.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onadata/apps/logger/migrations/0029_populate_daily_xform_counters_for_year.py b/onadata/apps/logger/migrations/0029_populate_daily_xform_counters_for_year.py index 2f820a6d5..920bc3270 100644 --- a/onadata/apps/logger/migrations/0029_populate_daily_xform_counters_for_year.py +++ b/onadata/apps/logger/migrations/0029_populate_daily_xform_counters_for_year.py @@ -10,7 +10,7 @@ def populate_daily_counts_for_year(apps, schema_editor): !!! ATTENTION !!! If you have existing projects, you need to run this management command: - > python manage.py populate_submission_counters -f --skip_monthly + > python manage.py populate_submission_counters -f --skip-monthly Until you do, total usage counts from the KPI endpoints /api/v2/service_usage and /api/v2/asset_usage will be incorrect @@ -22,7 +22,7 @@ def populate_daily_counts_for_year(apps, schema_editor): This might take a while. If it is too slow, you may want to re-run the migration with SKIP_HEAVY_MIGRATIONS=True and run the following management command: - > python manage.py populate_submission_counters -f --skip_monthly + > python manage.py populate_submission_counters -f --skip-monthly """ ) call_command('populate_submission_counters', force=True, skip_monthly=True) From c4d62c99f992677f0905e79356572f9dae6a4ab6 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 5 Oct 2023 12:17:21 -0400 Subject: [PATCH 5/7] Typo --- ..._monthly_counter.py => 0030_backfill_lost_monthly_counters.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename onadata/apps/logger/migrations/{0030_backfill_lost_monthly_counter.py => 0030_backfill_lost_monthly_counters.py} (100%) diff --git a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counter.py b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py similarity index 100% rename from onadata/apps/logger/migrations/0030_backfill_lost_monthly_counter.py rename to onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py From 8be441d2439777bbda3ebac6565dcb0c785c70de Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 5 Oct 2023 12:19:41 -0400 Subject: [PATCH 6/7] Do nothing on fresh installs --- .../logger/migrations/0030_backfill_lost_monthly_counters.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py index 79879bb26..f9f0a14b8 100644 --- a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py +++ b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py @@ -14,6 +14,10 @@ def populate_missing_monthly_counters(apps, schema_editor): first_daily_counter = DailyXFormSubmissionCounter.objects.order_by( 'date' ).first() + + if not first_daily_counter: + return + MonthlyXFormSubmissionCounter.objects.annotate( date=Cast( Concat( From f90d0e021bedabada7e007471cb00d133fcf17b4 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Fri, 6 Oct 2023 07:37:44 -0400 Subject: [PATCH 7/7] Use previous migration date for lower bound range --- .../0030_backfill_lost_monthly_counters.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py index f9f0a14b8..503e8ab37 100644 --- a/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py +++ b/onadata/apps/logger/migrations/0030_backfill_lost_monthly_counters.py @@ -1,4 +1,5 @@ from django.db import migrations +from django.db.migrations.recorder import MigrationRecorder from django.db.models import Sum from django.db.models import Value, F, DateField from django.db.models.functions import Cast, Concat @@ -11,13 +12,14 @@ def populate_missing_monthly_counters(apps, schema_editor): DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter') # noqa MonthlyXFormSubmissionCounter = apps.get_model('logger', 'MonthlyXFormSubmissionCounter') # noqa - first_daily_counter = DailyXFormSubmissionCounter.objects.order_by( - 'date' - ).first() - - if not first_daily_counter: + if not DailyXFormSubmissionCounter.objects.all().exists(): return + previous_migration = MigrationRecorder.Migration.objects.filter( + app='logger', name='0029_populate_daily_xform_counters_for_year' + ).first() + + # Delete monthly counters in the range if any (to avoid conflicts in bulk_create below) MonthlyXFormSubmissionCounter.objects.annotate( date=Cast( Concat( @@ -25,11 +27,14 @@ def populate_missing_monthly_counters(apps, schema_editor): ), DateField(), ) - ).filter(date__gte=first_daily_counter.date.replace(day=1)).delete() + ).filter(date__gte=previous_migration.applied.date().replace(day=1)).delete() records = ( DailyXFormSubmissionCounter.objects.filter( - date__range=[first_daily_counter.date.replace(day=1), now().date()] + date__range=[ + previous_migration.applied.date().replace(day=1), + now().date() + ] ) .annotate(year=ExtractYear('date'), month=ExtractMonth('date')) .values('month', 'year') @@ -39,7 +44,7 @@ def populate_missing_monthly_counters(apps, schema_editor): # Do not use `ignore_conflicts=True` to ensure all counters are successfully # create. - # TODO use `update_conflicts` with Django 4.2 + # TODO use `update_conflicts` with Django 4.2 and avoid `.delete()` above MonthlyXFormSubmissionCounter.objects.bulk_create( [ MonthlyXFormSubmissionCounter(