Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create null xform instances for DailyXFormSubmissionCounter #895

Merged
merged 5 commits into from
Sep 15, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -163,6 +163,7 @@ def build_counters(self, xf: 'logger.XForm') -> tuple[list, dict]:
submission_date = values['date_created__date']
daily_counters.append(DailyXFormSubmissionCounter(
xform_id=xf.pk,
user=xf.user,
date=submission_date,
counter=values['num_of_submissions'],
))
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
from django.conf import settings
from django.db import migrations, models
from django.db.models import Subquery, deletion, OuterRef


def add_user_to_daily_submission_counter(apps, schema_editor):
DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter')
# delete any counters where xform and user are None, since we can't associate them with a user
DailyXFormSubmissionCounter.objects.filter(xform=None, user=None).delete()
# add the user to the counter, based on the xform user
DailyXFormSubmissionCounter.objects.all().exclude(xform=None).update(
user=Subquery(
DailyXFormSubmissionCounter.objects.filter(pk=OuterRef('pk')).values('xform__user')[:1]
),
)


def delete_null_xform_daily_counters(apps, schema_editor):
DailyXFormSubmissionCounter = apps.get_model('logger', 'DailyXFormSubmissionCounter')
# to migrate backwards, we need to delete any null xform instances
DailyXFormSubmissionCounter.objects.filter(xform=None).delete()


class Migration(migrations.Migration):

dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('logger', '0027_on_delete_cascade_monthlyxformsubmissioncounter'),
]

operations = [
migrations.AddField(
model_name='DailyXFormSubmissionCounter',
name='user',
field=models.ForeignKey('auth.User', related_name='daily_users', null=True, on_delete=models.CASCADE),
),
migrations.RunPython(
add_user_to_daily_submission_counter,
migrations.RunPython.noop,
),
migrations.AlterField(
model_name='dailyxformsubmissioncounter',
name='xform',
field=models.ForeignKey(null=True, on_delete=deletion.CASCADE,
related_name='daily_counters', to='logger.xform'),
),
migrations.RunPython(
migrations.RunPython.noop,
delete_null_xform_daily_counters,
),
migrations.AlterUniqueTogether(
name='dailyxformsubmissioncounter',
unique_together=set(),
),
migrations.AddIndex(
model_name='dailyxformsubmissioncounter',
index=models.Index(fields=['date', 'user'], name='logger_dail_date_f738ed_idx'),
),
migrations.AddConstraint(
model_name='dailyxformsubmissioncounter',
constraint=models.UniqueConstraint(fields=('date', 'user', 'xform'), name='daily_unique_with_xform'),
),
migrations.AddConstraint(
model_name='dailyxformsubmissioncounter',
constraint=models.UniqueConstraint(condition=models.Q(('xform', None)), fields=('date', 'user'),
name='daily_unique_without_xform'),
),
]
Original file line number Diff line number Diff line change
@@ -31,7 +31,7 @@ def populate_daily_counts_for_year(apps, schema_editor):
class Migration(migrations.Migration):

dependencies = [
('logger', '0027_on_delete_cascade_monthlyxformsubmissioncounter'),
('logger', '0028_add_user_to_daily_submission_counters'),
('main', '0012_add_validate_password_flag_to_profile'),
]

39 changes: 34 additions & 5 deletions onadata/apps/logger/models/daily_xform_submission_counter.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,47 @@
# coding: utf-8
import datetime

from django.contrib.auth.models import User
from django.db import models

from onadata.apps.logger.models.xform import XForm
from django.db.models import UniqueConstraint, Q, F


class DailyXFormSubmissionCounter(models.Model):
date = models.DateField()
user = models.ForeignKey(User, related_name='daily_counts', null=True, on_delete=models.CASCADE)
xform = models.ForeignKey(
XForm, related_name='daily_counters', on_delete=models.CASCADE
'logger.XForm', related_name='daily_counters', null=True, on_delete=models.CASCADE
)
counter = models.IntegerField(default=0)

class Meta:
unique_together = (('date', 'xform'),)
constraints = [
UniqueConstraint(fields=['date', 'user', 'xform'],
name='daily_unique_with_xform'),
UniqueConstraint(fields=['date', 'user'],
condition=Q(xform=None),
name='daily_unique_without_xform')
]
indexes = [
models.Index(fields=('date', 'user')),
]

@classmethod
def update_catch_all_counter_on_delete(cls, sender, instance, **kwargs):
daily_counters = cls.objects.filter(
xform_id=instance.pk, counter__gte=1
)

for daily_counter in daily_counters:
criteria = dict(
date=daily_counter.date,
user=daily_counter.user,
xform=None,
)
# make sure an instance exists with `xform = NULL`
cls.objects.get_or_create(**criteria)
# add the count for the project being deleted to the null-xform
# instance, atomically!
cls.objects.filter(**criteria).update(
counter=F('counter') + daily_counter.counter
)

1 change: 1 addition & 0 deletions onadata/apps/logger/models/instance.py
Original file line number Diff line number Diff line change
@@ -142,6 +142,7 @@ def update_xform_daily_counter(sender, instance, created, **kwargs):
DailyXFormSubmissionCounter.objects.get_or_create(
date=date_created,
xform=instance.xform,
user=instance.xform.user,
)

# update the count for the current submission
8 changes: 7 additions & 1 deletion onadata/apps/logger/models/xform.py
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@
from taggit.managers import TaggableManager

from onadata.apps.logger.fields import LazyDefaultBooleanField
from onadata.apps.logger.models.daily_xform_submission_counter import DailyXFormSubmissionCounter
from onadata.apps.logger.models.monthly_xform_submission_counter import (
MonthlyXFormSubmissionCounter,
)
@@ -36,7 +37,6 @@
from onadata.libs.models.base_model import BaseModel
from onadata.libs.utils.hash import get_hash


XFORM_TITLE_LENGTH = 255
title_pattern = re.compile(r"<h:title>([^<]+)</h:title>")

@@ -335,3 +335,9 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs):
sender=XForm,
dispatch_uid='update_catch_all_monthly_xform_submission_counter',
)

pre_delete.connect(
DailyXFormSubmissionCounter.update_catch_all_counter_on_delete,
sender=XForm,
dispatch_uid='update_catch_all_daily_xform_submission_counter',
)
31 changes: 28 additions & 3 deletions onadata/apps/logger/tests/models/test_xform_submission_counters.py
Original file line number Diff line number Diff line change
@@ -47,7 +47,7 @@ def test_data_retrieval(self):
self._publish_transportation_form_and_submit_instance()

daily_counter = DailyXFormSubmissionCounter.objects.filter(
xform__user__username='bob'
user__username='bob'
).order_by('date').last()
today = timezone.now().date()
self.assertEqual(daily_counter.date, today)
@@ -65,7 +65,7 @@ def test_delete_daily_counters(self):
"""
self._publish_transportation_form_and_submit_instance()
counter = DailyXFormSubmissionCounter.objects.filter(
xform__user__username='bob'
user__username='bob'
).order_by('date').last()
counter.date = counter.date - timedelta(
days=settings.DAILY_COUNTERS_MAX_DAYS + 1
@@ -79,7 +79,7 @@ def test_delete_daily_counters(self):
daily_counters = DailyXFormSubmissionCounter.objects.count()
self.assertEqual(daily_counters, 0)

def test_deleted_xform_counters_are_merged(self):
def test_deleted_monthly_xform_counters_are_merged(self):
"""
Test that the monthly counter with `xform = NULL` contains the sum of
counters for all xforms deleted within the current month
@@ -104,3 +104,28 @@ def test_deleted_xform_counters_are_merged(self):
assert (
MonthlyXFormSubmissionCounter.objects.get(**criteria).counter == 2
)

def test_deleted_daily_xform_counters_are_merged(self):
"""
Test that the daily counter with `xform = NULL` contains the sum of
counters for all xforms deleted within the current day
"""
today = timezone.now().date()
criteria = dict(
date=today,
user=User.objects.get(username='bob'),
xform=None,
)
assert not DailyXFormSubmissionCounter.objects.filter(
**criteria
).exists()
self._publish_transportation_form_and_submit_instance()
XForm.objects.filter(user__username='bob').first().delete()
assert (
DailyXFormSubmissionCounter.objects.get(**criteria).counter == 1
)
self._publish_transportation_form_and_submit_instance()
XForm.objects.filter(user__username='bob').first().delete()
assert (
DailyXFormSubmissionCounter.objects.get(**criteria).counter == 2
)