Merge pull request #924 from kobotoolbox/add_revision_cleanup_task

Add revision cleanup task

onadata/apps/logger/maintenance_tasks.py

@@ -0,0 +1,16 @@
+from datetime import timedelta
+
+from django.conf import settings
+from django.utils import timezone
+from reversion.models import Revision
+
+
+def remove_old_revisions():
+    days = settings.KOBOCAT_REVERSION_RETENTION_DAYS
+    delete_queryset = Revision.objects.filter(
+        date_created__lt=timezone.now() - timedelta(days=days),
+    )
+    while True:
+        count, _ = delete_queryset.filter(pk__in=delete_queryset[:1000]).delete()
+        if not count:
+            break

onadata/apps/logger/tasks.py

@@ -15,6 +15,7 @@
 from django.utils import timezone
 
 from onadata.celery import app
+from .maintenance_tasks import remove_old_revisions
 from .models.daily_xform_submission_counter import DailyXFormSubmissionCounter
 from .models import Instance, XForm
 
@@ -122,3 +123,13 @@ def list_created_by_month(model, date_field):
 @app.task()
 def sync_storage_counters():
     call_command('update_attachment_storage_bytes', verbosity=3, sync=True)
+
+
+LIMIT_HOURS_23 = 82800
+
+@app.task(time_limit=LIMIT_HOURS_23, soft_time_limit=LIMIT_HOURS_23)
+def perform_maintenance():
+    """
+    Run daily maintenance tasks
+    """
+    remove_old_revisions()

onadata/apps/logger/tests/models/test_instance.py

@@ -1,15 +1,18 @@
 # coding: utf-8
 import os
-import reversion
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from dateutil import parser
-from django.utils.timezone import utc
+from django.utils import timezone
+from django.test import override_settings
 from django_digest.test import DigestAuth
 from mock import patch
+from reversion import create_revision, is_registered, set_date_created
+from reversion.models import Revision
 
 from onadata.apps.main.tests.test_base import TestBase
 from onadata.apps.logger.models import XForm, Instance
+from onadata.apps.logger.maintenance_tasks import remove_old_revisions
 from onadata.apps.logger.models.instance import get_id_string_from_xml_str
 from onadata.apps.viewer.models import ParsedInstance
 from onadata.libs.utils.common_tags import MONGO_STRFTIME, SUBMISSION_TIME,\
@@ -21,6 +24,16 @@ class TestInstance(TestBase):
     def setUp(self):
         super().setUp()
 
+    def create_transportation_fixture_xml_path(self, index = 0):
+        return os.path.join(
+            self.this_directory,
+            "fixtures",
+            "transportation",
+            "instances",
+            self.surveys[index],
+            self.surveys[index] + ".xml",
+        )
+
     def test_stores_json(self):
         self._publish_transportation_form_and_submit_instance()
         instances = Instance.objects.all()
@@ -30,7 +43,7 @@ def test_stores_json(self):
 
     @patch('django.utils.timezone.now')
     def test_json_assigns_attributes(self, mock_time):
-        mock_time.return_value = datetime.utcnow().replace(tzinfo=utc)
+        mock_time.return_value = timezone.datetime.now(timezone.utc)
         self._publish_transportation_form_and_submit_instance()
 
         xform_id_string = XForm.objects.all()[0].id_string
@@ -44,13 +57,11 @@ def test_json_assigns_attributes(self, mock_time):
 
     @patch('django.utils.timezone.now')
     def test_json_stores_user_attribute(self, mock_time):
-        mock_time.return_value = datetime.utcnow().replace(tzinfo=utc)
+        mock_time.return_value = timezone.datetime.now(timezone.utc)
         self._publish_transportation_form()
 
         # submit instance with a request user
-        path = os.path.join(
-            self.this_directory, 'fixtures', 'transportation', 'instances',
-            self.surveys[0], self.surveys[0] + '.xml')
+        path = self.create_transportation_fixture_xml_path()
 
         auth = DigestAuth(self.login_username, self.login_password)
         self._make_submission(path, auth=auth)
@@ -115,4 +126,29 @@ def test_get_id_string_from_xml_str(self):
         self.assertEqual(id_string, 'id_string')
 
     def test_reversion(self):
-        self.assertTrue(reversion.is_registered(Instance))
+        self.assertTrue(is_registered(Instance))
+
+    @override_settings(KOBOCAT_REVERSION_RETENTION_DAYS=2)
+    def test_revision_cleanup(self):
+        days_ago_3 = timezone.now() - timedelta(days=3)
+        self._publish_transportation_form()
+
+        path = self.create_transportation_fixture_xml_path()
+
+        with create_revision():
+            self._make_submission(path, forced_submission_time=days_ago_3)
+            set_date_created(days_ago_3)
+        old_revision = Revision.objects.first()
+
+        path = self.create_transportation_fixture_xml_path(1)
+
+        with create_revision():
+            self._make_submission(path)
+        new_revision = Revision.objects.first()
+
+        assert Revision.objects.count() == 2
+
+        remove_old_revisions()
+
+        assert not Revision.objects.filter(id=old_revision.id).exists()
+        assert Revision.objects.filter(id=new_revision.id).exists()

onadata/settings/base.py

@@ -665,6 +665,7 @@ def skip_suspicious_operations(record):
 REVERSION_MIDDLEWARE_SKIPPED_URL_PATTERNS = {
     r'/api/v1/users/(.*)': ['DELETE']
 }
+KOBOCAT_REVERSION_RETENTION_DAYS = env.int("KOBOCAT_REVERSION_RETENTION_DAYS", 90)
 
 # run heavy migration scripts by default
 # NOTE: this should be set to False for major deployments. This can take a long time
@@ -708,16 +709,22 @@ def skip_suspicious_operations(record):
 CELERY_BEAT_SCHEDULE = {
     # Periodically mark exports stuck in the "pending" state as "failed"
     # See https://github.com/kobotoolbox/kobocat/issues/315
-    'log-stuck-exports-and-mark-failed': {
-        'task': 'onadata.apps.viewer.tasks.log_stuck_exports_and_mark_failed',
-        'schedule': timedelta(hours=6),
-        'options': {'queue': 'kobocat_queue'}
+    "log-stuck-exports-and-mark-failed": {
+        "task": "onadata.apps.viewer.tasks.log_stuck_exports_and_mark_failed",
+        "schedule": timedelta(hours=6),
+        "options": {"queue": "kobocat_queue"},
+    },
+    "delete-daily-xform-submissions-counter": {
+        "task": "onadata.apps.logger.tasks.delete_daily_counters",
+        "schedule": crontab(hour=0, minute=0),
+        "options": {"queue": "kobocat_queue"},
+    },
+    # Run maintenance every day at 20:00 UTC
+    "perform-maintenance": {
+        "task": "onadata.apps.logger.tasks.perform_maintenance",
+        "schedule": crontab(hour=20, minute=0),
+        "options": {"queue": "kobocat_queue"},
     },
-    'delete-daily-xform-submissions-counter': {
-        'task': 'onadata.apps.logger.tasks.delete_daily_counters',
-        'schedule': crontab(hour=0, minute=0),
-        'options': {'queue': 'kobocat_queue'}
-    }
 }
 
 CELERY_TASK_DEFAULT_QUEUE = "kobocat_queue"