Realtime Django Bugfix

deployment/docker-dev/REQUIREMENTS.txt

@@ -1,26 +1,26 @@
-requests
+requests==2.11.1
 Django>=1.7,<1.8
-django-bootstrap3-datetimepicker
-psycopg2
-pytz
-celery
-django-celery
-django-kombu
+django-bootstrap3-datetimepicker==2.2.3
+psycopg2==2.6.2
+pytz==2016.7
+celery==3.1.24
+django-celery==3.1.17
+django-kombu==0.9.4
 django-leaflet==0.14.0
-django-braces
-django-model-utils
+django-braces==1.9.0
+django-model-utils==2.6
 django-pipeline<1.6,>=1.5
-nodeenv
-raven
-django-user-map
+nodeenv==1.0.0
+raven==5.29.0
+django-user-map==1.1.1
 djangorestframework==3.2.5
 djangorestframework-gis==0.9.6
-markdown
+Markdown==2.6.7
 django-filter==0.11.0
-pillow
-numpy
-hammock
-scrapy
-python-dateutil
-unicodecsv
-beautifulsoup4
+Pillow==3.4.2
+numpy==1.11.2
+hammock==0.2.4
+Scrapy==1.2.0
+python-dateutil==2.5.3
+unicodecsv==0.14.1
+beautifulsoup4==4.5.1

deployment/docker-prod/REQUIREMENTS.txt

@@ -1,26 +1,26 @@
-requests
+requests==2.11.1
 Django>=1.7,<1.8
-django-bootstrap3-datetimepicker
-psycopg2
-pytz
-celery
-django-celery
-django-kombu
+django-bootstrap3-datetimepicker==2.2.3
+psycopg2==2.6.2
+pytz==2016.7
+celery==3.1.24
+django-celery==3.1.17
+django-kombu==0.9.4
 django-leaflet==0.14.0
-django-braces
-django-model-utils
+django-braces==1.9.0
+django-model-utils==2.6
 django-pipeline<1.6,>=1.5
-nodeenv
-raven
-django-user-map
+nodeenv==1.0.0
+raven==5.29.0
+django-user-map==1.1.1
 djangorestframework==3.2.5
 djangorestframework-gis==0.9.6
-markdown
+Markdown==2.6.7
 django-filter==0.11.0
-pillow
-numpy
-hammock
-scrapy
-python-dateutil
-unicodecsv
-beautifulsoup4
+Pillow==3.4.2
+numpy==1.11.2
+hammock==0.2.4
+Scrapy==1.2.0
+python-dateutil==2.5.3
+unicodecsv==0.14.1
+beautifulsoup4==4.5.1

django_project/core/settings/celery_config.py

@@ -22,6 +22,7 @@
 CELERY_DEFAULT_ROUTING_KEY = "default"
 CELERY_CREATE_MISSING_QUEUES = True
 CELERYD_CONCURRENCY = 1
+CELERYD_PREFETCH_MULTIPLIER = 1
 
 CELERY_QUEUES = [
     Queue('default', routing_key='default'),

django_project/core/settings/test_travis.py

@@ -12,3 +12,6 @@
         'PORT': '',
     }
 }
+
+# For local testing without celery worker
+CELERY_ALWAYS_EAGER = True

django_project/core/settings/utils.py

@@ -23,5 +23,6 @@ def ensure_secret_key_file():
         with open(secret_path, 'w') as f:
             f.write("SECRET_KEY = " + repr(secret_key) + "\n")
 
+
 # Import the secret key
 ensure_secret_key_file()

django_project/realtime/admin.py

@@ -70,7 +70,9 @@ class AshReportAdmin(ModelAdmin):
 
 class VolcanoAdmin(ModelAdmin):
     """Admin class for volcano model"""
-    list_display = ('volcano_name', 'location', 'elevation', 'province', 'district', 'morphology')
+    list_display = (
+        'volcano_name', 'location', 'elevation', 'province', 'district',
+        'morphology')
     list_filter = ('province', 'district', 'morphology')
     search_fields = ['volcano_name', 'province', 'district', 'morphology']
 

django_project/realtime/forms/__init__.py

@@ -0,0 +1,6 @@
+# coding=utf-8
+
+__copyright__ = "Copyright 2016, The InaSAFE Project"
+__license__ = "GPL version 3"
+__email__ = "[email protected]"
+__revision__ = ':%H$'

django_project/realtime/forms.py → django_project/realtime/forms/ash.py

@@ -1,54 +1,12 @@
 # coding=utf-8
 """Forms for realtime app."""
 from bootstrap3_datetime.widgets import DateTimePicker
-
 from django import forms
 from django.utils.translation import ugettext_lazy as _
-from realtime.models.earthquake import Earthquake
 from realtime.models.ash import Ash
 
 
-class EarthquakeForm(forms.ModelForm):
-    class Meta:
-        model = Earthquake
-        fields = [
-            'shake_id',
-            'magnitude',
-            'time',
-            'depth',
-            'location',
-            'location_description'
-        ]
-
-
-date_format = 'YYYY-MM-DD'
-date_picker = DateTimePicker(
-    format=date_format,
-    options={
-        'pickTime': False
-    })
 datetime_format = 'YYYY-MM-DD HH:mm:ss'
-datetime_picker = DateTimePicker(
-    format=datetime_format,
-    options={
-        'pickTime': True,
-        'pickSeconds': True,
-    })
-
-
-class FilterForm(forms.Form):
-    start_date = forms.DateField(widget=date_picker, label=_('Start Date'))
-    end_date = forms.DateField(widget=date_picker, label=_('End Date'))
-    minimum_magnitude = forms.IntegerField(
-        min_value=0,
-        max_value=10,
-        label=_('Minimum Magnitude'))
-    maximum_magnitude = forms.IntegerField(
-        min_value=0,
-        max_value=10,
-        label=_('Maximum Magnitude'))
-    # hidden field for felt shakes
-    felt = forms.BooleanField()
 
 
 class AshUploadForm(forms.ModelForm):
@@ -70,7 +28,12 @@ class Meta:
 
     event_time = forms.DateTimeField(
         # initial=datetime.datetime.now(),
-        widget=datetime_picker)
+        widget=DateTimePicker(
+            format=datetime_format,
+            options={
+                'pickTime': True,
+                'pickSeconds': True,
+            }))
     alert_level = forms.ChoiceField(
         choices=[
             ('normal', 'Normal'),

django_project/realtime/forms/earthquake.py

@@ -0,0 +1,51 @@
+# coding=utf-8
+"""Forms for realtime app."""
+from bootstrap3_datetime.widgets import DateTimePicker
+from django import forms
+from django.utils.translation import ugettext_lazy as _
+
+from realtime.models.earthquake import Earthquake
+
+
+class EarthquakeForm(forms.ModelForm):
+    class Meta:
+        model = Earthquake
+        fields = [
+            'shake_id',
+            'magnitude',
+            'time',
+            'depth',
+            'location',
+            'location_description'
+        ]
+
+
+date_format = 'YYYY-MM-DD'
+datetime_format = 'YYYY-MM-DD HH:mm:ss'
+
+
+class FilterForm(forms.Form):
+    start_date = forms.DateField(
+        widget=DateTimePicker(
+            format=date_format,
+            options={
+                'pickTime': False
+            }),
+        label=_('Start Date'))
+    end_date = forms.DateField(
+        widget=DateTimePicker(
+            format=date_format,
+            options={
+                'pickTime': False
+            }),
+        label=_('End Date'))
+    minimum_magnitude = forms.IntegerField(
+        min_value=0,
+        max_value=10,
+        label=_('Minimum Magnitude'))
+    maximum_magnitude = forms.IntegerField(
+        min_value=0,
+        max_value=10,
+        label=_('Maximum Magnitude'))
+    # hidden field for felt shakes
+    felt = forms.BooleanField()

django_project/realtime/forms/flood.py

@@ -0,0 +1,37 @@
+# coding=utf-8
+"""Forms for realtime app."""
+from bootstrap3_datetime.widgets import DateTimePicker
+from django import forms
+from django.utils.translation import ugettext_lazy as _
+
+date_format = 'YYYY-MM-DD'
+datetime_format = 'YYYY-MM-DD HH:mm:ss'
+
+
+class FilterForm(forms.Form):
+    start_date = forms.DateField(
+        widget=DateTimePicker(
+            format=date_format,
+            options={
+                'pickTime': False
+            }),
+        label=_('Start Date'))
+    end_date = forms.DateField(
+        widget=DateTimePicker(
+            format=date_format,
+            options={
+                'pickTime': False
+            }),
+        label=_('End Date'))
+    min_people_affected = forms.IntegerField(
+        min_value=0,
+        label=_('Minimum'))
+    max_people_affected = forms.IntegerField(
+        min_value=0,
+        label=_('Maximum'))
+    min_boundary_flooded = forms.IntegerField(
+        min_value=0,
+        label=_('Minimum'))
+    max_boundary_flooded = forms.IntegerField(
+        min_value=0,
+        label=_('Maximum'))

django_project/realtime/management/commands/recalculateimpactdata.py

@@ -0,0 +1,76 @@
+# coding=utf-8
+import datetime
+
+import pytz
+from django.core.management.base import BaseCommand
+from realtime.tasks.flood import recalculate_impact_info
+
+from realtime.models.flood import Flood
+
+__author__ = 'Rizky Maulana Nugraha "lucernae" <[email protected]>'
+__date__ = '07/02/17'
+
+
+class Command(BaseCommand):
+    """Script to recalculate impact data.
+
+    """
+    help = (
+        'Command to re-calculate total affected and boundary flooded in '
+        'flood.')
+
+    def handle(self, *args, **options):
+        using_range = False
+        if len(args) == 3:
+            if args[0] == 'range':
+                using_range = True
+                Command.recalculate_flood_from_range(args[1], args[2])
+
+        if len(args) > 0 and not using_range:
+            for a in args:
+                print 'Process flood : %s' % a
+                flood = Flood.objects.get(event_id=a)
+                try:
+                    recalculate_impact_info(flood)
+                except Exception as e:
+                    print e
+
+        elif not using_range:
+            floods = Flood.objects.all().order_by('-time')
+            print 'Process flood (%s)' % len(floods)
+            for flood in floods:
+                try:
+                    recalculate_impact_info(flood)
+                except Exception as e:
+                    print e
+
+    @staticmethod
+    def recalculate_flood_from_range(start_event_id, end_event_id):
+        format_str = '%Y%m%d%H-6-rw'
+        start_time = datetime.datetime.strptime(start_event_id, format_str)
+        if not end_event_id == 'now':
+            end_time = datetime.datetime.strptime(end_event_id, format_str)
+        else:
+            end_time = datetime.datetime.utcnow()
+        # convert to UTC
+        start_time = start_time.replace(tzinfo=pytz.UTC)
+        end_time = end_time.replace(tzinfo=pytz.UTC)
+        time_diff = end_time - start_time
+        total_hours = int(time_diff.total_seconds() / 3600)
+        success = 0
+        failed = 0
+        for i in range(0, total_hours):
+            hour_diff = datetime.timedelta(hours=i + 1)
+            target_time = start_time + hour_diff
+            event_id = target_time.strftime(format_str)
+            try:
+                print 'Processing flood: %s' % event_id
+                flood = Flood.objects.get(event_id=event_id)
+                recalculate_impact_info(flood)
+                success += 1
+            except Exception as e:
+                failed += 1
+                print e
+
+        print 'Recalculate process done'
+        print 'Success: %s. Failed: %s' % (success, failed)

django_project/realtime/migrations/0024_auto_20170206_2044.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('realtime', '0023_auto_20161022_0957'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='ash',
+            name='hazard_file',
+            field=models.FileField(help_text=b'Hazard file formatted as GeoTIFF (*.tif) in EPSG:4326.', upload_to=b'ash/hazard_file/%Y/%m/%d', verbose_name=b'Hazard File'),
+            preserve_default=True,
+        ),
+    ]