Improved Monitoring

django_logtail/admin.py

@@ -0,0 +1,98 @@
+import json
+from os.path import getsize, isfile
+
+from django.contrib import admin
+from django.http import Http404, HttpResponse
+from django.template.response import TemplateResponse
+from django.urls import re_path
+
+from django_logtail import app_settings
+from django_logtail.models import Log
+
+
+class HttpUnauthorized(HttpResponse):
+    status_code = 401
+
+
+class LogAdmin(admin.ModelAdmin):
+    class Media:
+        js = (
+            "admin/js/vendor/jquery/jquery.min.js",
+            "logtail/js/logtail.js",
+        )
+        css = {"all": ("logtail/css/logtail.css",)}
+
+    def has_add_permission(self, request):
+        return False
+
+    def log_view(self, request, logfile="", seek_to="0"):
+        context = self.log_context(logfile, seek_to)
+        return HttpResponse(self.iter_json(context), content_type="application/json")
+
+    def log_context(self, logfile, seek_to):
+        context = {}
+        seek_to = int(seek_to)
+        try:
+            log_file = app_settings.LOGTAIL_FILES[logfile]
+        except KeyError:
+            raise Http404("No such log file")
+
+        try:
+            file_length = getsize(log_file)
+        except OSError:
+            raise Http404("Cannot access file")
+
+        if seek_to > file_length:
+            seek_to = file_length
+
+        try:
+            context["log"] = open(log_file, "r")
+            context["log"].seek(seek_to)
+            context["starts"] = seek_to
+        except IOError:
+            raise Http404("Cannot access file")
+
+        return context
+
+    def iter_json(self, context):
+        yield '{"starts": "%d",' '"data": "' % context["starts"]
+
+        while True:
+            line = context["log"].readline()
+            if line:
+                yield json.dumps(line).strip('"')
+            else:
+                yield '", "ends": "%d"}' % context["log"].tell()
+                context["log"].close()
+                return
+
+    def changelist_view(self, request, extra_context=None):
+        context = dict(
+            title="Logtail",
+            app_label=self.model._meta.app_label,
+            cl=None,
+            media=self.media,
+            has_add_permission=self.has_add_permission(request),
+            update_interval=app_settings.LOGTAIL_UPDATE_INTERVAL,
+            logfiles=((li, f) for li, f in app_settings.LOGTAIL_FILES.items() if isfile(f)),
+        )
+
+        return TemplateResponse(
+            request,
+            "logtail/logtail_list.html",
+            context,
+        )
+
+    def get_urls(self):
+        urls = super(LogAdmin, self).get_urls()
+        urls.insert(
+            0,
+            re_path(
+                r"^(?P<logfile>[-\w\.]+)/(?P<seek_to>\d+)/$",
+                self.admin_site.admin_view(self.log_view),
+            ),
+        )
+        return urls
+
+
+admin.site.register(Log, LogAdmin)

django_logtail/app_settings.py

@@ -0,0 +1,4 @@
+from django.conf import settings
+
+LOGTAIL_FILES = getattr(settings, "LOGTAIL_FILES", {})
+LOGTAIL_UPDATE_INTERVAL = getattr(settings, "LOGTAIL_UPDATE_INTERVAL", "3000")

django_logtail/models.py

@@ -0,0 +1,7 @@
+from django.db import models
+
+
+class Log(models.Model):
+    class Meta:
+        managed = False
+        permissions = (("can_view_logs", "Can view logs"),)

django_logtail/static/logtail/css/logtail.css

@@ -0,0 +1,9 @@
+#log-tail {
+  height: auto;
+  overflow-x: scroll;
+  white-space: pre;
+  font-family: monospace;
+}
+.object-tools {
+  margin-top: 5px;
+}

django_logtail/static/logtail/js/logtail.js

@@ -0,0 +1,72 @@
+(function($) {
+  window.LogTailer = function(baseUrl, updateInterval, timeout, autoScroll, autoPoll) {
+    this.baseUrl = baseUrl;
+    this.timeout = timeout || (1000 * 60 * 8);
+    this.updateInterval = updateInterval || 4000;
+    this.autoScroll = autoScroll || false;
+    this.autoPoll = autoPoll || true;
+    this.requesting = false;
+    this.position = 0;
+    this.$textElement = $("#log-tail");
+  }
+
+  LogTailer.prototype = {
+
+    poll: function(log) {
+      this.log = log;
+      this.position = 0;
+      this.requesting = false;
+
+      if (this.interval) {
+        clearInterval(this.interval);
+      }
+      this.$textElement.text('');
+
+      var thisTailer = this;
+      this.interval = setInterval(function() {
+        thisTailer.makeRequest();
+      }, this.updateInterval);
+
+      // Make a request now to avoid waiting
+      thisTailer.makeRequest();
+    },
+
+    stop: function() {
+      if (this.interval) {
+        clearInterval(this.interval);
+      }
+    },
+
+    makeRequest: function() {
+      if (!this.requesting && this.autoPoll) {
+        this.requesting = true;
+
+        var fullUrl = this.baseUrl + this.log + "/" + this.position + "/",
+            thisTailer = this;
+
+        $.getJSON(fullUrl, function(data) {
+          thisTailer.handleResponse(data);
+        });
+      }
+    },
+
+    handleResponse: function(data) {
+      if (this.requesting) {
+        this.position = data['ends'];
+        this.$textElement.text(this.$textElement.text() + data["data"]);
+
+        if (this.autoScroll) {
+          this.scrollBottom();
+        }
+      }
+      this.requesting = false;
+    },
+
+    scrollBottom: function() {
+      $('html, body').animate(
+        {scrollTop: $(document).height()-$(window).height()},
+        0
+      );
+    }
+  }
+})(django.jQuery);

django_logtail/templates/logtail/logtail_list.html

@@ -0,0 +1,78 @@
+{% extends "admin/change_list.html" %}
+
+
+{% block breadcrumbs %}
+<div class="breadcrumbs">
+  <a href="{% url 'admin:index' %}">Home</a>
+</div>
+{% endblock %}
+
+{% block content %}
+  <div id="content-main">
+    <div class="module">
+      <table id="change-history">
+        <thead>
+          <tr>
+            <th scope="col">Log</th>
+            <th scope="col">Path</th>
+          </tr>
+        </thead>
+        <tbody>
+          {% for log, path in logfiles %}
+            <tr>
+              <th scope="row"><a href="#" class="logfile_change">{{ log }}</a></th>
+              <td>{{ path }}</td>
+            </tr>
+          {% endfor %}
+        </tbody>
+      </table>
+    </div>
+    <div>
+<div id="log-tail">
+</div>
+    </div>
+  </div>
+  <ul class="object-tools">
+    <li><a href="#" id="auto-scroll-toggle" class="historylink">Turn Auto-Scroll On</a></li>
+    <li><a href="#" id="auto-poll-toggle" class="historylink">Turn Auto-Poll Off</a></li>
+  </ul>
+
+  <script type="text/javascript">
+  (function($) {
+    var baseUrl = "{% url 'admin:django_logtail_log_changelist' %}";
+    var updateInterval = {{ update_interval }};
+    window.tailer = new window.LogTailer(baseUrl, updateInterval);
+
+    // Configure JQuery Events
+    $('.logfile_change').on('click', function() {
+      var log = $(this).text();
+      window.tailer.poll(log);
+    });
+
+    $('#auto-scroll-toggle').on('click', function(e) {
+      e.preventDefault();
+      if (window.tailer.autoScroll) {
+        window.tailer.autoScroll = false;
+        $(this).text('Turn Auto-Scroll On');
+      }
+      else {
+        window.tailer.autoScroll = true;
+        $(this).text('Turn Auto-Scroll Off');
+      }
+    });
+
+    $('#auto-poll-toggle').on('click', function(e) {
+      e.preventDefault();
+      if (window.tailer.autoPoll) {
+        window.tailer.autoPoll = false;
+        $(this).text('Turn Auto-Poll On');
+      }
+      else {
+        window.tailer.autoPoll = true;
+        $(this).text('Turn Auto-Poll Off');
+      }
+    });
+  })(django.jQuery);
+
+  </script>
+{% endblock %}

goals_zone/settings.py

@@ -67,15 +67,16 @@
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "django.contrib.postgres",
+    "background_task",
     "matches.apps.MatchesConfig",
     "msg_events.apps.MsgEventsConfig",
     "monitoring.apps.MonitoringConfig",
     "corsheaders",
     "ner.apps.NerConfig",
-    "background_task",
     "rest_framework",
     "rangefilter",
     "django_hosts",
+    "django_logtail",
 ]
 
 MIDDLEWARE = [
@@ -160,6 +161,17 @@
     "PAGE_SIZE": 50,
 }
 
+LOGTAIL_FILES = {
+    "Background Tasks": "/home/goals_zone/logs/background_tasks.log",
+    "Background Tasks Errors": "/home/goals_zone/logs/background_tasks-error.log",
+    "Gunicorn": "/home/goals_zone/logs/gunicorn-error.log",
+    "goals.zone": "/home/goals_zone/logs/goals_zone.log",
+    "goals.zone Errors": "/home/goals_zone/logs/goals_zone-error.log",
+    "nginx Access": "/home/goals_zone/logs/nginx-access.log",
+    "nginx Errors": "/home/goals_zone/logs/nginx-error.log",
+    "Supervisor": "/var/log/supervisor/supervisord.log",
+}
+
 # Internationalization
 # https://docs.djangoproject.com/en/2.2/topics/i18n/
 

monitoring/admin.py

@@ -1,5 +1,6 @@
 import csv
 
+from background_task.models import Task
 from django.contrib import admin
 from django.http import HttpResponse
 from rangefilter.filters import DateRangeFilter
@@ -34,3 +35,19 @@ class PerformanceMonitorEventAdmin(admin.ModelAdmin):
         ("timestamp", DateRangeFilter),
     )
     actions = [export_performance_monitor_events]
+
+
+def reset_background_task(self, request, queryset):
+    queryset.update(attempts=0, run_at="2019-01-01 00:00:00.000000 +00:00")
+
+
+reset_background_task.short_description = "Reset Background Tasks"
+
+
+class BackgroundTaskAdmin(admin.ModelAdmin):
+    list_display = ("task_name", "run_at", "has_error", "attempts", "failed_at", "last_error")
+    actions = [reset_background_task]
+
+
+admin.site.unregister(Task)
+admin.site.register(Task, BackgroundTaskAdmin)