Docker revamp

docker-compose.yml

@@ -4,9 +4,11 @@ version: '3'
 
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.16
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.3.1
     environment:
-      - cluster.name=docker-cluster
+      - cluster.name=elasticsearch
+      - node.name=stockdata
+      - cluster.initial_master_nodes=stockdata
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     volumes:
@@ -19,19 +21,19 @@ services:
         soft: 65536
         hard: 65536
       nproc:
-        soft: 2048
-        hard: 2048
+        soft: 4096
+        hard: 4096
     #expose this for local dev only!
-    #ports:
-    #  - "9200:9200"
+    ports:
+      - "9200:9200"
   redis:
     build:
       context: ./redis-docker
     #expose this for local dev only!
     #ports:
     #  - "6379:6379"
   kibana:
-    image: docker.elastic.co/kibana/kibana:5.6.16
+    image: docker.elastic.co/kibana/kibana:7.3.1
     depends_on:
       - elasticsearch
     ports:

kibana_exports/export.7.3.ndjson

@@ -0,0 +1,6 @@
+{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"date\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"location\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"location.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"location\",\"subType\":\"multi\"},{\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"message\",\"subType\":\"multi\"},{\"name\":\"polarity\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sentiment\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sentiment.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"sentiment\",\"subType\":\"multi\"},{\"name\":\"subjectivity\",\"type\":\"number\",\"esTypes\":[\"float\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]","timeFieldName":"date","title":"stocksight_tmpl"},"id":"tmpl_index_pattern","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2019-08-31T04:11:15.828Z","version":"WzcsMV0="}
+{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"tmpl_polarity","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"tmpl_polarity\",\"type\":\"metric\",\"params\":{\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"type\":\"range\",\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":32}},\"dimensions\":{\"metrics\":[{\"type\":\"vis_dimension\",\"accessor\":0,\"format\":{\"id\":\"number\",\"params\":{}}},{\"type\":\"vis_dimension\",\"accessor\":1,\"format\":{\"id\":\"number\",\"params\":{}}},{\"type\":\"vis_dimension\",\"accessor\":2,\"format\":{\"id\":\"number\",\"params\":{}}},{\"type\":\"vis_dimension\",\"accessor\":3,\"format\":{\"id\":\"number\",\"params\":{}}},{\"type\":\"vis_dimension\",\"accessor\":4,\"format\":{\"id\":\"number\",\"params\":{}}}]},\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"polarity\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"median\",\"schema\":\"metric\",\"params\":{\"field\":\"polarity\",\"percents\":[50],\"customLabel\":\"\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"polarity\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"polarity\"}}]}"},"id":"2faf7b70-cba6-11e9-aabf-190f41383a44","migrationVersion":{"visualization":"7.3.1"},"references":[{"id":"tmpl_index_pattern","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2019-08-31T04:17:03.654Z","version":"WzgsMV0="}
+{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"tmpl_sentinel","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"tmpl_sentinel\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"top\",\"isDonut\":false,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"sentiment.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"8a885170-cba6-11e9-aabf-190f41383a44","migrationVersion":{"visualization":"7.3.1"},"references":[{"id":"tmpl_index_pattern","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2019-08-31T04:19:36.071Z","version":"WzksMV0="}
+{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"tmpl_articles","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"tmpl_articles\",\"type\":\"line\",\"params\":{\"type\":\"line\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"line\",\"mode\":\"normal\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}]}"},"id":"a775ddc0-cba6-11e9-aabf-190f41383a44","migrationVersion":{"visualization":"7.3.1"},"references":[{"id":"tmpl_index_pattern","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2019-08-31T04:20:24.604Z","version":"WzEwLDFd"}
+{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"tmpl_titles","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"}}}}","version":1,"visState":"{\"title\":\"tmpl_titles\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"},\"showTotal\":false,\"totalFunc\":\"sum\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"avg\"},{\"accessor\":2,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"avg\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"polarity\",\"customLabel\":\"Polarity\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"message.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Title\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"subjectivity\",\"customLabel\":\"Subjectivity\"}}]}"},"id":"277680e0-cba9-11e9-aabf-190f41383a44","migrationVersion":{"visualization":"7.3.1"},"references":[{"id":"tmpl_index_pattern","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2019-08-31T04:39:07.986Z","version":"WzEzLDFd"}
+{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"hidePanelTitles\":false}","panelsJSON":"[{\"gridData\":{\"x\":10,\"y\":0,\"w\":14,\"h\":15,\"i\":\"1\"},\"version\":\"7.3.1\",\"panelIndex\":\"1\",\"embeddableConfig\":{},\"title\":\"Polarity\",\"panelRefName\":\"panel_0\"},{\"gridData\":{\"x\":0,\"y\":0,\"w\":10,\"h\":15,\"i\":\"2\"},\"version\":\"7.3.1\",\"panelIndex\":\"2\",\"embeddableConfig\":{\"vis\":{\"legendOpen\":false}},\"title\":\"Sentiniel\",\"panelRefName\":\"panel_1\"},{\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":15,\"i\":\"3\"},\"version\":\"7.3.1\",\"panelIndex\":\"3\",\"embeddableConfig\":{},\"title\":\"Article Count\",\"panelRefName\":\"panel_2\"},{\"gridData\":{\"x\":0,\"y\":15,\"w\":24,\"h\":18,\"i\":\"4\"},\"version\":\"7.3.1\",\"panelIndex\":\"4\",\"embeddableConfig\":{},\"title\":\"Title Analysis\",\"panelRefName\":\"panel_3\"}]","timeRestore":false,"title":"tmpl_dashboard","version":1},"id":"eefa1870-cba9-11e9-aabf-190f41383a44","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"2faf7b70-cba6-11e9-aabf-190f41383a44","name":"panel_0","type":"visualization"},{"id":"8a885170-cba6-11e9-aabf-190f41383a44","name":"panel_1","type":"visualization"},{"id":"a775ddc0-cba6-11e9-aabf-190f41383a44","name":"panel_2","type":"visualization"},{"id":"277680e0-cba9-11e9-aabf-190f41383a44","name":"panel_3","type":"visualization"}],"type":"dashboard","updated_at":"2019-08-31T04:57:26.923Z","version":"WzI1LDFd"}

src/Sentiment/NewsHeadlineListener.py

@@ -27,6 +27,7 @@ def __init__(self, symbol,url=None):
         for htext, htext_url in new_headlines:
 
             md5_hash = hashlib.md5((htext+htext_url).encode()).hexdigest()
+            logger.debug("Hash"+md5_hash)
 
             if rds.exists(md5_hash) is 0:
 
@@ -49,19 +50,21 @@ def __init__(self, symbol,url=None):
                         logger.info("Text contains token from ignore list, not adding")
                         rds.set(md5_hash,1,2628000)
                         continue
-                # check required tokens from config
-                tokenspass = False
+
 
 
                 if self.symbol in nltk_tokens_required:
                     nltk_tokens = nltk_tokens_required[self.symbol]
                 else:
                     nltk_tokens = nltk_tokens_required['default']
 
+                # check required tokens from config
+                tokenspass = False
                 for t in nltk_tokens:
                     if t in tokens:
                         tokenspass = True
                         break
+
                 if not tokenspass:
                     logger.info("Text does not contain token from required list, not adding")
                     rds.set(md5_hash,1,2628000)
@@ -72,7 +75,7 @@ def __init__(self, symbol,url=None):
 
                 logger.info("Adding news headline to elasticsearch")
                 # add news headline data and sentiment info to elasticsearch
-                es.index(index=self.symbol,
+                es.index(index='stocksight_'+self.symbol,
                          doc_type="newsheadline",
                          body={"date": datenow,
                                "location": htext_url,

src/delindex.py

@@ -7,12 +7,12 @@
 
     parser = argparse.ArgumentParser()
     parser.add_argument("-d", "--delindex", action="store_true",
-                    help="Delete existing Elasticsearch index first")
+                    help="Delete existing Elasticsearch index")
     args = parser.parse_args()
 
     eslogger = logging.getLogger('elasticsearch')
     eslogger.setLevel(logging.INFO)
 
     if args.delindex:
-        eslogger.info('Deleting existing Elasticsearch index ' + args.index)
+        eslogger.info('Deleting existing Elasticsearch index ' + args.delindex)
         es.indices.delete(index=args.index, ignore=[400, 404])

src/news.sentiment.py

@@ -11,11 +11,8 @@
 """
 
 import argparse
-
-try:
-    import urllib.parse as urlparse
-except ImportError:
-    import urlparse
+import time
+import random
 
 # import elasticsearch host, twitter keys and tokens
 from Sentiment.NewsHeadlineListener import *
@@ -122,6 +119,7 @@
 
                 # create instance of NewsHeadlineListener
                 newslistener = NewsHeadlineListener(symbol, url)
+                time.sleep(random.randrange(2,5))
             except Exception as e:
                 logger.warning("%s" % e)
                 pass