Docker revamp

README.md

@@ -139,3 +139,10 @@ optional arguments:
   -q, --quiet           Run quiet with no message output
   -V, --version         Prints version and exits
   ```
+
+### HOWTO DOCKER
+- Change config.py
+- Change startup.sh to include your tickers
+- run docker-compose up
+- ???
+- Profit

docker-compose.yml

@@ -21,6 +21,9 @@ services:
       nproc:
         soft: 2048
         hard: 2048
+    #expose this for local dev only!
+    #ports:
+    #  - "9200:9200"
   kibana:
     image: docker.elastic.co/kibana/kibana:5.6.16
     depends_on:

requirements.txt

@@ -5,3 +5,4 @@ tweepy
 beautifulsoup4
 textblob
 vaderSentiment
+pytz

src/config.py.sample → src/config.sample.py

@@ -1,15 +1,18 @@
+#Global Config
 elasticsearch_host = "elasticsearch"
 elasticsearch_port = 9200
 elasticsearch_user = ""
 elasticsearch_password = ""
+
+#Sentiment Analyizers config
 consumer_key = ""
 consumer_secret = ""
 access_token = ""
 access_token_secret = ""
 nltk_tokens_required = {
- 'default': ("Tesla", "@Tesla", "#Tesla", "tesla", "TSLA", "tsla", "#TSLA", "#tsla", "elonmusk", "Elon", "Musk"),
- 'tsla':  ("Tesla", "@Tesla", "#Tesla", "tesla", "TSLA", "tsla", "#TSLA", "#tsla", "elonmusk", "Elon", "Musk"),
- 'amd': ('amd','ryzen','epyc','radeon','server','data','center','crossfire','threadripper')
+ 'default': ("increase","decrease","buying","sold","buy","selling","winning","losing"),
+ 'tsla':  ("tesla", "@tesla", "#tesla",  "tsla", "#tsla", "elonmusk", "elon", "musk"),
+ 'amd': ('amd','ryzen','epyc','radeon','crossfire','threadripper')
 }
 nltk_tokens_ignored = ("win", "Win", "giveaway", "Giveaway")
 twitter_feeds = ["@elonmusk", "@cnbc", "@benzinga", "@stockwits",
@@ -19,5 +22,13 @@
                  "@Carl_C_Icahn", "@ReformedBroker", "@bespokeinvest", "@stlouisfed",
                  "@muddywatersre", "@mcuban", "@AswathDamodaran", "@elerianm",
                  "@MorganStanley", "@ianbremmer", "@GoldmanSachs", "@Wu_Tang_Finance",
-                 "@Schuldensuehner", "@NorthmanTrader", "@Frances_Coppola", "@bySamRo",
-                 "@BuzzFeed","@nytimes"]
+                 "@Schuldensuehner", "@NorthmanTrader", "@Frances_Coppola", "@BuzzFeed","@nytimes"]
+sentiment_frequency = 3600
+
+#Stock Price fetcher config
+price_frequency = 900
+weekday_start = 1
+weekday_end = 5
+hour_start = 9
+hour_end = 18
+timezone_str = 'America/Toronto'

src/sentiment.py

@@ -200,15 +200,18 @@ def on_timeout(self):
 
 
 class NewsHeadlineListener:
-    def __init__(self, url=None, frequency=3600):
+    def __init__(self, url=None, frequency=sentiment_frequency):
         self.url = url
         self.headlines = []
         self.followedlinks = []
         self.frequency = frequency
+        self.max_cache = 1000;
 
         while True:
             new_headlines = self.get_news_headlines(self.url)
 
+            self.cleanup()
+
             # add any new headlines
             for htext, htext_url in new_headlines:
                 if htext not in self.headlines:
@@ -265,6 +268,21 @@ def __init__(self, url=None, frequency=3600):
 
             logger.info("Will get news headlines again in %s sec..." % self.frequency)
             time.sleep(self.frequency)
+    def cleanup(self):
+        new_headline = []
+        new_followlink = []
+        if len(self.headlines) > self.max_cache:
+            for i in range(self.max_cache / 2, len(self.headlines) - 1):
+              new_headline.append(self.headlines[i])
+
+        self.headlines = new_headline
+
+        if len(self.followedlinks) > self.max_cache:
+            for i in range(self.max_cache / 2, len(self.followedlinks) - 1):
+              new_followlink.append(self.followedlinks[i])
+
+        self.followedlinks = new_followlink
+
 
     def get_news_headlines(self, url):
 
@@ -494,8 +512,8 @@ def get_twitter_users_from_file(file):
                         help="Use twitter user ids from file")
     parser.add_argument("-n", "--newsheadlines", metavar="SYMBOL",
                         help="Get news headlines instead of Twitter using stock symbol, example: TSLA")
-    parser.add_argument("--frequency", metavar="FREQUENCY", default=3600, type=int,
-                        help="How often in seconds to retrieve news headlines (default: 3600 sec)")
+    parser.add_argument("--frequency", metavar="FREQUENCY", default=sentiment_frequency, type=int,
+                        help="How often in seconds to retrieve news headlines (default: %d sec)" % sentiment_frequency)
     parser.add_argument("--followlinks", action="store_true",
                         help="Follow links on news headlines and scrape relevant text from landing page")
     parser.add_argument("-v", "--verbose", action="store_true",

src/startup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-sleep 15
+sleep 30
 python sentiment.py -n TSLA --followlinks -i tsla &
 sleep 1
 python stockprice.py -s TSLA -i tsla &

src/stockprice.py

@@ -14,8 +14,10 @@
 import logging
 import sys
 import time
-
+import datetime
+import re
 import requests
+from pytz import timezone
 
 try:
     from elasticsearch5 import Elasticsearch
@@ -24,7 +26,8 @@
 from random import randint
 
 # import elasticsearch host
-from config import elasticsearch_host, elasticsearch_port, elasticsearch_user, elasticsearch_password
+from config import elasticsearch_host, elasticsearch_port, elasticsearch_user, elasticsearch_password, \
+                   price_frequency, weekday_start, weekday_end, hour_start, hour_end, timezone_str
 
 
 STOCKSIGHT_VERSION = '0.1-b.5'
@@ -37,19 +40,31 @@
 es = Elasticsearch(hosts=[{'host': elasticsearch_host, 'port': elasticsearch_port}],
                    http_auth=(elasticsearch_user, elasticsearch_password))
 
+regex = re
+
 class GetStock:
 
     def get_price(self, url, symbol):
-        import re
+
+        eastern_timezone = timezone(timezone_str)
 
         while True:
 
+            if self.isNotLive(eastern_timezone):
+                #logger.info("Stock market is not live. Current time: %s" % datetime.datetime.now(timezone).strftime("%Y-%m-%d %H:%M"))
+                today = datetime.datetime.now(eastern_timezone)
+                logger.info("Stock market is not live. Current time: %s" % today.strftime('%H'))
+                logger.info("Will get stock data again in %s sec..." % args.frequency)
+                time.sleep(args.frequency)
+                continue
+
+
             logger.info("Grabbing stock data for symbol %s..." % symbol)
 
             try:
 
                 # add stock symbol to url
-                url = re.sub("SYMBOL", symbol, url)
+                url = regex.sub("SYMBOL", symbol, url)
                 # get stock data (json) from url
                 try:
                     r = requests.get(url)
@@ -113,6 +128,17 @@ def get_price(self, url, symbol):
             logger.info("Will get stock data again in %s sec..." % args.frequency)
             time.sleep(args.frequency)
 
+    def isNotLive(self, timezone):
+        today = datetime.datetime.now(timezone);
+        if today.weekday() >= weekday_start and \
+           today.weekday() <= weekday_end and \
+           today.hour() >= hour_start and \
+           today.hour() <= hour_end:
+            return False;
+
+        return True;
+
+
 
 if __name__ == '__main__':
 
@@ -124,8 +150,8 @@ def get_price(self, url, symbol):
                         help="Delete existing Elasticsearch index first")
     parser.add_argument("-s", "--symbol", metavar="SYMBOL",
                         help="Stock symbol to use, example: TSLA")
-    parser.add_argument("-f", "--frequency", metavar="FREQUENCY", default=600, type=int,
-                        help="How often in seconds to retrieve stock data (default: 120 sec)")
+    parser.add_argument("-f", "--frequency", metavar="FREQUENCY", default=price_frequency, type=int,
+                        help="How often in seconds to retrieve stock data (default: %d sec)" % price_frequency)
     parser.add_argument("-v", "--verbose", action="store_true",
                         help="Increase output verbosity")
     parser.add_argument("--debug", action="store_true",