Merge pull request #196 from jpappel/counts_scripts

Make get, correct, and set count scripts Python 3.8 Compatible

scripts/README.md

@@ -0,0 +1,13 @@
+# Cron jobs
+
+## `get_correct-set.sh`
+
+On the production machine `get_correct_set.sh` is set to run as cronjob on the root user.
+
+The current cron job is as follows:
+
+```
+0 */6 * * * /home/cs334/mirruations/scripts/get_correct_set.sh
+```
+
+Output logs are stored in `/var/log/mirruations_counts.log`.

scripts/correct_counts.py

@@ -16,13 +16,18 @@ class JobsInQueueException(Exception):
 
 def strategy_cap(recieved: Counts, ignore_queue: bool) -> Counts:
     filtered = deepcopy(recieved)
+    no_changes = True
     if filtered["queue_size"] != 0 and not ignore_queue:
         raise JobsInQueueException(f'Found jobs in job queue: {filtered["queue_size"]}')
     for entity_type in ("dockets", "documents", "comments"):
         total_ = filtered[entity_type]["total"]
         downloaded = filtered[entity_type]["downloaded"]
+        no_changes &= total_ > downloaded
         filtered[entity_type]["downloaded"] = min(total_, downloaded)
 
+    if no_changes:
+        print("No downloaded count exceeds it's total", file=sys.stderr)
+
     return filtered
 
 
@@ -86,10 +91,10 @@ def strategy_diff(recieved: Counts, ignore_queue: bool) -> Counts:
                 with open(pathlib.Path(args.input), "r") as fp:
                     input_counts = json.load(fp, cls=CountsDecoder)
             except FileNotFoundError:
-                print(f"Missing file {args.input}, exitting", file=sys.stderr)
+                print(f"Missing file {args.input}, exiting", file=sys.stderr)
                 sys.exit(2)
     except JSONDecodeError:
-        print(f"Malformed input file {args.input}, exitting", file=sys.stderr)
+        print(f"Malformed input file {args.input}, exiting", file=sys.stderr)
         sys.exit(2)
 
     try:
@@ -98,7 +103,7 @@ def strategy_diff(recieved: Counts, ignore_queue: bool) -> Counts:
         elif args.strategy == "diff_total_with_jobs":
             modified_counts = strategy_diff(input_counts, args.ignore_queue)
         else:
-            print(f"Unrecognized strategy {args.strategy}, exitting", file=sys.stderr)
+            print(f"Unrecognized strategy {args.strategy}, exiting", file=sys.stderr)
             sys.exit(1)
     except JobsInQueueException as e:
         print(

scripts/get_correct_set.cron

@@ -0,0 +1 @@
+0 */6 * * * /home/cs334/mirruations/scripts/get_correct_set.sh

scripts/get_correct_set.sh

@@ -2,14 +2,14 @@
 
 WORK_DIR="/home/cs334/mirrulations/scripts/"
 LOG_FILE=/var/log/mirrulations_counts.log
-START_TIME=$(date -u -Iseconds)
-echo "$START_TIME: RUnning" > $LOG_FILE
-cd $WORK_DIR
+START_TIME=$(date -Iseconds)
 
-PYTHON=".venv/bin/python3"
+echo "$START_TIME: Running" >> $LOG_FILE
+cd $WORK_DIR
+source .venv/bin/python3/activate
 
-$PYTHON get_counts redis -o "/tmp/mirrulations_$START_TIME.json" 2>> $LOG_FILE &&
-    $PYTHON correct_counts -i "/tmp/mirrulations_$START_TIME.json" -o "/tmp/mirrulations_${START_TIME}_corrected.json" 2>> $LOG_FILE &&
-    $PYTHON set_counts -y -i "/tmp/mirrulations_${START_TIME}_corrected.json" 2>> $LOG_FILE
+./get_counts.py -o "/tmp/mirrulations_$START_TIME.json" redis 2>> $LOG_FILE &&
+    ./correct_counts.py -i "/tmp/mirrulations_$START_TIME.json" -o "/tmp/mirrulations_${START_TIME}_corrected.json" 2>> $LOG_FILE &&
+    ./set_counts.py -y -i "/tmp/mirrulations_${START_TIME}_corrected.json" 2>> $LOG_FILE
 
-rm "/tmp/mirrulations_${START_TIME}_corrected.json" "/tmp/mirrulations_$START_TIME.json"
+rm -f "/tmp/mirrulations_${START_TIME}_corrected.json" "/tmp/mirrulations_$START_TIME.json"

scripts/get_counts.py

@@ -8,6 +8,7 @@
 import sys
 from counts import Counts, CountsEncoder
 from job_queue import RabbitMQ
+from typing import Dict
 
 import redis
 import requests
@@ -20,7 +21,7 @@ class MissingRedisKeyException(Exception):
 
 
 def _download_regulation_count(
-    url: str, headers: dict[str, str], params: dict[str, str]
+    url: str, headers: Dict[str, str], params: Dict[str, str]
 ) -> int:
     response = requests.get(
         url,
@@ -229,7 +230,7 @@ def get_redis(db: redis.Redis) -> Counts:
     if source == "regulations":
         api_key = args.api_key
         if api_key is None or api_key == "":
-            print("No api key found, exitting", file=sys.stderr)
+            print("No api key found, exiting", file=sys.stderr)
             sys.exit(1)
         output = get_regulation(api_key, args.last_timestamp)
     elif source == "dashboard":
@@ -241,10 +242,10 @@ def get_redis(db: redis.Redis) -> Counts:
         try:
             output = get_redis(db)
         except MissingRedisKeyException as e:
-            print(f"Missing a redis key, exitting\n{e}", file=sys.stderr)
+            print(f"Missing a redis key, exiting\n{e}", file=sys.stderr)
             sys.exit(1)
     else:
-        print("Unrecognized source, exitting", file=sys.stderr)
+        print("Unrecognized source, exiting", file=sys.stderr)
         sys.exit(1)
 
     if args.output == "-":

scripts/set_counts.py

@@ -15,7 +15,7 @@
 ANSI_FG_RED = "\033[31m"
 
 
-def _get_vals(db: redis.Redis, entity_type: str) -> dict[str, int | str]:
+def _get_vals(db: redis.Redis, entity_type: str) -> dict:
     done_raw: str | None = db.get(f"num_{entity_type}_done")
     if done_raw is not None:
         done = int(done_raw)
@@ -77,7 +77,7 @@ def set_values(db: redis.Redis, counts: Counts):
             )
         except Exception as e:
             print(
-                f"Error occurred while setting values for {entity_type}, exitting",
+                f"Error occurred while setting values for {entity_type}, exiting",
                 file=sys.stderr,
             )
             print(e)
@@ -137,11 +137,11 @@ def set_values(db: redis.Redis, counts: Counts):
                     input_counts = json.load(fp, cls=CountsDecoder)
             except FileNotFoundError:
                 print(
-                    f"Input file {args.input} does not exist, exitting", file=sys.stderr
+                    f"Input file {args.input} does not exist, exiting", file=sys.stderr
                 )
                 sys.exit(2)
     except json.JSONDecodeError:
-        print(f"Malformed input file {args.input}, exitting", file=sys.stderr)
+        print(f"Malformed input file {args.input}, exiting", file=sys.stderr)
         sys.exit(2)
 
     db = redis.Redis(args.host, args.port, args.db, decode_responses=True)
@@ -160,5 +160,5 @@ def set_values(db: redis.Redis, counts: Counts):
         if changes_confirmed:
             set_values(db, input_counts)
         else:
-            print("No values set, exitting")
+            print("No values set, exiting")
             sys.exit()