Improve logging

scripts/correct_counts.py

@@ -4,6 +4,7 @@
 import json
 import pathlib
 import sys
+from json import JSONDecodeError
 from counts import Counts, CountsEncoder, CountsDecoder
 
 import argparse
@@ -15,13 +16,11 @@ class JobsInQueueException(Exception):
 
 def strategy_cap(recieved: Counts, ignore_queue: bool) -> Counts:
     filtered = deepcopy(recieved)
+    if filtered["queue_size"] != 0 and not ignore_queue:
+        raise JobsInQueueException(f'Found jobs in job queue: {filtered["queue_size"]}')
     for entity_type in ("dockets", "documents", "comments"):
         total_ = filtered[entity_type]["total"]
         downloaded = filtered[entity_type]["downloaded"]
-        if filtered["queue_size"] != 0 and not ignore_queue:
-            raise JobsInQueueException(
-                f'{entity_type} has {filtered[entity_type]["jobs"]} in queue'
-            )
         filtered[entity_type]["downloaded"] = min(total_, downloaded)
 
     return filtered
@@ -79,11 +78,19 @@ def strategy_diff(recieved: Counts, ignore_queue: bool) -> Counts:
 
     args = parser.parse_args()
 
-    if args.input == "-":
-        input_counts: Counts = json.load(sys.stdin, cls=CountsDecoder)
-    else:
-        with open(pathlib.Path(args.input), "r") as fp:
-            input_counts = json.load(fp, cls=CountsDecoder)
+    try:
+        if args.input == "-":
+            input_counts: Counts = json.load(sys.stdin, cls=CountsDecoder)
+        else:
+            try:
+                with open(pathlib.Path(args.input), "r") as fp:
+                    input_counts = json.load(fp, cls=CountsDecoder)
+            except FileNotFoundError:
+                print(f"Missing file {args.input}, exitting", file=sys.stderr)
+                sys.exit(2)
+    except JSONDecodeError:
+        print(f"Malformed input file {args.input}, exitting", file=sys.stderr)
+        sys.exit(2)
 
     try:
         if args.strategy == "cap_with_total":

scripts/get_counts.py

@@ -127,7 +127,7 @@ def get_redis(db: redis.Redis) -> Counts:
         "creation_timestamp": dt.datetime.now(dt.timezone.utc),
     }
     queue = RabbitMQ("jobs_waiting_queue")
-    counts["queue_size"] = queue
+    counts["queue_size"] = queue.size()
 
     for entity_type in ("dockets", "documents", "comments"):
         # Getting any of these values can raise an exception

scripts/job_queue.py

@@ -19,12 +19,12 @@ def __init__(self, queue_name):
 
     def _ensure_channel(self):
         if self.connection is None or not self.connection.is_open:
-            connection_parameter = pika.ConnectionParameters("rabbitmq")
+            connection_parameter = pika.ConnectionParameters("localhost")
             self.connection = pika.BlockingConnection(connection_parameter)
             self.channel = self.connection.channel()
             self.channel.queue_declare(self.queue_name, durable=True)
 
-    def size(self):
+    def size(self) -> int:
         """
         Get the number of jobs in the queue.
         Can't be sure Channel is active between ensure_channel()
@@ -38,3 +38,4 @@ def size(self):
             return queue.method.message_count
         except pika.exceptions.StreamLostError:
             print("FAILURE: RabbitMQ Channel Connection Lost", file=sys.stderr)
+            return 0

scripts/set_counts.py

@@ -76,7 +76,10 @@ def set_values(db: redis.Redis, counts: Counts):
                 counts[entity_type]["last_timestamp"].strftime("%Y-%m-%d %H:%M:%S"),
             )
         except Exception as e:
-            print(f"Error occurred while setting values for {entity_type}, exitting", file=sys.stderr)
+            print(
+                f"Error occurred while setting values for {entity_type}, exitting",
+                file=sys.stderr,
+            )
             print(e)
             return
 
@@ -125,11 +128,21 @@ def set_values(db: redis.Redis, counts: Counts):
 
     args = parser.parse_args()
 
-    if args.input == "-":
-        input_counts: Counts = json.load(sys.stdin, cls=CountsDecoder)
-    else:
-        with open(pathlib.Path(args.input), "r") as fp:
-            input_counts = json.load(fp, cls=CountsDecoder)
+    try:
+        if args.input == "-":
+            input_counts: Counts = json.load(sys.stdin, cls=CountsDecoder)
+        else:
+            try:
+                with open(pathlib.Path(args.input), "r") as fp:
+                    input_counts = json.load(fp, cls=CountsDecoder)
+            except FileNotFoundError:
+                print(
+                    f"Input file {args.input} does not exist, exitting", file=sys.stderr
+                )
+                sys.exit(2)
+    except json.JSONDecodeError:
+        print(f"Malformed input file {args.input}, exitting", file=sys.stderr)
+        sys.exit(2)
 
     db = redis.Redis(args.host, args.port, args.db, decode_responses=True)
     changes_confirmed: bool = args.changes_confirmed