feat(chalice): support pagination for table of errors-per-domain

tahayk · Aug 14, 2024 · deb3e8e · deb3e8e
1 parent dcdd47c
commit deb3e8e
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 27 deletions.
diff --git a/api/chalicelib/core/metrics.py b/api/chalicelib/core/metrics.py
@@ -1,3 +1,4 @@
+import logging
 import math
 
 import schemas
@@ -8,6 +9,8 @@
 from chalicelib.utils.TimeUTC import TimeUTC
 from chalicelib.utils.metrics_helper import __get_step_size
 
+logger = logging.getLogger(__name__)
+
 
 # Written by David Aznaurov, inspired by numpy.quantile
 def __quantiles(a, q, interpolation='higher'):
@@ -1716,25 +1719,41 @@ def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
     return {"value": avg, "chart": rows, "unit": schemas.TemplatePredefinedUnits.MILLISECOND}
 
 
-def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
+def get_errors_per_domains(project_id, limit, page, startTimestamp=TimeUTC.now(delta_days=-1),
                            endTimestamp=TimeUTC.now(), **args):
     pg_sub_query = __get_constraints(project_id=project_id, data=args)
     pg_sub_query.append("requests.success = FALSE")
+    params = {"project_id": project_id,
+              "startTimestamp": startTimestamp,
+              "endTimestamp": endTimestamp,
+              "limit_s": (page - 1) * limit,
+              "limit_e": page * limit,
+              **__get_constraint_values(args)}
 
     with pg_client.PostgresClient() as cur:
-        pg_query = f"""SELECT
-                            requests.host AS domain,
-                            COUNT(requests.session_id) AS errors_count
-                        FROM events_common.requests INNER JOIN sessions USING (session_id)
-                        WHERE {" AND ".join(pg_sub_query)}
-                        GROUP BY requests.host
-                        ORDER BY errors_count DESC
-                        LIMIT 5;"""
-        cur.execute(cur.mogrify(pg_query, {"project_id": project_id,
-                                           "startTimestamp": startTimestamp,
-                                           "endTimestamp": endTimestamp, **__get_constraint_values(args)}))
-        rows = cur.fetchall()
-    return helper.list_to_camel_case(rows)
+        pg_query = f"""SELECT SUM(errors_count) AS count,
+                              COUNT(raw.domain) AS total,
+                              jsonb_agg(raw) FILTER ( WHERE rn > %(limit_s)s 
+                                                        AND rn <= %(limit_e)s ) AS values
+                        FROM (SELECT requests.host                                                 AS domain,
+                                     COUNT(requests.session_id)                                    AS errors_count,
+                                     row_number() over (ORDER BY COUNT(requests.session_id) DESC ) AS rn
+                              FROM events_common.requests
+                                       INNER JOIN sessions USING (session_id)
+                              WHERE {" AND ".join(pg_sub_query)}
+                              GROUP BY requests.host
+                              ORDER BY errors_count DESC) AS raw;"""
+        pg_query = cur.mogrify(pg_query, params)
+        logger.debug("-----------")
+        logger.debug(pg_query)
+        logger.debug("-----------")
+        cur.execute(pg_query)
+        row = cur.fetchone()
+        if row:
+            for r in row["values"]:
+                r.pop("rn")
+
+    return helper.dict_to_camel_case(row)
 
 
 def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),

diff --git a/ee/api/chalicelib/core/metrics.py b/ee/api/chalicelib/core/metrics.py
@@ -1,14 +1,17 @@
+import logging
 import math
+from math import isnan
 
 import schemas
-from chalicelib.utils import exp_ch_helper
 from chalicelib.utils import args_transformer
+from chalicelib.utils import ch_client
+from chalicelib.utils import exp_ch_helper
 from chalicelib.utils import helper
 from chalicelib.utils.TimeUTC import TimeUTC
-from chalicelib.utils import ch_client
-from math import isnan
 from chalicelib.utils.metrics_helper import __get_step_size
 
+logger = logging.getLogger(__name__)
+
 
 def __get_basic_constraints(table_name=None, time_constraint=True, round_start=False, data={}, identifier="project_id"):
     if table_name:
@@ -1655,28 +1658,44 @@ def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
     return {"value": avg, "chart": rows, "unit": schemas.TemplatePredefinedUnits.MILLISECOND}
 
 
-def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
+def get_errors_per_domains(project_id, limit, page, startTimestamp=TimeUTC.now(delta_days=-1),
                            endTimestamp=TimeUTC.now(), **args):
     ch_sub_query = __get_basic_constraints(table_name="requests", data=args)
     ch_sub_query.append("requests.event_type = 'REQUEST'")
     ch_sub_query.append("requests.success = 0")
     meta_condition = __get_meta_constraint(args)
     ch_sub_query += meta_condition
-
+    params = {"project_id": project_id,
+              "startTimestamp": startTimestamp,
+              "endTimestamp": endTimestamp,
+              **__get_constraint_values(args),
+              "limit_s": (page - 1) * limit,
+              "limit": limit}
     with ch_client.ClickHouseClient() as ch:
         ch_query = f"""SELECT
                             requests.url_host AS domain,
-                            COUNT(1) AS errors_count
+                            COUNT(1) AS errors_count,
+                            COUNT(1) OVER ()          AS total,
+                            SUM(errors_count) OVER () AS count
                         FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS requests
                         WHERE {" AND ".join(ch_sub_query)}
                         GROUP BY requests.url_host
                         ORDER BY errors_count DESC
-                        LIMIT 5;"""
-        params = {"project_id": project_id,
-                  "startTimestamp": startTimestamp,
-                  "endTimestamp": endTimestamp, **__get_constraint_values(args)}
+                        LIMIT %(limit)s OFFSET %(limit_s)s;"""
+        logger.debug("-----------")
+        logger.debug(ch.format(query=ch_query, params=params))
+        logger.debug("-----------")
         rows = ch.execute(query=ch_query, params=params)
-    return helper.list_to_camel_case(rows)
+        response = {"count": 0, "total": 0, "values": []}
+        if len(rows) > 0:
+            response["count"] = rows[0]["count"]
+            response["total"] = rows[0]["total"]
+            rows = helper.list_to_camel_case(rows)
+            for r in rows:
+                r.pop("count")
+                r.pop("total")
+
+    return response
 
 
 def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(),
@@ -2801,8 +2820,8 @@ def get_top_metrics_avg_time_to_interactive(project_id, startTimestamp=TimeUTC.n
 
 
 def get_unique_users(project_id, startTimestamp=TimeUTC.now(delta_days=-1),
-                           endTimestamp=TimeUTC.now(),
-                           density=7, **args):
+                     endTimestamp=TimeUTC.now(),
+                     density=7, **args):
     step_size = __get_step_size(startTimestamp, endTimestamp, density)
     ch_sub_query = __get_basic_constraints(table_name="sessions", data=args)
     ch_sub_query_chart = __get_basic_constraints(table_name="sessions", round_start=True, data=args)