diff --git a/api/chalicelib/core/metrics.py b/api/chalicelib/core/metrics.py index a012bbe146..b1caf75098 100644 --- a/api/chalicelib/core/metrics.py +++ b/api/chalicelib/core/metrics.py @@ -1,3 +1,4 @@ +import logging import math import schemas @@ -8,6 +9,8 @@ from chalicelib.utils.TimeUTC import TimeUTC from chalicelib.utils.metrics_helper import __get_step_size +logger = logging.getLogger(__name__) + # Written by David Aznaurov, inspired by numpy.quantile def __quantiles(a, q, interpolation='higher'): @@ -1716,25 +1719,41 @@ def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return {"value": avg, "chart": rows, "unit": schemas.TemplatePredefinedUnits.MILLISECOND} -def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), +def get_errors_per_domains(project_id, limit, page, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): pg_sub_query = __get_constraints(project_id=project_id, data=args) pg_sub_query.append("requests.success = FALSE") + params = {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + "limit_s": (page - 1) * limit, + "limit_e": page * limit, + **__get_constraint_values(args)} with pg_client.PostgresClient() as cur: - pg_query = f"""SELECT - requests.host AS domain, - COUNT(requests.session_id) AS errors_count - FROM events_common.requests INNER JOIN sessions USING (session_id) - WHERE {" AND ".join(pg_sub_query)} - GROUP BY requests.host - ORDER BY errors_count DESC - LIMIT 5;""" - cur.execute(cur.mogrify(pg_query, {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)})) - rows = cur.fetchall() - return helper.list_to_camel_case(rows) + pg_query = f"""SELECT SUM(errors_count) AS count, + COUNT(raw.domain) AS total, + jsonb_agg(raw) FILTER ( WHERE rn > %(limit_s)s + AND rn <= %(limit_e)s ) AS values + FROM (SELECT requests.host AS domain, + COUNT(requests.session_id) AS errors_count, + row_number() over (ORDER BY COUNT(requests.session_id) DESC ) AS rn + FROM events_common.requests + INNER JOIN sessions USING (session_id) + WHERE {" AND ".join(pg_sub_query)} + GROUP BY requests.host + ORDER BY errors_count DESC) AS raw;""" + pg_query = cur.mogrify(pg_query, params) + logger.debug("-----------") + logger.debug(pg_query) + logger.debug("-----------") + cur.execute(pg_query) + row = cur.fetchone() + if row: + for r in row["values"]: + r.pop("rn") + + return helper.dict_to_camel_case(row) def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), diff --git a/ee/api/chalicelib/core/metrics.py b/ee/api/chalicelib/core/metrics.py index 3d27a6081a..2ec25e4356 100644 --- a/ee/api/chalicelib/core/metrics.py +++ b/ee/api/chalicelib/core/metrics.py @@ -1,14 +1,17 @@ +import logging import math +from math import isnan import schemas -from chalicelib.utils import exp_ch_helper from chalicelib.utils import args_transformer +from chalicelib.utils import ch_client +from chalicelib.utils import exp_ch_helper from chalicelib.utils import helper from chalicelib.utils.TimeUTC import TimeUTC -from chalicelib.utils import ch_client -from math import isnan from chalicelib.utils.metrics_helper import __get_step_size +logger = logging.getLogger(__name__) + def __get_basic_constraints(table_name=None, time_constraint=True, round_start=False, data={}, identifier="project_id"): if table_name: @@ -1655,28 +1658,44 @@ def get_slowest_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), return {"value": avg, "chart": rows, "unit": schemas.TemplatePredefinedUnits.MILLISECOND} -def get_errors_per_domains(project_id, startTimestamp=TimeUTC.now(delta_days=-1), +def get_errors_per_domains(project_id, limit, page, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), **args): ch_sub_query = __get_basic_constraints(table_name="requests", data=args) ch_sub_query.append("requests.event_type = 'REQUEST'") ch_sub_query.append("requests.success = 0") meta_condition = __get_meta_constraint(args) ch_sub_query += meta_condition - + params = {"project_id": project_id, + "startTimestamp": startTimestamp, + "endTimestamp": endTimestamp, + **__get_constraint_values(args), + "limit_s": (page - 1) * limit, + "limit": limit} with ch_client.ClickHouseClient() as ch: ch_query = f"""SELECT requests.url_host AS domain, - COUNT(1) AS errors_count + COUNT(1) AS errors_count, + COUNT(1) OVER () AS total, + SUM(errors_count) OVER () AS count FROM {exp_ch_helper.get_main_events_table(startTimestamp)} AS requests WHERE {" AND ".join(ch_sub_query)} GROUP BY requests.url_host ORDER BY errors_count DESC - LIMIT 5;""" - params = {"project_id": project_id, - "startTimestamp": startTimestamp, - "endTimestamp": endTimestamp, **__get_constraint_values(args)} + LIMIT %(limit)s OFFSET %(limit_s)s;""" + logger.debug("-----------") + logger.debug(ch.format(query=ch_query, params=params)) + logger.debug("-----------") rows = ch.execute(query=ch_query, params=params) - return helper.list_to_camel_case(rows) + response = {"count": 0, "total": 0, "values": []} + if len(rows) > 0: + response["count"] = rows[0]["count"] + response["total"] = rows[0]["total"] + rows = helper.list_to_camel_case(rows) + for r in rows: + r.pop("count") + r.pop("total") + + return response def get_sessions_per_browser(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp=TimeUTC.now(), @@ -2801,8 +2820,8 @@ def get_top_metrics_avg_time_to_interactive(project_id, startTimestamp=TimeUTC.n def get_unique_users(project_id, startTimestamp=TimeUTC.now(delta_days=-1), - endTimestamp=TimeUTC.now(), - density=7, **args): + endTimestamp=TimeUTC.now(), + density=7, **args): step_size = __get_step_size(startTimestamp, endTimestamp, density) ch_sub_query = __get_basic_constraints(table_name="sessions", data=args) ch_sub_query_chart = __get_basic_constraints(table_name="sessions", round_start=True, data=args)