From 44dc50813cae3f51659190eb94441b4e2a85dd0f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 19 Jun 2024 18:48:14 +0200 Subject: [PATCH] refactor(chalice): upgraded dependencies feat(chalice): support heatmaps --- api/Pipfile | 8 +- api/chalicelib/core/__init__.py | 1 - api/chalicelib/core/click_maps.py | 80 ------- api/chalicelib/core/custom_metrics.py | 5 +- api/chalicelib/core/heatmaps.py | 148 ++++++++++--- api/requirements-alerts.txt | 2 +- api/requirements.txt | 2 +- api/routers/core_dynamic.py | 2 +- api/schemas/schemas.py | 2 + ee/api/.gitignore | 2 - ee/api/Pipfile | 10 +- ee/api/chalicelib/core/custom_metrics.py | 11 +- ee/api/chalicelib/core/heatmaps.py | 256 +++++++++++++++++++++++ ee/api/clean-dev.sh | 2 - ee/api/requirements-alerts.txt | 2 +- ee/api/requirements-crons.txt | 2 +- ee/api/requirements.txt | 2 +- ee/api/routers/core_dynamic.py | 4 +- 18 files changed, 400 insertions(+), 141 deletions(-) delete mode 100644 api/chalicelib/core/click_maps.py create mode 100644 ee/api/chalicelib/core/heatmaps.py diff --git a/api/Pipfile b/api/Pipfile index 78fd673631..da61d68932 100644 --- a/api/Pipfile +++ b/api/Pipfile @@ -9,15 +9,15 @@ requests = "==2.32.3" boto3 = "==1.34.125" pyjwt = "==2.8.0" psycopg2-binary = "==2.9.9" +psycopg = {extras = ["binary", "pool"], version = "==3.1.19"} elasticsearch = "==8.14.0" jira = "==3.8.0" fastapi = "==0.111.0" -python-decouple = "==3.8" -apscheduler = "==4.0.0a5" -redis = "==5.1.0b6" -psycopg = {extras = ["binary", "pool"], version = "==3.1.19"} uvicorn = {extras = ["standard"], version = "==0.30.1"} +python-decouple = "==3.8" pydantic = {extras = ["email"], version = "==2.3.0"} +apscheduler = "==3.10.4" +redis = "==5.1.0b6" [dev-packages] diff --git a/api/chalicelib/core/__init__.py b/api/chalicelib/core/__init__.py index f5667e079e..e69de29bb2 100644 --- a/api/chalicelib/core/__init__.py +++ b/api/chalicelib/core/__init__.py @@ -1 +0,0 @@ -from . import sessions as sessions_legacy \ No newline at end of file diff --git a/api/chalicelib/core/click_maps.py b/api/chalicelib/core/click_maps.py deleted file mode 100644 index 6cf30d3c3e..0000000000 --- a/api/chalicelib/core/click_maps.py +++ /dev/null @@ -1,80 +0,0 @@ -import logging - -import schemas -from chalicelib.core import sessions_mobs, sessions_legacy as sessions_search, events -from chalicelib.utils import pg_client, helper - -logger = logging.getLogger(__name__) -SESSION_PROJECTION_COLS = """s.project_id, -s.session_id::text AS session_id, -s.user_uuid, -s.user_id, -s.user_os, -s.user_browser, -s.user_device, -s.user_device_type, -s.user_country, -s.start_ts, -s.duration, -s.events_count, -s.pages_count, -s.errors_count, -s.user_anonymous_id, -s.platform, -s.issue_score, -to_jsonb(s.issue_types) AS issue_types, -favorite_sessions.session_id NOTNULL AS favorite, -COALESCE((SELECT TRUE - FROM public.user_viewed_sessions AS fs - WHERE s.session_id = fs.session_id - AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed """ - - -def search_short_session(data: schemas.ClickMapSessionsSearch, project_id, user_id, include_mobs: bool = True): - no_platform = True - for f in data.filters: - if f.type == schemas.FilterType.platform: - no_platform = False - break - if no_platform: - data.filters.append(schemas.SessionSearchFilterSchema(type=schemas.FilterType.platform, - value=[schemas.PlatformType.desktop], - operator=schemas.SearchEventOperator._is)) - - full_args, query_part = sessions_search.search_query_parts(data=data, error_status=None, errors_only=False, - favorite_only=data.bookmarked, issue=None, - project_id=project_id, user_id=user_id) - - with pg_client.PostgresClient() as cur: - data.order = schemas.SortOrderType.desc - data.sort = 'duration' - - # meta_keys = metadata.get(project_id=project_id) - meta_keys = [] - main_query = cur.mogrify(f"""SELECT {SESSION_PROJECTION_COLS} - {"," if len(meta_keys) > 0 else ""}{",".join([f'metadata_{m["index"]}' for m in meta_keys])} - {query_part} - ORDER BY {data.sort} {data.order.value} - LIMIT 1;""", full_args) - logger.debug("--------------------") - logger.debug(main_query) - logger.debug("--------------------") - try: - cur.execute(main_query) - except Exception as err: - logger.warning("--------- CLICK MAP SHORT SESSION SEARCH QUERY EXCEPTION -----------") - logger.warning(main_query.decode('UTF-8')) - logger.warning("--------- PAYLOAD -----------") - logger.warning(data.model_dump_json()) - logger.warning("--------------------") - raise err - - session = cur.fetchone() - if session: - if include_mobs: - session['domURL'] = sessions_mobs.get_urls(session_id=session["session_id"], project_id=project_id) - session['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session["session_id"]) - session['events'] = events.get_by_session_id(project_id=project_id, session_id=session["session_id"], - event_type=schemas.EventType.location) - - return helper.dict_to_camel_case(session) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 2a2a639d68..6b9caf2144 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -5,7 +5,7 @@ from fastapi import HTTPException, status import schemas -from chalicelib.core import sessions, funnels, errors, issues, click_maps, sessions_mobs, product_analytics, \ +from chalicelib.core import sessions, funnels, errors, issues, heatmaps, sessions_mobs, product_analytics, \ custom_metrics_predefined from chalicelib.utils import helper, pg_client from chalicelib.utils.TimeUTC import TimeUTC @@ -90,7 +90,7 @@ def __get_click_map_chart(project_id, user_id, data: schemas.CardClickMap, inclu return None data.series[0].filter.filters += data.series[0].filter.events data.series[0].filter.events = [] - return click_maps.search_short_session(project_id=project_id, user_id=user_id, + return heatmaps.search_short_session(project_id=project_id, user_id=user_id, data=schemas.ClickMapSessionsSearch( **data.series[0].filter.model_dump()), include_mobs=include_mobs) @@ -178,6 +178,7 @@ def get_chart(project_id: int, data: schemas.CardSchema, user_id: int): schemas.MetricType.timeseries: __get_timeseries_chart, schemas.MetricType.table: __get_table_chart, schemas.MetricType.click_map: __get_click_map_chart, + schemas.MetricType.heat_map: __get_click_map_chart, schemas.MetricType.funnel: __get_funnel_chart, schemas.MetricType.insights: not_supported, schemas.MetricType.pathAnalysis: __get_path_analysis_chart diff --git a/api/chalicelib/core/heatmaps.py b/api/chalicelib/core/heatmaps.py index 899d648eff..17d6f8b00a 100644 --- a/api/chalicelib/core/heatmaps.py +++ b/api/chalicelib/core/heatmaps.py @@ -1,8 +1,10 @@ import logging import schemas -from chalicelib.utils import helper, pg_client -from chalicelib.utils import sql_helper as sh +from chalicelib.core import sessions_mobs, sessions, events +from chalicelib.utils import pg_client, helper + +# from chalicelib.utils import sql_helper as sh logger = logging.getLogger(__name__) @@ -19,42 +21,41 @@ def get_by_url(project_id, data: schemas.GetHeatmapPayloadSchema): "duration IS NOT NULL", "normalized_x IS NOT NULL"] query_from = "events.clicks INNER JOIN sessions USING (session_id)" - q_count = "count(1) AS count" has_click_rage_filter = False - if len(data.filters) > 0: - for i, f in enumerate(data.filters): - if f.type == schemas.FilterType.issue and len(f.value) > 0: - has_click_rage_filter = True - q_count = "max(real_count) AS count,TRUE AS click_rage" - query_from += """INNER JOIN events_common.issues USING (timestamp, session_id) - INNER JOIN issues AS mis USING (issue_id) - INNER JOIN LATERAL ( - SELECT COUNT(1) AS real_count - FROM events.clicks AS sc - INNER JOIN sessions as ss USING (session_id) - WHERE ss.project_id = 2 - AND (sc.url = %(url)s OR sc.path = %(url)s) - AND sc.timestamp >= %(startDate)s - AND sc.timestamp <= %(endDate)s - AND ss.start_ts >= %(startDate)s - AND ss.start_ts <= %(endDate)s - AND sc.selector = clicks.selector) AS r_clicks ON (TRUE)""" - constraints += ["mis.project_id = %(project_id)s", - "issues.timestamp >= %(startDate)s", - "issues.timestamp <= %(endDate)s"] - f_k = f"issue_value{i}" - args = {**args, **sh.multi_values(f.value, value_key=f_k)} - constraints.append(sh.multi_conditions(f"%({f_k})s = ANY (issue_types)", - f.value, value_key=f_k)) - constraints.append(sh.multi_conditions(f"mis.type = %({f_k})s", - f.value, value_key=f_k)) + # TODO: is this used ? + # if len(data.filters) > 0: + # for i, f in enumerate(data.filters): + # if f.type == schemas.FilterType.issue and len(f.value) > 0: + # has_click_rage_filter = True + # query_from += """INNER JOIN events_common.issues USING (timestamp, session_id) + # INNER JOIN issues AS mis USING (issue_id) + # INNER JOIN LATERAL ( + # SELECT COUNT(1) AS real_count + # FROM events.clicks AS sc + # INNER JOIN sessions as ss USING (session_id) + # WHERE ss.project_id = 2 + # AND (sc.url = %(url)s OR sc.path = %(url)s) + # AND sc.timestamp >= %(startDate)s + # AND sc.timestamp <= %(endDate)s + # AND ss.start_ts >= %(startDate)s + # AND ss.start_ts <= %(endDate)s + # AND sc.selector = clicks.selector) AS r_clicks ON (TRUE)""" + # constraints += ["mis.project_id = %(project_id)s", + # "issues.timestamp >= %(startDate)s", + # "issues.timestamp <= %(endDate)s"] + # f_k = f"issue_value{i}" + # args = {**args, **sh.multi_values(f.value, value_key=f_k)} + # constraints.append(sh.multi_conditions(f"%({f_k})s = ANY (issue_types)", + # f.value, value_key=f_k)) + # constraints.append(sh.multi_conditions(f"mis.type = %({f_k})s", + # f.value, value_key=f_k)) if data.click_rage and not has_click_rage_filter: constraints.append("""(issues.session_id IS NULL OR (issues.timestamp >= %(startDate)s AND issues.timestamp <= %(endDate)s - AND mis.project_id = %(project_id)s))""") - q_count += ",COALESCE(bool_or(mis.type = 'click_rage'), FALSE) AS click_rage" + AND mis.project_id = %(project_id)s + AND mis.type='click_rage'))""") query_from += """LEFT JOIN events_common.issues USING (timestamp, session_id) LEFT JOIN issues AS mis USING (issue_id)""" with pg_client.PostgresClient() as cur: @@ -77,3 +78,86 @@ def get_by_url(project_id, data: schemas.GetHeatmapPayloadSchema): rows = cur.fetchall() return helper.list_to_camel_case(rows) + + +SESSION_PROJECTION_COLS = """s.project_id, +s.session_id::text AS session_id, +s.user_uuid, +s.user_id, +s.user_os, +s.user_browser, +s.user_device, +s.user_device_type, +s.user_country, +s.start_ts, +s.duration, +s.events_count, +s.pages_count, +s.errors_count, +s.user_anonymous_id, +s.platform, +s.issue_score, +to_jsonb(s.issue_types) AS issue_types, +favorite_sessions.session_id NOTNULL AS favorite, +COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed """ + + +def search_short_session(data: schemas.ClickMapSessionsSearch, project_id, user_id, + include_mobs: bool = True, exclude_sessions: list[str] = [], + _depth: int = 3): + no_platform = True + for f in data.filters: + if f.type == schemas.FilterType.platform: + no_platform = False + break + if no_platform: + data.filters.append(schemas.SessionSearchFilterSchema(type=schemas.FilterType.platform, + value=[schemas.PlatformType.desktop], + operator=schemas.SearchEventOperator._is)) + + full_args, query_part = sessions.search_query_parts(data=data, error_status=None, errors_only=False, + favorite_only=data.bookmarked, issue=None, + project_id=project_id, user_id=user_id) + full_args["exclude_sessions"] = tuple(exclude_sessions) + if len(exclude_sessions) > 0: + query_part += "\n AND session_id NOT IN (%(exclude_sessions)s)" + with pg_client.PostgresClient() as cur: + data.order = schemas.SortOrderType.desc + data.sort = 'duration' + main_query = cur.mogrify(f"""SELECT {SESSION_PROJECTION_COLS} + {query_part} + ORDER BY {data.sort} {data.order.value} + LIMIT 1;""", full_args) + logger.debug("--------------------") + logger.debug(main_query) + logger.debug("--------------------") + try: + cur.execute(main_query) + except Exception as err: + logger.warning("--------- CLICK MAP SHORT SESSION SEARCH QUERY EXCEPTION -----------") + logger.warning(main_query.decode('UTF-8')) + logger.warning("--------- PAYLOAD -----------") + logger.warning(data.model_dump_json()) + logger.warning("--------------------") + raise err + + session = cur.fetchone() + if session: + if include_mobs: + session['domURL'] = sessions_mobs.get_urls(session_id=session["session_id"], project_id=project_id) + session['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session["session_id"]) + if _depth > 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0: + return search_short_session(data=data, project_id=project_id, user_id=user_id, + include_mobs=include_mobs, + exclude_sessions=exclude_sessions + [session["session_id"]], + _depth=_depth - 1) + elif _depth == 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0: + logger.info("couldn't find an existing replay after 3 iterations for heatmap") + + session['events'] = events.get_by_session_id(project_id=project_id, session_id=session["session_id"], + event_type=schemas.EventType.location) + + return helper.dict_to_camel_case(session) diff --git a/api/requirements-alerts.txt b/api/requirements-alerts.txt index 9252e979d8..5f8958cab7 100644 --- a/api/requirements-alerts.txt +++ b/api/requirements-alerts.txt @@ -14,4 +14,4 @@ fastapi==0.111.0 uvicorn[standard]==0.30.1 python-decouple==3.8 pydantic[email]==2.3.0 -apscheduler==4.0.0a5 +apscheduler==3.10.4 diff --git a/api/requirements.txt b/api/requirements.txt index 012727b825..0864398bba 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -14,6 +14,6 @@ fastapi==0.111.0 uvicorn[standard]==0.30.1 python-decouple==3.8 pydantic[email]==2.3.0 -apscheduler==4.0.0a5 +apscheduler==3.10.4 redis==5.1.0b6 diff --git a/api/routers/core_dynamic.py b/api/routers/core_dynamic.py index 8478c9ea6c..b61fc54f7f 100644 --- a/api/routers/core_dynamic.py +++ b/api/routers/core_dynamic.py @@ -7,7 +7,7 @@ import schemas from chalicelib.core import sessions, errors, errors_viewed, errors_favorite, sessions_assignments, heatmaps, \ - sessions_favorite, assist, sessions_notes, click_maps, sessions_replay, signup, feature_flags + sessions_favorite, assist, sessions_notes, sessions_replay, signup, feature_flags from chalicelib.core import sessions_viewed from chalicelib.core import tenants, users, projects, license from chalicelib.core import webhook diff --git a/api/schemas/schemas.py b/api/schemas/schemas.py index fa02b0c33f..bcee02924c 100644 --- a/api/schemas/schemas.py +++ b/api/schemas/schemas.py @@ -934,6 +934,8 @@ class MetricType(str, Enum): retention = "retention" stickiness = "stickiness" click_map = "clickMap" + # click_map and heat_map are the same + heat_map = "heatMap" insights = "insights" diff --git a/ee/api/.gitignore b/ee/api/.gitignore index b5b8dc4141..08c0c48998 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -190,7 +190,6 @@ Pipfile.lock /chalicelib/core/authorizers.py /chalicelib/core/autocomplete.py /chalicelib/core/canvas.py -/chalicelib/core/click_maps.py /chalicelib/core/collaboration_base.py /chalicelib/core/collaboration_msteams.py /chalicelib/core/collaboration_slack.py @@ -201,7 +200,6 @@ Pipfile.lock /chalicelib/core/events_mobile.py /chalicelib/core/feature_flags.py /chalicelib/core/funnels.py -/chalicelib/core/heatmaps.py /chalicelib/core/integration_base.py /chalicelib/core/integration_base_issue.py /chalicelib/core/integration_github.py diff --git a/ee/api/Pipfile b/ee/api/Pipfile index 552cbd1e3f..66e1f563d4 100644 --- a/ee/api/Pipfile +++ b/ee/api/Pipfile @@ -9,19 +9,19 @@ requests = "==2.32.3" boto3 = "==1.34.125" pyjwt = "==2.8.0" psycopg2-binary = "==2.9.9" +psycopg = {extras = ["binary", "pool"], version = "==3.1.19"} elasticsearch = "==8.14.0" jira = "==3.8.0" fastapi = "==0.111.0" +uvicorn = {extras = ["standard"], version = "==0.30.1"} gunicorn = "==22.0.0" python-decouple = "==3.8" -apscheduler = "==4.0.0a5" +pydantic = {extras = ["email"], version = "==2.3.0"} +apscheduler = "==3.10.4" +clickhouse-driver = {extras = ["lz4"], version = "==0.2.8"} python3-saml = "==1.16.0" redis = "==5.1.0b6" azure-storage-blob = "==12.21.0b1" -psycopg = {extras = ["binary", "pool"], version = "==3.1.19"} -uvicorn = {extras = ["standard"], version = "==0.30.1"} -pydantic = {extras = ["email"], version = "==2.3.0"} -clickhouse-driver = {extras = ["lz4"], version = "==0.2.8"} [dev-packages] diff --git a/ee/api/chalicelib/core/custom_metrics.py b/ee/api/chalicelib/core/custom_metrics.py index 3a1086a599..9a546afd1b 100644 --- a/ee/api/chalicelib/core/custom_metrics.py +++ b/ee/api/chalicelib/core/custom_metrics.py @@ -5,7 +5,7 @@ from fastapi import HTTPException, status import schemas -from chalicelib.core import funnels, issues, click_maps, sessions_insights, sessions_mobs, sessions_favorite, \ +from chalicelib.core import funnels, issues, heatmaps, sessions_insights, sessions_mobs, sessions_favorite, \ product_analytics, custom_metrics_predefined from chalicelib.utils import helper, pg_client from chalicelib.utils.TimeUTC import TimeUTC @@ -101,10 +101,10 @@ def __get_click_map_chart(project_id, user_id, data: schemas.CardClickMap, inclu return None data.series[0].filter.filters += data.series[0].filter.events data.series[0].filter.events = [] - return click_maps.search_short_session(project_id=project_id, user_id=user_id, - data=schemas.ClickMapSessionsSearch( - **data.series[0].filter.model_dump()), - include_mobs=include_mobs) + return heatmaps.search_short_session(project_id=project_id, user_id=user_id, + data=schemas.ClickMapSessionsSearch( + **data.series[0].filter.model_dump()), + include_mobs=include_mobs) # EE only @@ -198,6 +198,7 @@ def get_chart(project_id: int, data: schemas.CardSchema, user_id: int): schemas.MetricType.timeseries: __get_timeseries_chart, schemas.MetricType.table: __get_table_chart, schemas.MetricType.click_map: __get_click_map_chart, + schemas.MetricType.heat_map: __get_click_map_chart, schemas.MetricType.funnel: __get_funnel_chart, schemas.MetricType.insights: __get_insights_chart, schemas.MetricType.pathAnalysis: __get_path_analysis_chart diff --git a/ee/api/chalicelib/core/heatmaps.py b/ee/api/chalicelib/core/heatmaps.py new file mode 100644 index 0000000000..f3ca8e2a76 --- /dev/null +++ b/ee/api/chalicelib/core/heatmaps.py @@ -0,0 +1,256 @@ +import logging + +from decouple import config + +import schemas +from chalicelib.core import sessions_mobs, events + +# from chalicelib.utils import sql_helper as sh + +if config("EXP_SESSIONS_SEARCH", cast=bool, default=False): + from chalicelib.core import sessions_exp as sessions +else: + from chalicelib.core import sessions + +from chalicelib.utils import pg_client, helper, ch_client, exp_ch_helper + +logger = logging.getLogger(__name__) + + +def get_by_url(project_id, data: schemas.GetHeatmapPayloadSchema): + args = {"startDate": data.startTimestamp, "endDate": data.endTimestamp, + "project_id": project_id, "url": data.url} + constraints = ["main_events.project_id = toUInt16(%(project_id)s)", + "(main_events.url_hostpath = %(url)s OR main_events.url_path = %(url)s)", + "main_events.datetime >= toDateTime(%(startDate)s/1000)", + "main_events.datetime <= toDateTime(%(endDate)s/1000)", + "main_events.event_type='CLICK'", + "isNotNull(main_events.normalized_x)"] + query_from = f"{exp_ch_helper.get_main_events_table(data.startTimestamp)} AS main_events" + has_click_rage_filter = False + # TODO: is this used ? + # if len(data.filters) > 0: + # for i, f in enumerate(data.filters): + # if f.type == schemas.FilterType.issue and len(f.value) > 0: + # has_click_rage_filter = True + # query_from += """INNER JOIN events_common.issues USING (timestamp, session_id) + # INNER JOIN issues AS mis USING (issue_id) + # INNER JOIN LATERAL ( + # SELECT COUNT(1) AS real_count + # FROM events.clicks AS sc + # INNER JOIN sessions as ss USING (session_id) + # WHERE ss.project_id = 2 + # AND (sc.url = %(url)s OR sc.path = %(url)s) + # AND sc.timestamp >= %(startDate)s + # AND sc.timestamp <= %(endDate)s + # AND ss.start_ts >= %(startDate)s + # AND ss.start_ts <= %(endDate)s + # AND sc.selector = clicks.selector) AS r_clicks ON (TRUE)""" + # constraints += ["mis.project_id = %(project_id)s", + # "issues.timestamp >= %(startDate)s", + # "issues.timestamp <= %(endDate)s"] + # f_k = f"issue_value{i}" + # args = {**args, **sh.multi_values(f.value, value_key=f_k)} + # constraints.append(sh.multi_conditions(f"%({f_k})s = ANY (issue_types)", + # f.value, value_key=f_k)) + # constraints.append(sh.multi_conditions(f"mis.type = %({f_k})s", + # f.value, value_key=f_k)) + + if data.click_rage and not has_click_rage_filter: + constraints.append("""(issues.session_id IS NULL + OR (issues.datetime >= toDateTime(%(startDate)s/1000) + AND issues.datetime <= toDateTime(%(endDate)s/1000) + AND issues.project_id = toUInt16(%(project_id)s) + AND issues.event_type = 'ISSUE' + AND issues.project_id = toUInt16(%(project_id)s + AND mis.project_id = toUInt16(%(project_id)s + AND mis.type='click_rage'))))""") + query_from += """ LEFT JOIN experimental.events AS issues ON (main_events.session_id=issues.session_id) + LEFT JOIN experimental.issues AS mis ON (issues.issue_id=mis.issue_id)""" + with ch_client.ClickHouseClient() as cur: + query = cur.format(f"""SELECT main_events.normalized_x AS normalized_x, + main_events.normalized_y AS normalized_y + FROM {query_from} + WHERE {" AND ".join(constraints)} + LIMIT 500;""", args) + logger.debug("---------") + logger.debug(query) + logger.debug("---------") + try: + rows = cur.execute(query) + + except Exception as err: + logger.warning("--------- HEATMAP 2 SEARCH QUERY EXCEPTION CH -----------") + logger.warning(query) + logger.warning("--------- PAYLOAD -----------") + logger.warning(data) + logger.warning("--------------------") + raise err + + return helper.list_to_camel_case(rows) + + +if not config("EXP_SESSIONS_SEARCH", cast=bool, default=False): + # this part is identical to FOSS + SESSION_PROJECTION_COLS = """s.project_id, + s.session_id::text AS session_id, + s.user_uuid, + s.user_id, + s.user_os, + s.user_browser, + s.user_device, + s.user_device_type, + s.user_country, + s.start_ts, + s.duration, + s.events_count, + s.pages_count, + s.errors_count, + s.user_anonymous_id, + s.platform, + s.issue_score, + to_jsonb(s.issue_types) AS issue_types, + favorite_sessions.session_id NOTNULL AS favorite, + COALESCE((SELECT TRUE + FROM public.user_viewed_sessions AS fs + WHERE s.session_id = fs.session_id + AND fs.user_id = %(userId)s LIMIT 1), FALSE) AS viewed """ + + + def search_short_session(data: schemas.ClickMapSessionsSearch, project_id, user_id, + include_mobs: bool = True, exclude_sessions: list[str] = [], + _depth: int = 3): + no_platform = True + for f in data.filters: + if f.type == schemas.FilterType.platform: + no_platform = False + break + if no_platform: + data.filters.append(schemas.SessionSearchFilterSchema(type=schemas.FilterType.platform, + value=[schemas.PlatformType.desktop], + operator=schemas.SearchEventOperator._is)) + + full_args, query_part = sessions.search_query_parts(data=data, error_status=None, errors_only=False, + favorite_only=data.bookmarked, issue=None, + project_id=project_id, user_id=user_id) + full_args["exclude_sessions"] = tuple(exclude_sessions) + if len(exclude_sessions) > 0: + query_part += "\n AND session_id NOT IN (%(exclude_sessions)s)" + with pg_client.PostgresClient() as cur: + data.order = schemas.SortOrderType.desc + data.sort = 'duration' + main_query = cur.mogrify(f"""SELECT {SESSION_PROJECTION_COLS} + {query_part} + ORDER BY {data.sort} {data.order.value} + LIMIT 1;""", full_args) + logger.debug("--------------------") + logger.debug(main_query) + logger.debug("--------------------") + try: + cur.execute(main_query) + except Exception as err: + logger.warning("--------- CLICK MAP SHORT SESSION SEARCH QUERY EXCEPTION -----------") + logger.warning(main_query.decode('UTF-8')) + logger.warning("--------- PAYLOAD -----------") + logger.warning(data.model_dump_json()) + logger.warning("--------------------") + raise err + + session = cur.fetchone() + if session: + if include_mobs: + session['domURL'] = sessions_mobs.get_urls(session_id=session["session_id"], project_id=project_id) + session['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session["session_id"]) + if _depth > 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0: + return search_short_session(data=data, project_id=project_id, user_id=user_id, + include_mobs=include_mobs, + exclude_sessions=exclude_sessions + [session["session_id"]], + _depth=_depth - 1) + elif _depth == 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0: + logger.info("couldn't find an existing replay after 3 iterations for heatmap") + + session['events'] = events.get_by_session_id(project_id=project_id, session_id=session["session_id"], + event_type=schemas.EventType.location) + + return helper.dict_to_camel_case(session) +else: + # use CH + SESSION_PROJECTION_COLS = """ + s.project_id, + s.session_id AS session_id, + s.user_uuid AS user_uuid, + s.user_id AS user_id, + s.user_os AS user_os, + s.user_browser AS user_browser, + s.user_device AS user_device, + s.user_device_type AS user_device_type, + s.user_country AS user_country, + s.user_city AS user_city, + s.user_state AS user_state, + toUnixTimestamp(s.datetime)*1000 AS start_ts, + s.duration AS duration, + s.events_count AS events_count, + s.pages_count AS pages_count, + s.errors_count AS errors_count, + s.user_anonymous_id AS user_anonymous_id, + s.platform AS platform, + s.timezone AS timezone, + coalesce(issue_score,0) AS issue_score, + s.issue_types AS issue_types """ + + + def search_short_session(data: schemas.ClickMapSessionsSearch, project_id, user_id, + include_mobs: bool = True, exclude_sessions: list[str] = [], + _depth: int = 3): + no_platform = True + for f in data.filters: + if f.type == schemas.FilterType.platform: + no_platform = False + break + if no_platform: + data.filters.append(schemas.SessionSearchFilterSchema(type=schemas.FilterType.platform, + value=[schemas.PlatformType.desktop], + operator=schemas.SearchEventOperator._is)) + + full_args, query_part = sessions.search_query_parts_ch(data=data, error_status=None, errors_only=False, + favorite_only=data.bookmarked, issue=None, + project_id=project_id, user_id=user_id) + full_args["exclude_sessions"] = tuple(exclude_sessions) + if len(exclude_sessions) > 0: + query_part += "\n AND session_id NOT IN (%(exclude_sessions)s)" + with ch_client.ClickHouseClient() as cur: + data.order = schemas.SortOrderType.desc + data.sort = 'duration' + main_query = cur.format(f"""SELECT {SESSION_PROJECTION_COLS} + {query_part} + ORDER BY {data.sort} {data.order.value} + LIMIT 1;""", full_args) + logger.debug("--------------------") + logger.debug(main_query) + logger.debug("--------------------") + try: + session = cur.execute(main_query) + except Exception as err: + logger.warning("--------- CLICK MAP SHORT SESSION SEARCH QUERY EXCEPTION CH -----------") + logger.warning(main_query) + logger.warning("--------- PAYLOAD -----------") + logger.warning(data.model_dump_json()) + logger.warning("--------------------") + raise err + + if session: + if include_mobs: + session['domURL'] = sessions_mobs.get_urls(session_id=session["session_id"], project_id=project_id) + session['mobsUrl'] = sessions_mobs.get_urls_depercated(session_id=session["session_id"]) + if _depth > 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0: + return search_short_session(data=data, project_id=project_id, user_id=user_id, + include_mobs=include_mobs, + exclude_sessions=exclude_sessions + [session["session_id"]], + _depth=_depth - 1) + elif _depth == 0 and len(session['domURL']) == 0 and len(session['mobsUrl']) == 0: + logger.info("couldn't find an existing replay after 3 iterations for heatmap") + + session['events'] = events.get_by_session_id(project_id=project_id, session_id=session["session_id"], + event_type=schemas.EventType.location) + + return helper.dict_to_camel_case(session) diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index 0497187019..2af4ace664 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -11,7 +11,6 @@ rm -rf ./chalicelib/core/announcements.py rm -rf ./chalicelib/core/assist.py rm -rf ./chalicelib/core/authorizers.py rm -rf ./chalicelib/core/autocomplete.py -rm -rf ./chalicelib/core/click_maps.py rm -rf ./chalicelib/core/collaboration_base.py rm -rf ./chalicelib/core/collaboration_msteams.py rm -rf ./chalicelib/core/collaboration_slack.py @@ -22,7 +21,6 @@ rm -rf ./chalicelib/core/errors_favorite.py rm -rf ./chalicelib/core/events_mobile.py rm -rf ./chalicelib/core/feature_flags.py rm -rf ./chalicelib/core/funnels.py -rm -rf ./chalicelib/core/heatmaps.py rm -rf ./chalicelib/core/integration_base.py rm -rf ./chalicelib/core/integration_base_issue.py rm -rf ./chalicelib/core/integration_github.py diff --git a/ee/api/requirements-alerts.txt b/ee/api/requirements-alerts.txt index 488acf27b9..f94f151f5a 100644 --- a/ee/api/requirements-alerts.txt +++ b/ee/api/requirements-alerts.txt @@ -14,7 +14,7 @@ fastapi==0.111.0 uvicorn[standard]==0.30.1 python-decouple==3.8 pydantic[email]==2.3.0 -apscheduler==4.0.0a5 +apscheduler==3.10.4 clickhouse-driver[lz4]==0.2.8 azure-storage-blob==12.21.0b1 \ No newline at end of file diff --git a/ee/api/requirements-crons.txt b/ee/api/requirements-crons.txt index 2721c9aa8c..31c4396a32 100644 --- a/ee/api/requirements-crons.txt +++ b/ee/api/requirements-crons.txt @@ -13,7 +13,7 @@ jira==3.8.0 fastapi==0.111.0 python-decouple==3.8 pydantic[email]==2.3.0 -apscheduler==4.0.0a5 +apscheduler==3.10.4 clickhouse-driver[lz4]==0.2.8 redis==5.1.0b6 diff --git a/ee/api/requirements.txt b/ee/api/requirements.txt index 9af7a2bfc6..2a7b014cd2 100644 --- a/ee/api/requirements.txt +++ b/ee/api/requirements.txt @@ -15,7 +15,7 @@ uvicorn[standard]==0.30.1 gunicorn==22.0.0 python-decouple==3.8 pydantic[email]==2.3.0 -apscheduler==4.0.0a5 +apscheduler==3.10.4 clickhouse-driver[lz4]==0.2.8 # TODO: enable after xmlsec fix https://github.com/xmlsec/python-xmlsec/issues/252 diff --git a/ee/api/routers/core_dynamic.py b/ee/api/routers/core_dynamic.py index 4223b8aa2b..9096219d1b 100644 --- a/ee/api/routers/core_dynamic.py +++ b/ee/api/routers/core_dynamic.py @@ -7,7 +7,7 @@ import schemas from chalicelib.core import sessions, assist, heatmaps, sessions_favorite, sessions_assignments, errors, errors_viewed, \ - errors_favorite, sessions_notes, click_maps, sessions_replay, signup, feature_flags + errors_favorite, sessions_notes, sessions_replay, signup, feature_flags from chalicelib.core import sessions_viewed from chalicelib.core import tenants, users, projects, license from chalicelib.core import webhook @@ -569,7 +569,7 @@ def get_all_notes(projectId: int, data: schemas.SearchNoteSchema = Body(...), @app.post('/{projectId}/click_maps/search', tags=["click maps"], dependencies=[OR_scope(Permissions.session_replay)]) def click_map_search(projectId: int, data: schemas.ClickMapSessionsSearch = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return {"data": click_maps.search_short_session(user_id=context.user_id, data=data, project_id=projectId)} + return {"data": heatmaps.search_short_session(user_id=context.user_id, data=data, project_id=projectId)} @app.post('/{project_id}/feature-flags/search', tags=["feature flags"],