diff --git a/apps/site/load_tests/.tool-versions b/apps/site/load_tests/.tool-versions new file mode 100644 index 0000000000..2ab81990b6 --- /dev/null +++ b/apps/site/load_tests/.tool-versions @@ -0,0 +1 @@ +python 3.11.2 diff --git a/apps/site/load_tests/socket_user.py b/apps/site/load_tests/socket_user.py new file mode 100644 index 0000000000..9414a4c81a --- /dev/null +++ b/apps/site/load_tests/socket_user.py @@ -0,0 +1,124 @@ +import json +import logging +import re +import time +import gevent +import websocket +from locust import User + + +class SocketIOUser(User): + """ + A locust that includes a socket io websocket connection. + You could easily use this a template for plain WebSockets, + socket.io just happens to be my use case. You can use multiple + inheritance to combine this with an HttpUser + (class MyUser(HttpUser, SocketIOUser) + """ + + abstract = True + message_regex = re.compile(r"(\d*)(.*)") + description_regex = re.compile(r"<([0-9]+)>$") + + def connect(self, host: str, header=[], **kwargs): + # websocket.enableTrace(True) + self.ws = websocket.create_connection(host, header=header, **kwargs) + self.sleep_ref = 1 + gevent.spawn(self.receive_loop) + + def on_message(self, message): # override this method in your subclass for custom handling + m = self.message_regex.match(message) + response_time = 0 # unknown + if m is None: + # uh oh... + raise Exception( + f"got no matches for {self.message_regex} in {message}") + code = m.group(1) + json_string = m.group(2) + if code == "0": + name = "0 open" + elif code == "3": + name = "3 heartbeat" + elif code == "40": + name = "40 message ok" + elif code == "42": + # this is rather specific to our use case. Some messages contain an originating timestamp, + # and we use that to calculate the delay & report it as locust response time + # see it as inspiration rather than something you just pick up and use + current_timestamp = time.time() + obj = json.loads(json_string) + # logging.debug(json_string) + ts_type, payload = obj + name = f"{code} {ts_type} apiUri: {payload['apiUri']}" + + if payload["value"] != "": + value = payload["value"] + + if "draw" in value: + description = value["draw"]["description"] + description_match = self.description_regex.search( + description) + if description_match: + sent_timestamp = int(description_match.group(1)) + response_time = current_timestamp - sent_timestamp + else: + # differentiate samples that have no timestamps from ones that do + name += "_" + elif "source_ts" in value: + sent_timestamp = value["source_ts"] + response_time = (current_timestamp - sent_timestamp) * 1000 + else: + name += "_missingTimestamp" + else: + print(f"Received unexpected message: {message}") + return + + self.environment.events.request.fire( + request_type="WSR", + name=name, + response_time=response_time, + response_length=len(message), + exception=None, + context=self.context(), + ) + + def receive_loop(self): + while True: + message = self.ws.recv() + # logging.debug(f"WSR: {message}") + self.on_message(message) + + def send(self, body, name=None, context={}): + if not name: + if body == "2": + name = "2 heartbeat" + else: + # hoping this is a subscribe type message, try to detect name + m = re.search(r'(\d*)\["([a-z]*)"', body) + assert m is not None + code = m.group(1) + action = m.group(2) + url_part = re.search(r'"url": *"([^"]*)"', body) + assert url_part is not None + url = re.sub(r"/[0-9_]*/", "/:id/", url_part.group(1)) + name = f"{code} {action} url: {url}" + + self.environment.events.request.fire( + request_type="WSS", + name=name, + response_time=None, + response_length=len(body), + exception=None, + context={**self.context(), **context}, + ) + # logging.debug(f"WSS: {body}") + self.ws.send(body) + + def sleep_with_heartbeat(self, seconds): + while seconds >= 0: + gevent.sleep(min(15, seconds)) + seconds -= 15 + self.sleep_ref += 1 + # [null,"2","phoenix","heartbeat",{}] + self.send( + f'{{"topic":"phoenix","event":"heartbeat","payload":{{}},"ref":{self.sleep_ref}}}', name="Heartbeat") diff --git a/apps/site/load_tests/stops-bus.txt b/apps/site/load_tests/stops-bus.txt new file mode 100644 index 0000000000..0f43b6c60f --- /dev/null +++ b/apps/site/load_tests/stops-bus.txt @@ -0,0 +1,94 @@ +place-nubn +61 +17091 +place-crtst +place-wtcst +place-conrd +27092 +17093 +17094 +17095 +1 +247 +30249 +place-estav +place-boxdt +place-belsq +121009 +111803 +111640 +6555 +49001 +55 +128 +7 +11802 +1799 +11803 +1807 +2777 +111830 +111146 +109805 +109821 +109898 +16535 +6555 +177 +6551 +903 +9031 +178 +1994 +65541 +7834 +7845 +86971 +88333 +903 +7645 +86944 +2099 +72805 +5700 +4513 +4524 +7223 +117 +7415 +7486 +47527 +94320 +6773 +6600 +4674 +6613 +4728 +4520 +4524 +4563 +4736 +4709 +4747 +4807 +5700 +4522 +22751 +49807 +16881 +4511 +65 +4510 +45003 +6902 +3065 +3067 +38813 +13142 +3149 +3181 +3199 +3160 +3327 +3806 +3824 diff --git a/apps/site/load_tests/stops-commuter-rail.txt b/apps/site/load_tests/stops-commuter-rail.txt new file mode 100644 index 0000000000..c1290d165e --- /dev/null +++ b/apps/site/load_tests/stops-commuter-rail.txt @@ -0,0 +1,164 @@ +place-DB-2222 +place-DB-2205 +place-DB-2249 +place-DB-2230 +place-DB-2265 +place-DB-0095 +place-sstat +place-DB-2240 +place-DB-2258 +place-FR-0361 +place-FR-0064 +place-FR-0115 +place-FR-0201 +place-FR-0494 +place-FR-0132 +place-FR-0167 +place-FR-0301 +place-FR-0451 +place-north +place-portr +place-FR-0394 +place-FR-0253 +place-FR-3338 +place-FR-0098 +place-FR-0074 +place-FR-0219 +place-WML-0102 +place-bbsta +place-WML-0035 +place-WML-0214 +place-WML-0364 +place-WML-0025 +place-WML-0177 +place-WML-0081 +place-sstat +place-WML-0274 +place-WML-0125 +place-WML-0135 +place-WML-0147 +place-WML-0199 +place-WML-0091 +place-WML-0340 +place-WML-0442 +place-bbsta +place-FB-0118 +place-FB-0109 +place-forhl +place-FB-0303 +place-FS-0049 +place-FB-0275 +place-NEC-2203 +place-FB-0125 +place-FB-0230 +place-FB-0148 +place-FB-0143 +place-DB-0095 +place-rugg +place-sstat +place-FB-0191 +place-FB-0166 +place-brntn +place-GRB-0199 +place-GRB-0146 +place-GRB-0276 +place-jfk +place-GRB-0183 +place-GRB-0233 +place-qnctr +place-sstat +place-GRB-0162 +place-GRB-0118 +place-NHRML-0127 +place-WR-0228 +place-WR-0205 +place-WR-0325 +place-WR-0085 +place-WR-0329 +place-WR-0264 +place-mlmnl +place-WR-0075 +place-WR-0067 +place-north +place-WR-0163 +place-ogmnl +place-WR-0120 +place-WR-0099 +place-WR-0062 +place-PB-0194 +place-brntn +place-PB-0281 +place-PB-0245 +place-jfk +place-KB-0351 +place-qnctr +place-sstat +place-PB-0158 +place-PB-0212 +place-NHRML-0127 +place-NHRML-0254 +place-NHRML-0218 +place-north +place-NHRML-0073 +place-NHRML-0055 +place-NHRML-0152 +place-brntn +place-MM-0277 +place-MM-0200 +place-MM-0219 +place-MM-0150 +place-jfk +place-MM-0356 +place-MM-0186 +place-qnctr +place-sstat +place-bbsta +place-NB-0072 +place-forhl +place-NB-0109 +place-NB-0076 +place-NB-0127 +place-NB-0137 +place-NB-0120 +place-NB-0064 +place-rugg +place-sstat +place-NB-0080 +place-ER-0183 +place-GB-0229 +place-chels +place-GB-0316 +place-ER-0227 +place-ER-0276 +place-ER-0115 +place-GB-0254 +place-GB-0198 +place-ER-0362 +place-ER-0208 +place-north +place-ER-0099 +place-GB-0353 +place-ER-0312 +place-ER-0168 +place-ER-0128 +place-GB-0296 +place-NEC-1969 +place-bbsta +place-SB-0156 +place-NEC-2139 +place-forhl +place-NEC-2203 +place-NEC-2040 +place-NEC-1891 +place-NEC-1851 +place-NEC-2173 +place-rugg +place-NEC-2108 +place-sstat +place-SB-0189 +place-NEC-1768 +place-NEC-1659 +place-bbsta +place-FB-0118 +place-FS-0049 +place-sstat diff --git a/apps/site/load_tests/stops-ferry.txt b/apps/site/load_tests/stops-ferry.txt new file mode 100644 index 0000000000..6216f16d06 --- /dev/null +++ b/apps/site/load_tests/stops-ferry.txt @@ -0,0 +1,17 @@ +Boat-Charlestown +Boat-Long-South +Boat-George +Boat-Hingham +Boat-Hull +Boat-Logan +Boat-Long +Boat-Rowes +Boat-Lewis +Boat-Long +Boat-Blossom +Boat-Long +Boat-Aquarium +Boat-Logan +Boat-Quincy +Boat-Fan +Boat-Winthrop diff --git a/apps/site/load_tests/stops-subway.txt b/apps/site/load_tests/stops-subway.txt new file mode 100644 index 0000000000..8f3347ef50 --- /dev/null +++ b/apps/site/load_tests/stops-subway.txt @@ -0,0 +1,125 @@ +place-alfcl +place-alsgr +place-amory +place-andrw +place-aport +place-aqucl +place-armnl +place-asmnl +place-astao +place-babck +place-balsq +place-bbsta +place-bckhl +place-bcnfd +place-bcnwa +place-bland +place-bmmnl +place-bndhl +place-bomnl +place-boyls +place-brdwy +place-brico +place-brkhl +place-brmnl +place-brntn +place-bucen +place-buest +place-butlr +place-bvmnl +place-capst +place-ccmnl +place-cedgr +place-cenav +place-chhil +place-chill +place-chmnl +place-chncl +place-chswk +place-clmnl +place-cntsq +place-coecl +place-cool +place-davis +place-denrd +place-dwnxg +place-eliot +place-engav +place-esomr +place-fbkst +place-fenwd +place-fenwy +place-fldcr +place-forhl +place-gilmn +place-gover +place-grigg +place-grnst +place-haecl +place-harsq +place-harvd +place-hsmnl +place-hwsst +place-hymnl +place-jaksn +place-jfk +place-kencl +place-knncl +place-kntst +place-lake +place-lech +place-lngmd +place-longw +place-masta +place-matt +place-mdftf +place-mfa +place-mgngl +place-miltt +place-mispk +place-mlmnl +place-mvbcl +place-newtn +place-newto +place-north +place-nqncy +place-nuniv +place-ogmnl +place-orhte +place-pktrm +place-portr +place-prmnl +place-qamnl +place-qnctr +place-rbmnl +place-rcmnl +place-river +place-rsmnl +place-rugg +place-rvrwy +place-sbmnl +place-sdmnl +place-shmnl +place-smary +place-smmnl +place-sougr +place-spmnl +place-sstat +place-state +place-sthld +place-stpul +place-sull +place-sumav +place-symcl +place-tapst +place-tumnl +place-unsqu +place-valrd +place-waban +place-wascm +place-welln +place-wimnl +place-wlsta +place-wondl +place-woodl +place-wrnst diff --git a/apps/site/load_tests/stopsredesign.py b/apps/site/load_tests/stopsredesign.py new file mode 100644 index 0000000000..c577e7a41a --- /dev/null +++ b/apps/site/load_tests/stopsredesign.py @@ -0,0 +1,142 @@ +from random import choice +from pyquery import PyQuery +from gevent.pool import Group +from locust import between +from locust import HttpUser +from locust.user.task import TaskSet, task +from socket_user import SocketIOUser +import time +import json +import glob +import os + +EXPECTED_RESPONSE_SEC = .826 + +dev_url = "dev.mbtace.com" +file_list = glob.glob(os.path.join(os.getcwd(), "stops-*.txt")) + + +def get_all_stops(): + stops = [] + for file_path in file_list: + with open(file_path) as f_input: + stops.extend(f_input.read().splitlines()) + + return stops + + +class StopPageVisitor(HttpUser, SocketIOUser): + # The time we expect a user to hang around, in seconds + wait_time = between(5, 120) + + def on_start(self): + self.wait() + self.start_time = time.time() + stops = get_all_stops() + self.connect(f"ws://{dev_url}/socket/websocket", + verify=False, suppress_origin=True) + self.stop_id = choice(stops) + # route IDs are determined dynamically so it's challenging to include here. these include + # 1. route alerts /api/alerts?route_ids=... + # 2. the routes themselves /api/routes/... + self.endpoints = [ + [f"/api/stops/{self.stop_id}/schedules?last_stop_departures=false&future_departures=true", + "Schedules Endpoint"], + [f"/api/map-config", "MapConfig Endpoint"], + [f"/api/stop/{self.stop_id}/facilities", + "Facilities Endpoint"], + [f"/api/stop/{self.stop_id}/route-patterns", + "Route Patterns Endpoint"], + [f"/api/stops/{self.stop_id}/alerts", + "Alerts for Stop Endpoint"], + [f"/api/fares/one-way?stop_id={self.stop_id}", + "One-Way Fares Endpoint"], + [f"/api/stop/{self.stop_id}", "Stop Endpoint"], + ] + super().on_start() + + @task + def load_endpoints(self): + for end in self.endpoints: + with self.client.get(end[0], name=end[1], catch_response=True) as r: + if r.elapsed.total_seconds() > EXPECTED_RESPONSE_SEC: + r.failure("request took too long") + else: + if r.ok: + r.success() + else: + r.failure(r.reason) + + @task + def load_stop_page(self): + with self.client.get(f"/stops/{self.stop_id}?active_flag=stops_redesign", name="Stop Page", catch_response=True) as stop_response: + if stop_response.elapsed.total_seconds() > 1: + stop_response.failure("request took too long") + else: + if stop_response.ok: + stop_response.success() + else: + stop_response.failure(stop_response.reason) + + @task + def subscribe_to_predictions(self): + self.start_time = time.time() + self.send(f'{{"topic":"predictions:stop:{self.stop_id}","event":"phx_join","payload":{{}},"ref":"1","join_ref":"1"}}', + name="Streaming Predictions Outbound Join") + # wait for additional pushes, while occasionally sending heartbeats, like a real client would + self.sleep_with_heartbeat(60) + + def on_message(self, message): + elapsed_time = (time.time() - self.start_time) * 1000 # convert to MS + if message is not None: + try: + data = json.loads(message) + topic = data['topic'] + event = data['event'] + print(f"{topic} {event}") + payload = data['payload'] + + if event == 'data': + # This is the event sending new predictions to the frontend. + exception = None + if elapsed_time > 10000: + exception = Exception("took more than 10s") + self.environment.events.request.fire( + request_type="WSR", + name="Streaming Predictions Inbound Predictions", + response_time=elapsed_time, + response_length=len(payload['predictions']), + exception=exception, + context=self.context(), + ) + + elif event == 'phx_reply': + status = payload['status'] + response = payload['response'] + if not status == "ok": + self.environment.events.request.fire( + request_type="WSR", + name="Streaming Predictions Inbound Predictions", + response_time=elapsed_time, + response_length=len(response), + exception=Exception("not-OK response"), + context=self.context(), + ) + + else: + self.environment.events.request.fire( + request_type="WSR", + name="Streaming Predictions Inbound Predictions", + response_time=elapsed_time, + response_length=len(payload), + exception=None, + context=self.context(), + ) + + except json.JSONDecodeError: + print(message) + + self.start_time = time.time() + # The original plugin was written using a very different format of + # websocket messages, oh well + # super().on_message(message)