benchmark_regression v1 (#7573)

# Overview
- add device_Info, labels in benchmark regression report schema
   - auto detect deviceInfos, add labels in case customize search
- switch the benchmark table from fortesting to benchmark db
- add unit test for lambda

# Migrate to benchmark db table
once this is deployed, we will route UI traffic to benchmark table, we
can either backfilling the data from fortesting, or can start over again
since it's regression report data, nothing memoriable

Author: yangw-dev

.github/workflows/tests.yml

@@ -47,5 +47,5 @@ jobs:
         pip install -r aws/lambda/tests/test_requirements.txt
         echo ::endgroup::
 
-        # Test aws lambda
-        pytest -v aws/lambda/tests
+        # Test aws lambda, add relative path to PYTHONPATH for importing
+        PYTHONPATH=aws/lambda:aws/lambda/benchmark_regression_summary_report pytest -v aws/lambda/tests

aws/lambda/benchmark_regression_summary_report/common/config.py

@@ -164,7 +164,7 @@
             "latency": RegressionPolicy(
                 name="latency",
                 condition="less_equal",
-                threshold=1.20,
+                threshold=1.35,
                 baseline_aggregation="median",
             ),
         },

aws/lambda/benchmark_regression_summary_report/common/regression_utils.py

@@ -74,6 +74,7 @@ class BenchmarkRegressionReport(TypedDict):
     results: List[PerGroupResult]
     baseline_meta_data: TimeSeriesMetaInfo
     new_meta_data: TimeSeriesMetaInfo
+    device_info: List[str]
 
 
 def get_regression_status(regression_summary: BenchmarkRegressionSummary) -> str:
@@ -103,6 +104,8 @@ def __init__(
         self.lastest_ts_info = self._get_meta_info(target_ts.time_series)
         self.target_ts = self._to_data_map(target_ts)
         self.baseline_ts = self._to_data_map(baseline_ts)
+        # collect device info from target_ts
+        self.device_info = self._to_device_info(target_ts)
 
     def generate(self) -> BenchmarkRegressionReport:
         if not self.baseline_ts or not self.target_ts:
@@ -148,7 +151,6 @@ def detect_regressions_with_policies(
 
             base_item = baseline_map.get(key)
             if not base_item:
-                logger.warning("Skip. No baseline item found for %s", key)
                 results.append(
                     PerGroupResult(
                         group_info=gi,
@@ -217,6 +219,7 @@ def detect_regressions_with_policies(
             results=results,
             baseline_meta_data=self.baseline_ts_info,
             new_meta_data=self.lastest_ts_info,
+            device_info=self.device_info,
         )
 
     def summarize_label_counts(
@@ -243,6 +246,22 @@ def _label_str(self, x) -> str:
             return (v if isinstance(v, str) else str(v)).lower()
         return str(x).lower()
 
+    def _to_device_info(self, data: "BenchmarkTimeSeriesApiData") -> List[str]:
+        result = set()
+        for ts_group in data.time_series:
+            device = ts_group.group_info.get("device", "")
+            arch = ts_group.group_info.get("arch", "")
+            key = ""
+            if device and arch:
+                key = f"{device}_{arch}"
+            elif device:
+                key = device
+
+            if not key:
+                continue
+            result.add(key)
+        return list(result)
+
     def _to_data_map(
         self, data: "BenchmarkTimeSeriesApiData", field: str = "value"
     ) -> Dict[tuple, BenchmarkRegressionPointGroup]:

aws/lambda/benchmark_regression_summary_report/common/report_manager.py

@@ -56,10 +56,10 @@ def __init__(
         # extract latest meta data from report
         self.baseline = self.raw_report["baseline_meta_data"]
         self.target = self.raw_report["new_meta_data"]
+        self.device_info = self.raw_report["device_info"]
         self.target_latest_commit = self.target["end"]["commit"]
         self.target_latest_ts_str = self.target["end"]["timestamp"]
         self.status = get_regression_status(self.raw_report["summary"])
-
         self.report_data = self._to_report_data(
             config_id=config.id,
             regression_report=self.raw_report,
@@ -74,30 +74,34 @@ def run(
         main method used to insert the report to db and create github comment in targeted issue
         """
         try:
-            applied_insertion = self.insert_to_db(cc)
+            applied_insertion = self.insert_to_db(cc, self.db_table_name)
         except Exception as e:
-            logger.error(f"failed to insert report to db, error: {e}")
+            logger.warning(f"failed to insert report to db, error: {str(e)}")
             raise
         if not applied_insertion:
-            logger.info("[%s] skip notification,  already exists in db", self.config_id)
+            logger.info(
+                "[%s] skip notification, already exists in db or this is dry-run",
+                self.config_id,
+            )
             return
         self.notify_github_comment(github_token)
+        logger.info("[%s] Done. ReportManager run is completed.", self.config_id)
 
-    def notify_github_comment(self, github_token: str):
+    def notify_github_comment(self, github_token: str) -> str:
         if self.status != "regression":
             logger.info(
                 "[%s] no regression found, skip notification",
                 self.config_id,
             )
-            return
+            return "skip_no_regression"
 
         github_notification = self.config.policy.get_github_notification_config()
         if not github_notification:
             logger.info(
                 "[%s] no github notification config found, skip notification",
                 self.config_id,
             )
-            return
+            return "skip_no_notification_config"
         logger.info("[%s] prepareing gitub comment content", self.config_id)
         content = self._to_markdown()
         if self.is_dry_run:
@@ -109,10 +113,14 @@ def notify_github_comment(self, github_token: str):
             logger.info("[dry run] printing comment content")
             print(json.dumps(content, indent=2, default=str))
             logger.info("[dry run] Done! Finish printing comment content")
-            return
-        logger.info("[%s] create comment to github issue", self.config_id)
-        github_notification.create_github_comment(content, github_token)
-        logger.info("[%s] done. comment is sent to github", self.config_id)
+            return "skip_dry_run"
+        try:
+            github_notification.create_github_comment(content, github_token)
+            logger.info("[%s] done. comment is sent to github", self.config_id)
+            return "success"
+        except Exception as e:
+            logger.warning(f"failed to insert report to db, error: {str(e)}")
+            return "failure"
 
     def _to_markdown(self) -> str:
         regression_items = [
@@ -134,8 +142,7 @@ def _to_markdown(self) -> str:
         )
 
     def insert_to_db(
-        self,
-        cc: clickhouse_connect.driver.client.Client,
+        self, cc: clickhouse_connect.driver.client.Client, table: str
     ) -> bool:
         logger.info(
             "[%s]prepare data for db insertion report (%s)...", self.config_id, self.id
@@ -176,6 +183,7 @@ def insert_to_db(
             "total_count": regression_summary["total_count"],
             "repo": self.repo,
             "report_json": report_json,
+            "device_info": self.device_info,
         }
 
         if self.is_dry_run:
@@ -196,7 +204,7 @@ def insert_to_db(
         try:
             if self._row_exists(
                 cc,
-                self.db_table_name,
+                table,
                 params["report_id"],
                 params["type"],
                 params["repo"],
@@ -208,7 +216,7 @@ def insert_to_db(
                     self.id,
                 )
                 return False
-            self._db_insert(cc, self.db_table_name, params)
+            self._db_insert(cc, table, params)
             logger.info(
                 "[%s] Done. inserted benchmark regression report(%s)",
                 self.config_id,
@@ -219,7 +227,7 @@ def insert_to_db(
             logger.exception(
                 "[%s] failed to insert report to target table %s",
                 self.config_id,
-                self.db_table_name,
+                table,
             )
             raise
 
@@ -256,7 +264,8 @@ def _db_insert(
                 suspected_regression_count,
                 total_count,
                 repo,
-                report
+                report,
+                device_info
             )
             VALUES
             (
@@ -271,9 +280,12 @@ def _db_insert(
                 %(suspected_regression_count)s,
                 %(total_count)s,
                 %(repo)s,
-                %(report_json)s
+                %(report_json)s,
+                %(device_info)s
             )
             """
+        # debugging only - uncomment to see the sql
+        # logger.info("[%s]inserting report to db, sql: %s", self.config_id, sql)
         cc.command(sql, parameters=params)
 
     def _row_exists(

aws/lambda/benchmark_regression_summary_report/lambda_function.py

@@ -18,7 +18,7 @@
 
 
 # TODO(elainewy): change this to benchmark.benchmark_regression_report once the table is created
-BENCHMARK_REGRESSION_REPORT_TABLE = "fortesting.benchmark_regression_report"
+BENCHMARK_REGRESSION_REPORT_TABLE = "benchmark.benchmark_regression_report"
 BENCHMARK_REGRESSION_TRACKING_CONFIG_IDS = ["compiler_regression"]
 
 logging.basicConfig(