Bundle analysis add to sizes to timeseries (#493)

Author: JerrySentry

services/bundle_analysis.py

@@ -13,16 +13,27 @@
     BundleAnalysisReportLoader,
     BundleChange,
 )
+from shared.bundle_analysis.models import AssetType
 from shared.bundle_analysis.storage import get_bucket_name
 from shared.reports.enums import UploadState
 from shared.storage import get_appropriate_storage_service
 from shared.storage.exceptions import FileNotInStorageError, PutRequestRateLimitError
 from shared.torngit.base import TorngitBaseAdapter
 from shared.torngit.exceptions import TorngitClientError
 from shared.yaml import UserYaml
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.orm import Session
 
 from database.enums import ReportType
-from database.models import Commit, CommitReport, Repository, Upload, UploadError
+from database.models import (
+    Commit,
+    CommitReport,
+    Measurement,
+    MeasurementName,
+    Repository,
+    Upload,
+    UploadError,
+)
 from database.models.core import GITHUB_APP_INSTALLATION_DEFAULT_NAME
 from services.archive import ArchiveService
 from services.report import BaseReportService
@@ -32,6 +43,7 @@
     get_repo_provider_service,
 )
 from services.storage import get_storage_client
+from services.timeseries import repository_datasets_query
 from services.urls import get_bundle_analysis_pull_url
 from services.yaml import read_yaml_field
 
@@ -253,6 +265,129 @@ def process_upload(self, commit: Commit, upload: Upload) -> ProcessingResult:
             session_id=session_id,
         )
 
+    def _save_to_timeseries(
+        self,
+        db_session: Session,
+        commit: Commit,
+        name: str,
+        measurable_id: str,
+        value: float,
+    ):
+        command = insert(Measurement.__table__).values(
+            name=name,
+            owner_id=commit.repository.ownerid,
+            repo_id=commit.repoid,
+            measurable_id=measurable_id,
+            branch=commit.branch,
+            commit_sha=commit.commitid,
+            timestamp=commit.timestamp,
+            value=value,
+        )
+        command = command.on_conflict_do_update(
+            index_elements=[
+                Measurement.name,
+                Measurement.owner_id,
+                Measurement.repo_id,
+                Measurement.measurable_id,
+                Measurement.commit_sha,
+                Measurement.timestamp,
+            ],
+            set_=dict(
+                branch=command.excluded.branch,
+                value=command.excluded.value,
+            ),
+        )
+        db_session.execute(command)
+        db_session.flush()
+
+    @sentry_sdk.trace
+    def save_measurements(self, commit: Commit, upload: Upload) -> ProcessingResult:
+        """
+        Save timeseries measurements for this bundle analysis report
+        """
+        try:
+            commit_report: CommitReport = upload.report
+            repo_hash = ArchiveService.get_archive_hash(commit_report.commit.repository)
+            storage_service = get_storage_client()
+            bundle_loader = BundleAnalysisReportLoader(storage_service, repo_hash)
+
+            # fetch existing bundle report from storage
+            bundle_analysis_report = bundle_loader.load(commit_report.external_id)
+
+            dataset_names = [
+                dataset.name for dataset in repository_datasets_query(commit.repository)
+            ]
+
+            db_session = commit.get_db_session()
+            for bundle_report in bundle_analysis_report.bundle_reports():
+                # For overall bundle size
+                if MeasurementName.bundle_analysis_report_size.value in dataset_names:
+                    self._save_to_timeseries(
+                        db_session,
+                        commit,
+                        MeasurementName.bundle_analysis_report_size.value,
+                        bundle_report.name,
+                        bundle_report.total_size(),
+                    )
+
+                # For individual javascript associated assets using UUID
+                if MeasurementName.bundle_analysis_asset_size.value in dataset_names:
+                    for asset in bundle_report.asset_reports():
+                        if asset.asset_type == AssetType.JAVASCRIPT:
+                            self._save_to_timeseries(
+                                db_session,
+                                commit,
+                                MeasurementName.bundle_analysis_asset_size.value,
+                                asset.uuid,
+                                asset.size,
+                            )
+
+                # For asset types sizes
+                asset_type_map = {
+                    MeasurementName.bundle_analysis_font_size: AssetType.FONT,
+                    MeasurementName.bundle_analysis_image_size: AssetType.IMAGE,
+                    MeasurementName.bundle_analysis_stylesheet_size: AssetType.STYLESHEET,
+                    MeasurementName.bundle_analysis_javascript_size: AssetType.JAVASCRIPT,
+                }
+                for measurement_name, asset_type in asset_type_map.items():
+                    if measurement_name.value in dataset_names:
+                        total_size = 0
+                        for asset in bundle_report.asset_reports():
+                            if asset.asset_type == asset_type:
+                                total_size += asset.size
+                        self._save_to_timeseries(
+                            db_session,
+                            commit,
+                            measurement_name.value,
+                            bundle_report.name,
+                            total_size,
+                        )
+
+            return ProcessingResult(
+                upload=upload,
+                commit=commit,
+            )
+        except Exception:
+            sentry_metrics.incr(
+                "bundle_analysis_upload",
+                tags={
+                    "result": "parser_error",
+                    "repository": commit.repository.repoid,
+                },
+            )
+            return ProcessingResult(
+                upload=upload,
+                commit=commit,
+                error=ProcessingError(
+                    code="measurement_save_error",
+                    params={
+                        "location": upload.storage_path,
+                        "repository": commit.repository.repoid,
+                    },
+                    is_retryable=False,
+                ),
+            )
+
 
 class ComparisonError(Exception):
     pass