From 820a53189c8c01fcb5397480e79b07887e9ea873 Mon Sep 17 00:00:00 2001
From: Jonas Finnemann Jensen <jonasfj@google.com>
Date: Fri, 8 Nov 2024 15:39:27 +0100
Subject: [PATCH] Increase concurrency and handle errors better (#8257)

* Increase concurrency and handle errors better

* Update app/lib/package/api_export/api_exporter.dart

Co-authored-by: Sigurd Meldgaard <sigurdm@google.com>

---------

Co-authored-by: Sigurd Meldgaard <sigurdm@google.com>
---
 app/lib/package/api_export/api_exporter.dart | 12 +++++++++++-
 app/lib/package/api_export/exported_api.dart |  2 +-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/app/lib/package/api_export/api_exporter.dart b/app/lib/package/api_export/api_exporter.dart
index 82e72a4e2..4cdee8fab 100644
--- a/app/lib/package/api_export/api_exporter.dart
+++ b/app/lib/package/api_export/api_exporter.dart
@@ -30,7 +30,7 @@ void registerApiExporter(ApiExporter value) =>
 /// The active API Exporter service or null if it hasn't been initialized.
 ApiExporter? get apiExporter => ss.lookup(#_apiExporter) as ApiExporter?;
 
-const _concurrency = 30;
+const _concurrency = 50;
 
 class ApiExporter {
   final ExportedApi _api;
@@ -134,6 +134,7 @@ class ApiExporter {
   Future<void> synchronizeExportedApi() async {
     final allPackageNames = <String>{};
     final packageQuery = dbService.query<Package>();
+    var errCount = 0;
     await packageQuery.run().parallelForEach(_concurrency, (pkg) async {
       final name = pkg.name!;
       if (pkg.isNotVisible) {
@@ -143,6 +144,9 @@ class ApiExporter {
 
       // TODO: Consider retries around all this logic
       await synchronizePackage(name);
+    }, onError: (e, st) {
+      _log.warning('synchronizePackage() failed', e, st);
+      errCount++;
     });
 
     await synchronizePackageNameCompletionData();
@@ -157,6 +161,12 @@ class ApiExporter {
     });
 
     await _api.garbageCollect(allPackageNames);
+
+    if (errCount > 0) {
+      throw Exception(
+        '$errCount exceptions happened while calling synchronizeExportedApi',
+      );
+    }
   }
 
   /// Sync package and into [ExportedApi], this will synchronize package into
diff --git a/app/lib/package/api_export/exported_api.dart b/app/lib/package/api_export/exported_api.dart
index e7c5dd51e..a97ffca81 100644
--- a/app/lib/package/api_export/exported_api.dart
+++ b/app/lib/package/api_export/exported_api.dart
@@ -44,7 +44,7 @@ const _minGarbageAge = Duration(hours: 3);
 /// All writes to the bucket containing the exported API should go through this
 /// interface.
 final class ExportedApi {
-  final Pool _pool = Pool(50);
+  final Pool _pool = Pool(80);
   final Storage _storage;
   final Bucket _bucket;
   final List<String> _prefixes = [