Increase concurrency and handle errors better (dart-lang#8257)

* Increase concurrency and handle errors better * Update app/lib/package/api_export/api_exporter.dart Co-authored-by: Sigurd Meldgaard <[email protected]> --------- Co-authored-by: Sigurd Meldgaard <[email protected]>
isoos · Nov 8, 2024 · 820a531 · 820a531
1 parent 088812e
commit 820a531
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 2 deletions.
diff --git a/app/lib/package/api_export/api_exporter.dart b/app/lib/package/api_export/api_exporter.dart
@@ -30,7 +30,7 @@ void registerApiExporter(ApiExporter value) =>
 /// The active API Exporter service or null if it hasn't been initialized.
 ApiExporter? get apiExporter => ss.lookup(#_apiExporter) as ApiExporter?;
 
-const _concurrency = 30;
+const _concurrency = 50;
 
 class ApiExporter {
   final ExportedApi _api;
@@ -134,6 +134,7 @@ class ApiExporter {
   Future<void> synchronizeExportedApi() async {
     final allPackageNames = <String>{};
     final packageQuery = dbService.query<Package>();
+    var errCount = 0;
     await packageQuery.run().parallelForEach(_concurrency, (pkg) async {
       final name = pkg.name!;
       if (pkg.isNotVisible) {
@@ -143,6 +144,9 @@ class ApiExporter {
 
       // TODO: Consider retries around all this logic
       await synchronizePackage(name);
+    }, onError: (e, st) {
+      _log.warning('synchronizePackage() failed', e, st);
+      errCount++;
     });
 
     await synchronizePackageNameCompletionData();
@@ -157,6 +161,12 @@ class ApiExporter {
     });
 
     await _api.garbageCollect(allPackageNames);
+
+    if (errCount > 0) {
+      throw Exception(
+        '$errCount exceptions happened while calling synchronizeExportedApi',
+      );
+    }
   }
 
   /// Sync package and into [ExportedApi], this will synchronize package into

diff --git a/app/lib/package/api_export/exported_api.dart b/app/lib/package/api_export/exported_api.dart
@@ -44,7 +44,7 @@ const _minGarbageAge = Duration(hours: 3);
 /// All writes to the bucket containing the exported API should go through this
 /// interface.
 final class ExportedApi {
-  final Pool _pool = Pool(50);
+  final Pool _pool = Pool(80);
   final Storage _storage;
   final Bucket _bucket;
   final List<String> _prefixes = [