Skip to content

Commit

Permalink
Verify more storage calls that they are inside a retry block. (dart-l…
Browse files Browse the repository at this point in the history
  • Loading branch information
isoos authored Dec 16, 2024
1 parent 878d0f2 commit 537363a
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 20 deletions.
5 changes: 3 additions & 2 deletions app/lib/fake/backend/fake_download_counts.dart
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,20 @@ import 'dart:io';
import 'package:gcloud/storage.dart';
import 'package:pub_dev/service/download_counts/computations.dart';
import 'package:pub_dev/shared/configuration.dart';
import 'package:pub_dev/shared/storage.dart';
import 'package:pub_dev/shared/utils.dart';

Future<void> generateFakeDownloadCounts(
String downloadCountsFileName, String dataFilePath) async {
final file = File(dataFilePath).readAsBytesSync();
await storageService
.bucket(activeConfiguration.downloadCountsBucketName!)
.writeBytes(downloadCountsFileName, file);
.writeBytesWithRetry(downloadCountsFileName, file);
}

Future<void> generateFake30DaysTotals(Map<String, int> totals) async {
await storageService
.bucket(activeConfiguration.reportsBucketName!)
.writeBytes(
.writeBytesWithRetry(
downloadCounts30DaysTotalsFileName, jsonUtf8Encoder.convert(totals));
}
3 changes: 2 additions & 1 deletion app/lib/fake/backend/fake_topics.dart
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@ import 'package:gcloud/storage.dart';

import '../../service/topics/count_topics.dart';
import '../../shared/configuration.dart';
import '../../shared/storage.dart';
import '../../shared/utils.dart';

Future<void> generateFakeTopicValues() async {
await storageService
.bucket(activeConfiguration.reportsBucketName!)
.writeBytes(topicsJsonFileName,
.writeBytesWithRetry(topicsJsonFileName,
jsonUtf8Encoder.convert({'ffi': 7, 'ui': 5, 'network': 6}));
}
12 changes: 6 additions & 6 deletions app/lib/package/tarball_storage.dart
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ class TarballStorage {
required String package,
required String version,
}) async {
await _storage.copyObject(
await _storage.copyObjectWithRetry(
sourceAbsoluteObjectName,
_canonicalBucket.absoluteObjectName(tarballObjectName(package, version)),
);
Expand All @@ -157,7 +157,7 @@ class TarballStorage {
Future<void> copyArchiveFromCanonicalToPublicBucket(
String package, String version) async {
final objectName = tarballObjectName(package, version);
await _storage.copyObject(
await _storage.copyObjectWithRetry(
_canonicalBucket.absoluteObjectName(objectName),
_publicBucket.absoluteObjectName(objectName),
);
Expand Down Expand Up @@ -186,7 +186,7 @@ class TarballStorage {
final objectName = tarballObjectName(package, version);
final info = await _canonicalBucket.tryInfo(objectName);
if (info != null) {
await _canonicalBucket.delete(objectName);
await _canonicalBucket.deleteWithRetry(objectName);
}
}

Expand Down Expand Up @@ -232,11 +232,11 @@ class TarballStorage {
_logger
.warning('Updating missing object in public bucket: $objectName');
try {
await _storage.copyObject(
await _storage.copyObjectWithRetry(
_canonicalBucket.absoluteObjectName(objectName),
_publicBucket.absoluteObjectName(objectName),
);
final newInfo = await _publicBucket.info(objectName);
final newInfo = await _publicBucket.infoWithRetry(objectName);
await updateContentDispositionToAttachment(newInfo, _publicBucket);
updatedCount++;
} on Exception catch (e, st) {
Expand Down Expand Up @@ -307,7 +307,7 @@ class TarballStorage {

for (final objectName in deleteObjects) {
_logger.shout('Deleting object from public bucket: "$objectName".');
await _publicBucket.delete(objectName);
await _publicBucket.deleteWithRetry(objectName);
}

return PublicBucketUpdateStat(
Expand Down
21 changes: 21 additions & 0 deletions app/lib/shared/storage.dart
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,27 @@ extension BucketExt on Bucket {
),
);
}

/// Create an new object in the bucket with specified content with the default retry.
Future<ObjectInfo> writeBytesWithRetry(
String name,
List<int> bytes, {
ObjectMetadata? metadata,
Acl? acl,
PredefinedAcl? predefinedAcl,
String? contentType,
}) async {
return await _retry(
() async => await writeBytes(
name,
bytes,
metadata: metadata,
acl: acl,
predefinedAcl: predefinedAcl,
contentType: contentType,
),
);
}
}

extension PageExt<T> on Page<T> {
Expand Down
11 changes: 0 additions & 11 deletions pkg/fake_gcloud/lib/retry_enforcer_storage.dart
Original file line number Diff line number Diff line change
Expand Up @@ -59,23 +59,20 @@ class RetryEnforcerStorage implements Storage {
Future<bool> bucketExists(String bucketName) async {
return await _verifyRetry(
() => _storage.bucketExists(bucketName),
ignore: true,
);
}

@override
Future<BucketInfo> bucketInfo(String bucketName) async {
return await _verifyRetry(
() async => await _storage.bucketInfo(bucketName),
ignore: true,
);
}

@override
Future copyObject(String src, String dest, {ObjectMetadata? metadata}) async {
return await _verifyRetry(
() => _storage.copyObject(src, dest, metadata: metadata),
ignore: true,
);
}

Expand All @@ -91,15 +88,13 @@ class RetryEnforcerStorage implements Storage {
predefinedAcl: predefinedAcl,
acl: acl,
),
ignore: true,
);
}

@override
Future deleteBucket(String bucketName) async {
return await _verifyRetry(
() => _storage.deleteBucket(bucketName),
ignore: true,
);
}

Expand All @@ -112,7 +107,6 @@ class RetryEnforcerStorage implements Storage {
Future<Page<String>> pageBucketNames({int pageSize = 50}) async {
return await _verifyRetry(
() => _storage.pageBucketNames(pageSize: pageSize),
ignore: true,
);
}
}
Expand All @@ -134,15 +128,13 @@ class _RetryEnforcerBucket implements Bucket {
Future delete(String name) async {
return await _verifyRetry(
() async => await _bucket.delete(name),
ignore: true,
);
}

@override
Future<ObjectInfo> info(String name) async {
return await _verifyRetry(
() async => await _bucket.info(name),
ignore: true,
);
}

Expand All @@ -167,7 +159,6 @@ class _RetryEnforcerBucket implements Bucket {
delimiter: delimiter,
pageSize: pageSize,
)),
ignore: true,
);
}

Expand Down Expand Up @@ -222,7 +213,6 @@ class _RetryEnforcerBucket implements Bucket {
predefinedAcl: predefinedAcl,
contentType: contentType,
),
ignore: true,
);
}
}
Expand All @@ -241,7 +231,6 @@ class _RetryEnforcerPage<T> implements Page<T> {
Future<Page<T>> next({int pageSize = 50}) async {
return await _verifyRetry(
() async => _RetryEnforcerPage(await _page.next(pageSize: pageSize)),
ignore: true,
);
}
}

0 comments on commit 537363a

Please sign in to comment.