From e5889f79767e1b412957471337d98ad2cfff9508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20So=C3=B3s?= Date: Wed, 4 Dec 2024 17:11:12 +0100 Subject: [PATCH] Migrate isBlocked flags to isModerated. (#8356) --- CHANGELOG.md | 1 + .../tool/backfill/backfill_new_fields.dart | 76 ++++++++++++++++++- app/test/shared/test_services.dart | 10 ++- .../maintenance/migrate_isblocked_test.dart | 52 +++++++++++++ pkg/fake_gcloud/lib/mem_datastore.dart | 3 + 5 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 app/test/tool/maintenance/migrate_isblocked_test.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index 9402cd808b..604626b693 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ AppEngine version, listed here to ease deployment and troubleshooting. * Upgraded dartdoc to `8.3.0`. * Upgraded pana to `0.22.16`. * Upgraded dependencies. + * Note: started migrating `isBlocked` flags to `isModerated`. ## `20241121t150900-all` * Bump runtimeVersion to `2024.11.21`. diff --git a/app/lib/tool/backfill/backfill_new_fields.dart b/app/lib/tool/backfill/backfill_new_fields.dart index fc766f9875..2871f30097 100644 --- a/app/lib/tool/backfill/backfill_new_fields.dart +++ b/app/lib/tool/backfill/backfill_new_fields.dart @@ -2,7 +2,14 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'package:clock/clock.dart'; import 'package:logging/logging.dart'; +import 'package:pub_dev/account/models.dart'; +import 'package:pub_dev/package/api_export/api_exporter.dart'; +import 'package:pub_dev/package/backend.dart'; +import 'package:pub_dev/package/models.dart'; +import 'package:pub_dev/publisher/models.dart'; +import 'package:pub_dev/shared/datastore.dart'; final _logger = Logger('backfill_new_fields'); @@ -12,5 +19,72 @@ final _logger = Logger('backfill_new_fields'); /// CHANGELOG.md must be updated with the new fields, and the next /// release could remove the backfill from here. Future backfillNewFields() async { - _logger.info('Nothing to backfill'); + await migrateIsBlocked(); +} + +/// Migrates entities from the `isBlocked` fields to the new `isModerated` instead. +Future migrateIsBlocked() async { + _logger.info('Migrating isBlocked...'); + final pkgQuery = dbService.query()..filter('isBlocked =', true); + await for (final entity in pkgQuery.run()) { + await withRetryTransaction(dbService, (tx) async { + final pkg = await tx.lookupValue(entity.key); + // sanity check + if (!pkg.isBlocked) { + return; + } + pkg + ..isModerated = true + ..moderatedAt = pkg.moderatedAt ?? pkg.blocked ?? clock.now() + ..isBlocked = false + ..blocked = null + ..blockedReason = null; + tx.insert(pkg); + }); + + // sync exported API(s) + await apiExporter?.synchronizePackage(entity.name!, forceDelete: true); + + // retract or re-populate public archive files + await packageBackend.tarballStorage.updatePublicArchiveBucket( + package: entity.name!, + ageCheckThreshold: Duration.zero, + deleteIfOlder: Duration.zero, + ); + } + + final publisherQuery = dbService.query() + ..filter('isBlocked =', true); + await for (final entity in publisherQuery.run()) { + await withRetryTransaction(dbService, (tx) async { + final publisher = await tx.lookupValue(entity.key); + // sanity check + if (!publisher.isBlocked) { + return; + } + publisher + ..isModerated = true + ..moderatedAt = publisher.moderatedAt ?? clock.now() + ..isBlocked = false; + tx.insert(publisher); + }); + } + + final userQuery = dbService.query()..filter('isBlocked =', true); + await for (final entity in userQuery.run()) { + await withRetryTransaction(dbService, (tx) async { + final user = await tx.lookupValue(entity.key); + // sanity check + if (!user.isBlocked) { + return; + } + user + ..isModerated = true + ..moderatedAt = user.moderatedAt ?? clock.now() + ..isBlocked = false; + tx.insert(user); + }); + } + + _logger.info('isBlocked migration completed.'); } diff --git a/app/test/shared/test_services.dart b/app/test/shared/test_services.dart index 5fcbd06cf1..44444561a4 100644 --- a/app/test/shared/test_services.dart +++ b/app/test/shared/test_services.dart @@ -31,6 +31,7 @@ import 'package:pub_dev/shared/redis_cache.dart'; import 'package:pub_dev/shared/versions.dart'; import 'package:pub_dev/task/cloudcompute/fakecloudcompute.dart'; import 'package:pub_dev/task/global_lock.dart'; +import 'package:pub_dev/tool/backfill/backfill_new_fields.dart'; import 'package:pub_dev/tool/test_profile/import_source.dart'; import 'package:pub_dev/tool/test_profile/importer.dart'; import 'package:pub_dev/tool/test_profile/models.dart'; @@ -120,7 +121,6 @@ class FakeAppengineEnv { await fork(() async { await fn(); }); - // post-test integrity check final problems = await IntegrityChecker(dbService).findProblems().toList(); if (problems.isNotEmpty && @@ -131,6 +131,14 @@ class FakeAppengineEnv { } else if (problems.isEmpty && integrityProblem != null) { throw Exception('Integrity problem expected but not present.'); } + + // TODO: run all background tasks here + await backfillNewFields(); + + // re-run integrity checks on the updated state + final laterProblems = + await IntegrityChecker(dbService).findProblems().toList(); + expect(laterProblems, problems); }, ); }) as R; diff --git a/app/test/tool/maintenance/migrate_isblocked_test.dart b/app/test/tool/maintenance/migrate_isblocked_test.dart new file mode 100644 index 0000000000..3d59e13329 --- /dev/null +++ b/app/test/tool/maintenance/migrate_isblocked_test.dart @@ -0,0 +1,52 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:pub_dev/account/backend.dart'; +import 'package:pub_dev/package/backend.dart'; +import 'package:pub_dev/publisher/backend.dart'; +import 'package:pub_dev/shared/datastore.dart'; +import 'package:pub_dev/tool/backfill/backfill_new_fields.dart'; +import 'package:test/test.dart'; + +import '../../shared/test_services.dart'; + +void main() { + group('Migrate isBlocked', () { + testWithProfile('package', fn: () async { + final p1 = await packageBackend.lookupPackage('oxygen'); + await dbService.commit( + inserts: [p1!..updateIsBlocked(isBlocked: true, reason: 'abc')]); + await migrateIsBlocked(); + + final p2 = await packageBackend.lookupPackage('oxygen'); + expect(p2!.isModerated, true); + }); + + testWithProfile('publisher', fn: () async { + final p1 = await publisherBackend.getPublisher('example.com'); + await dbService.commit(inserts: [p1!..markForBlocked()]); + final members = + await publisherBackend.listPublisherMembers('example.com'); + for (final m in members) { + await accountBackend.updateBlockedFlag(m.userId, true); + } + final neon = await packageBackend.lookupPackage('neon'); + await dbService.commit(inserts: [neon!..isDiscontinued = true]); + + await migrateIsBlocked(); + + final p2 = await publisherBackend.getPublisher('example.com'); + expect(p2!.isModerated, true); + }); + + testWithProfile('user', fn: () async { + final u1 = await accountBackend.lookupUserByEmail('user@pub.dev'); + await dbService.commit(inserts: [u1..isBlocked = true]); + await migrateIsBlocked(); + + final u2 = await accountBackend.lookupUserByEmail('user@pub.dev'); + expect(u2.isModerated, true); + }); + }); +} diff --git a/pkg/fake_gcloud/lib/mem_datastore.dart b/pkg/fake_gcloud/lib/mem_datastore.dart index 106e1772c2..7738104167 100644 --- a/pkg/fake_gcloud/lib/mem_datastore.dart +++ b/pkg/fake_gcloud/lib/mem_datastore.dart @@ -172,6 +172,9 @@ class MemDatastore implements Datastore { return -1; } } + if (a is bool && b is bool) { + return a == b ? 0 : (a ? 1 : -1); + } if (a is Key && b is Key) { if (a.elements.length != 1) { throw UnimplementedError();