Cleanup leftover popularity-related methods and names. (#8481)

dart-lang · Jan 22, 2025 · 80d23b5 · 80d23b5
1 parent b38cc78
commit 80d23b5
Show file tree

Hide file tree

Showing 11 changed files with 57 additions and 61 deletions.
diff --git a/app/lib/fake/backend/fake_download_counts.dart b/app/lib/fake/backend/fake_download_counts.dart
@@ -3,14 +3,38 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:io';
+import 'dart:math' as math;
 
+import 'package:clock/clock.dart';
 import 'package:gcloud/storage.dart';
+import 'package:pub_dev/package/models.dart';
+import 'package:pub_dev/service/download_counts/backend.dart';
 import 'package:pub_dev/service/download_counts/computations.dart';
 import 'package:pub_dev/shared/configuration.dart';
+import 'package:pub_dev/shared/datastore.dart';
 import 'package:pub_dev/shared/storage.dart';
 import 'package:pub_dev/shared/utils.dart';
 
-Future<void> generateFakeDownloadCounts(
+/// Scans the datastore for packages and generates download count values with a
+/// deterministic random seed.
+///
+/// TODO: generate download counts in bucket first
+Future<void> generateFakeDownloadCountsInDatastore() async {
+  final query = dbService.query<Package>();
+  await for (final p in query.run()) {
+    final r = math.Random(p.name.hashCode.abs());
+    final count = (math.min(p.likes * p.likes, 50) + r.nextInt(50));
+    await downloadCountsBackend.updateDownloadCounts(
+      p.name!,
+      {
+        p.latestVersion!: count,
+      },
+      clock.now(),
+    );
+  }
+}
+
+Future<void> uploadFakeDownloadCountsToBucket(
     String downloadCountsFileName, String dataFilePath) async {
   final file = File(dataFilePath).readAsBytesSync();
   await storageService

diff --git a/app/lib/fake/backend/fake_popularity.dart b/app/lib/fake/backend/fake_popularity.dart
diff --git a/app/lib/fake/server/fake_analyzer_service.dart b/app/lib/fake/server/fake_analyzer_service.dart
@@ -10,7 +10,7 @@ import 'package:fake_gcloud/mem_storage.dart';
 import 'package:gcloud/service_scope.dart' as ss;
 import 'package:logging/logging.dart';
 import 'package:pub_dev/analyzer/handlers.dart';
-import 'package:pub_dev/fake/backend/fake_popularity.dart';
+import 'package:pub_dev/fake/backend/fake_download_counts.dart';
 import 'package:pub_dev/service/services.dart';
 import 'package:pub_dev/shared/configuration.dart';
 import 'package:pub_dev/shared/handler_helpers.dart';
@@ -38,7 +38,7 @@ class FakeAnalyzerService {
         storage: _storage,
         cloudCompute: _cloudCompute,
         fn: () async {
-          await generateFakeDownloadCounts();
+          await generateFakeDownloadCountsInDatastore();
 
           final handler = wrapHandler(_logger, analyzerServiceHandler);
           final server = await IOServer.bind('localhost', port);

diff --git a/app/lib/fake/server/fake_default_service.dart b/app/lib/fake/server/fake_default_service.dart
@@ -8,7 +8,7 @@ import 'package:fake_gcloud/mem_datastore.dart';
 import 'package:fake_gcloud/mem_storage.dart';
 import 'package:gcloud/service_scope.dart' as ss;
 import 'package:logging/logging.dart';
-import 'package:pub_dev/fake/backend/fake_popularity.dart';
+import 'package:pub_dev/fake/backend/fake_download_counts.dart';
 import 'package:pub_dev/fake/backend/fake_topics.dart';
 import 'package:pub_dev/frontend/handlers.dart';
 import 'package:pub_dev/package/name_tracker.dart';
@@ -48,7 +48,7 @@ class FakePubServer {
             await watchForResourceChanges();
           }
 
-          await generateFakeDownloadCounts();
+          await generateFakeDownloadCountsInDatastore();
           await generateFakeTopicValues();
           await nameTracker.startTracking();
 

diff --git a/app/lib/fake/server/fake_search_service.dart b/app/lib/fake/server/fake_search_service.dart
@@ -9,7 +9,7 @@ import 'package:fake_gcloud/mem_datastore.dart';
 import 'package:fake_gcloud/mem_storage.dart';
 import 'package:gcloud/service_scope.dart' as ss;
 import 'package:logging/logging.dart';
-import 'package:pub_dev/fake/backend/fake_popularity.dart';
+import 'package:pub_dev/fake/backend/fake_download_counts.dart';
 import 'package:pub_dev/search/handlers.dart';
 import 'package:pub_dev/search/updater.dart';
 import 'package:pub_dev/service/services.dart';
@@ -54,7 +54,7 @@ class FakeSearchService {
           });
           _logger.info('running on port $port');
 
-          await generateFakeDownloadCounts();
+          await generateFakeDownloadCountsInDatastore();
           // ignore: invalid_use_of_visible_for_testing_member
           await indexUpdater.updateAllPackages();
 

diff --git a/app/lib/frontend/templates/views/landing/page.dart b/app/lib/frontend/templates/views/landing/page.dart
@@ -46,7 +46,7 @@ d.Node landingPageNode({
         info: d
             .text('Some of the most downloaded packages over the past 60 days'),
         content: miniListNode('most-popular', mostPopularPackages!),
-        viewAllUrl: urls.listingByPopularity(),
+        viewAllUrl: urls.listingByDownloadCounts(),
         viewAllEvent: 'landing-most-popular-view-all',
         viewAllTitle: 'Search popular packages',
       ),

diff --git a/app/lib/search/mem_index.dart b/app/lib/search/mem_index.dart
@@ -313,9 +313,9 @@ class InMemoryPackageIndex {
     _adjustedOverallScores = _documents.map((doc) {
       final downloadScore = doc.downloadScore ?? 0.0;
       final likeScore = doc.likeScore ?? 0.0;
-      final popularity = (downloadScore + likeScore) / 2;
+      final combinedScore = (downloadScore + likeScore) / 2;
       final points = doc.grantedPoints / math.max(1, doc.maxPoints);
-      final overall = popularity * 0.5 + points * 0.5;
+      final overall = combinedScore * 0.5 + points * 0.5;
       doc.overallScore = overall;
       // adding a base score prevents later multiplication with zero
       return 0.4 + 0.6 * overall;

diff --git a/app/lib/search/models.dart b/app/lib/search/models.dart
@@ -33,7 +33,7 @@ class SearchSnapshot {
   }
 
   /// Updates the [PackageDocument] instance's scores for each package in the snapshot.
-  /// Sets `downloadScore`, `likeScore` and `popularityScore` fields, normalized into the
+  /// Sets `downloadScore` and `likeScore` fields, normalized into the
   /// range of [0.0 - 1.0] using the ordered list of their specific counts.
   void updateAllScores() {
     /// Updates the PackageDocument.downloadScore for each package in the snapshot.

diff --git a/app/lib/shared/urls.dart b/app/lib/shared/urls.dart
@@ -199,7 +199,7 @@ String searchUrl({
   return SearchForm(query: q).toSearchLink(page: page);
 }
 
-String listingByPopularity() =>
+String listingByDownloadCounts() =>
     SearchForm(order: SearchOrder.downloads).toSearchLink();
 
 String dartSdkMainUrl(String version) {

diff --git a/app/test/service/download_counts/download_counts_test.dart b/app/test/service/download_counts/download_counts_test.dart
@@ -51,7 +51,7 @@ void main() {
       final date = DateTime.parse('2024-01-05');
       final downloadCountsJsonFileName =
           'daily_download_counts/2024-01-05T00:00:00Z/data-000000000000.jsonl';
-      await generateFakeDownloadCounts(
+      await uploadFakeDownloadCountsToBucket(
           downloadCountsJsonFileName,
           path.join(Directory.current.path, 'test', 'service',
               'download_counts', 'fake_download_counts_data.jsonl'));
@@ -72,7 +72,7 @@ void main() {
       final date = DateTime.parse('2024-01-05');
       final downloadCountsJsonFileNameJan5 =
           'daily_download_counts/2024-01-05T00:00:00Z/data-000000000000.jsonl';
-      await generateFakeDownloadCounts(
+      await uploadFakeDownloadCountsToBucket(
           downloadCountsJsonFileNameJan5,
           path.join(Directory.current.path, 'test', 'service',
               'download_counts', 'fake_download_counts_data.jsonl'));
@@ -81,7 +81,7 @@ void main() {
       final nextDate = DateTime.parse('2024-01-06');
       final downloadCountsJsonFileNameJan6 =
           'daily_download_counts/2024-01-06T00:00:00Z/data-000000000000.jsonl';
-      await generateFakeDownloadCounts(
+      await uploadFakeDownloadCountsToBucket(
           downloadCountsJsonFileNameJan6,
           path.join(
               Directory.current.path,
@@ -113,7 +113,7 @@ void main() {
       final nextDate = DateTime.parse('2024-01-06');
       final downloadCountsJsonFileNameJan6 =
           'daily_download_counts/2024-01-06T00:00:00Z/data-000000000000.jsonl';
-      await generateFakeDownloadCounts(
+      await uploadFakeDownloadCountsToBucket(
           downloadCountsJsonFileNameJan6,
           path.join(
               Directory.current.path,
@@ -150,7 +150,7 @@ void main() {
       final nextDate = DateTime.parse('2024-01-06');
       final downloadCountsJsonFileNameJan6 =
           'daily_download_counts/2024-01-06T00:00:00Z/data-000000000000.jsonl';
-      await generateFakeDownloadCounts(
+      await uploadFakeDownloadCountsToBucket(
           downloadCountsJsonFileNameJan6,
           path.join(
               Directory.current.path,
@@ -202,7 +202,7 @@ void main() {
       final nextDate = DateTime.parse('2024-01-06');
       final downloadCountsJsonFileNameJan6 =
           'daily_download_counts/2024-01-06T00:00:00Z/data-000000000000.jsonl';
-      await generateFakeDownloadCounts(
+      await uploadFakeDownloadCountsToBucket(
           downloadCountsJsonFileNameJan6,
           path.join(Directory.current.path, 'test', 'service',
               'download_counts', 'fake_download_counts_data_empty.jsonl'));
@@ -234,7 +234,7 @@ void main() {
           formatDateForFileName(date),
           'data-000000000000.jsonl',
         ].join('/');
-        await generateFakeDownloadCounts(
+        await uploadFakeDownloadCountsToBucket(
           fileName,
           path.join(Directory.current.path, 'test', 'service',
               'download_counts', 'fake_download_counts_data.jsonl'),
@@ -268,7 +268,7 @@ void main() {
           formatDateForFileName(date),
           'data-000000000000.jsonl',
         ].join('/');
-        await generateFakeDownloadCounts(
+        await uploadFakeDownloadCountsToBucket(
           fileName,
           path.join(Directory.current.path, 'test', 'service',
               'download_counts', 'fake_download_counts_data.jsonl'),
@@ -302,7 +302,7 @@ void main() {
           formatDateForFileName(date),
           'data-000000000000.jsonl',
         ].join('/');
-        await generateFakeDownloadCounts(
+        await uploadFakeDownloadCountsToBucket(
           fileName,
           path.join(Directory.current.path, 'test', 'service',
               'download_counts', 'fake_download_counts_data.jsonl'),
@@ -356,7 +356,7 @@ void main() {
           formatDateForFileName(date),
           'data-000000000000.jsonl',
         ].join('/');
-        await generateFakeDownloadCounts(
+        await uploadFakeDownloadCountsToBucket(
           fileName,
           path.join(Directory.current.path, 'test', 'service',
               'download_counts', 'fake_download_counts_data.jsonl'),
@@ -422,13 +422,13 @@ void main() {
           'data-000000000001.jsonl',
         ].join('/');
 
-        await generateFakeDownloadCounts(
+        await uploadFakeDownloadCountsToBucket(
           fileName,
           path.join(Directory.current.path, 'test', 'service',
               'download_counts', 'fake_download_counts_data.jsonl'),
         );
 
-        await generateFakeDownloadCounts(
+        await uploadFakeDownloadCountsToBucket(
           fileName1,
           path.join(Directory.current.path, 'test', 'service',
               'download_counts', 'fake_download_counts_data1.jsonl'),
@@ -486,13 +486,13 @@ void main() {
         'data-000000000001.jsonl',
       ].join('/');
 
-      await generateFakeDownloadCounts(
+      await uploadFakeDownloadCountsToBucket(
         fileName,
         path.join(Directory.current.path, 'test', 'service', 'download_counts',
             'fake_download_counts_data.jsonl'),
       );
 
-      await generateFakeDownloadCounts(
+      await uploadFakeDownloadCountsToBucket(
         fileName1,
         path.join(Directory.current.path, 'test', 'service', 'download_counts',
             'fake_download_counts_data_empty.jsonl'),
@@ -555,13 +555,13 @@ void main() {
       'data-000000000001.jsonl',
     ].join('/');
 
-    await generateFakeDownloadCounts(
+    await uploadFakeDownloadCountsToBucket(
       fileName,
       path.join(Directory.current.path, 'test', 'service', 'download_counts',
           'fake_download_counts_data.jsonl'),
     );
 
-    await generateFakeDownloadCounts(
+    await uploadFakeDownloadCountsToBucket(
       fileName1,
       path.join(Directory.current.path, 'test', 'service', 'download_counts',
           'fake_download_counts_data1.jsonl'),
@@ -579,13 +579,13 @@ void main() {
       'data-000000000001.jsonl',
     ].join('/');
 
-    await generateFakeDownloadCounts(
+    await uploadFakeDownloadCountsToBucket(
       fileName2,
       path.join(Directory.current.path, 'test', 'service', 'download_counts',
           'fake_download_counts_data_empty.jsonl'),
     );
 
-    await generateFakeDownloadCounts(
+    await uploadFakeDownloadCountsToBucket(
       fileName3,
       path.join(Directory.current.path, 'test', 'service', 'download_counts',
           'fake_download_counts_data_empty.jsonl'),

diff --git a/app/test/shared/test_services.dart b/app/test/shared/test_services.dart
@@ -11,8 +11,8 @@ import 'package:gcloud/service_scope.dart';
 import 'package:meta/meta.dart';
 import 'package:pub_dev/account/models.dart';
 import 'package:pub_dev/fake/backend/fake_auth_provider.dart';
+import 'package:pub_dev/fake/backend/fake_download_counts.dart';
 import 'package:pub_dev/fake/backend/fake_email_sender.dart';
-import 'package:pub_dev/fake/backend/fake_popularity.dart';
 import 'package:pub_dev/fake/backend/fake_pub_worker.dart';
 import 'package:pub_dev/frontend/handlers/pubapi.client.dart';
 import 'package:pub_dev/frontend/static_files.dart';
@@ -107,7 +107,7 @@ class FakeAppengineEnv {
             );
           }
           if (processJobsWithFakeRunners) {
-            await generateFakeDownloadCounts();
+            await generateFakeDownloadCountsInDatastore();
             await processTasksWithFakePanaAndDartdoc();
           }
           await nameTracker.reloadFromDatastore();
@@ -211,7 +211,7 @@ void testWithFakeTime(
             source: importSource ?? ImportSource.autoGenerated(),
           );
           await nameTracker.reloadFromDatastore();
-          await generateFakeDownloadCounts();
+          await generateFakeDownloadCountsInDatastore();
           await indexUpdater.updateAllPackages();
           await asyncQueue.ongoingProcessing;
           fakeEmailSender.sentMessages.clear();