diff --git a/app/lib/search/dart_sdk_mem_index.dart b/app/lib/search/dart_sdk_mem_index.dart index dccf1a6173..4ecfb10536 100644 --- a/app/lib/search/dart_sdk_mem_index.dart +++ b/app/lib/search/dart_sdk_mem_index.dart @@ -7,10 +7,8 @@ import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; import 'package:pub_dartdoc_data/dartdoc_index.dart'; -import '../shared/cached_value.dart'; import 'backend.dart'; import 'sdk_mem_index.dart'; -import 'search_service.dart'; final _logger = Logger('search.dart_sdk_mem_index'); @@ -23,47 +21,22 @@ const dartSdkLibraryWeights = { }; /// Sets the Dart SDK in-memory index. -void registerDartSdkMemIndex(DartSdkMemIndex updater) => - ss.register(#_dartSdkMemIndex, updater); - -/// The active Dart SDK in-memory index. -DartSdkMemIndex get dartSdkMemIndex => - ss.lookup(#_dartSdkMemIndex) as DartSdkMemIndex; - -/// Dart SDK in-memory index that fetches `index.json` from -/// api.dart.dev and returns search results based on [SdkMemIndex]. -class DartSdkMemIndex { - final _index = CachedValue( - name: 'dart-sdk-index', - interval: Duration(days: 1), - maxAge: Duration(days: 30), - timeout: Duration(hours: 1), - updateFn: _createDartSdkMemIndex, - ); - - Future start() async { - await _index.start(); - } - - Future close() async { - await _index.close(); - } - - List search(String query, {int? limit}) { - if (!_index.isAvailable) return []; - return _index.value!.search(query, limit: limit); - } - - @visibleForTesting - void setDartdocIndex(DartdocIndex index, {String? version}) { - final smi = SdkMemIndex.dart(version: version); - smi.addDartdocIndex(index); - // ignore: invalid_use_of_visible_for_testing_member - _index.setValue(smi); +void registerDartSdkMemIndex(SdkMemIndex? index) { + if (index != null) { + ss.register(#_dartSdkMemIndex, index); } } -Future _createDartSdkMemIndex() async { +/// The active Dart SDK in-memory index. +SdkMemIndex? get dartSdkMemIndex => + ss.lookup(#_dartSdkMemIndex) as SdkMemIndex?; + +/// Tries to load Dart SDK's dartdoc `index.json` and build +/// a search index from it. +/// +/// Returns `null` when the loading of `index.json` failed, or when there +/// was an error parsing the file or building the index. +Future createDartSdkMemIndex() async { try { final index = SdkMemIndex.dart(); final content = DartdocIndex.parseJsonText( diff --git a/app/lib/search/flutter_sdk_mem_index.dart b/app/lib/search/flutter_sdk_mem_index.dart index a441d2a845..445acd2489 100644 --- a/app/lib/search/flutter_sdk_mem_index.dart +++ b/app/lib/search/flutter_sdk_mem_index.dart @@ -6,10 +6,8 @@ import 'package:gcloud/service_scope.dart' as ss; import 'package:logging/logging.dart'; import 'package:pub_dartdoc_data/dartdoc_index.dart'; -import '../shared/cached_value.dart'; import 'backend.dart'; import 'sdk_mem_index.dart'; -import 'search_service.dart'; /// The index.json file contains overlap with the Dart SDK and also repeats /// regular packages. The selected libraries are unique to the index.json. @@ -43,39 +41,19 @@ const _flutterApiPageDirWeights = { final _logger = Logger('search.flutter_sdk_mem_index'); /// Sets the Flutter SDK in-memory index. -void registerFlutterSdkMemIndex(FlutterSdkMemIndex updater) => - ss.register(#_flutterSdkMemIndex, updater); +void registerFlutterSdkMemIndex(SdkMemIndex? index) { + if (index != null) { + ss.register(#_flutterSdkMemIndex, index); + } +} /// The active Flutter SDK in-memory index. -FlutterSdkMemIndex get flutterSdkMemIndex => - ss.lookup(#_flutterSdkMemIndex) as FlutterSdkMemIndex; +SdkMemIndex? get flutterSdkMemIndex => + ss.lookup(#_flutterSdkMemIndex) as SdkMemIndex?; -/// Flutter SDK in-memory index that fetches `index.json` from +/// Creates Flutter SDK in-memory index that fetches `index.json` from /// api.flutter.dev and returns search results based on [SdkMemIndex]. -class FlutterSdkMemIndex { - final _index = CachedValue( - name: 'flutter-sdk-index', - interval: Duration(days: 1), - maxAge: Duration(days: 30), - timeout: Duration(hours: 1), - updateFn: _createFlutterSdkMemIndex, - ); - - Future start() async { - await _index.start(); - } - - Future close() async { - await _index.close(); - } - - List search(String query, {int? limit}) { - if (!_index.isAvailable) return []; - return _index.value!.search(query, limit: limit); - } -} - -Future _createFlutterSdkMemIndex() async { +Future createFlutterSdkMemIndex() async { try { final index = SdkMemIndex.flutter(); final content = DartdocIndex.parseJsonText( diff --git a/app/lib/search/result_combiner.dart b/app/lib/search/result_combiner.dart index 6e4d95b219..1ce2ea33eb 100644 --- a/app/lib/search/result_combiner.dart +++ b/app/lib/search/result_combiner.dart @@ -5,18 +5,17 @@ import 'dart:math' as math; import 'package:_pub_shared/search/tags.dart'; -import 'package:pub_dev/search/mem_index.dart'; -import 'dart_sdk_mem_index.dart'; -import 'flutter_sdk_mem_index.dart'; +import 'mem_index.dart'; +import 'sdk_mem_index.dart'; import 'search_service.dart'; /// Combines the results from the primary package index and the optional Dart /// SDK index. class SearchResultCombiner { final InMemoryPackageIndex primaryIndex; - final DartSdkMemIndex dartSdkMemIndex; - final FlutterSdkMemIndex flutterSdkMemIndex; + final SdkMemIndex? dartSdkMemIndex; + final SdkMemIndex? flutterSdkMemIndex; SearchResultCombiner({ required this.primaryIndex, @@ -33,8 +32,9 @@ class SearchResultCombiner { final queryFlutterSdk = query.tagsPredicate.hasNoTagPrefix('sdk:') || query.tagsPredicate.hasTag(SdkTag.sdkFlutter); final sdkLibraryHits = [ - ...dartSdkMemIndex.search(query.query!, limit: 2), - if (queryFlutterSdk) ...flutterSdkMemIndex.search(query.query!, limit: 2), + ...?dartSdkMemIndex?.search(query.query!, limit: 2), + if (queryFlutterSdk) + ...?flutterSdkMemIndex?.search(query.query!, limit: 2), ]; if (sdkLibraryHits.isNotEmpty) { // Do not display low SDK scores if all the first page package hits are more relevant. diff --git a/app/lib/service/entrypoint/search_index.dart b/app/lib/service/entrypoint/search_index.dart index 6537cc320b..3b31a2177e 100644 --- a/app/lib/service/entrypoint/search_index.dart +++ b/app/lib/service/entrypoint/search_index.dart @@ -36,8 +36,8 @@ Future main(List args, var message) async { } await fork(() async { await servicesWrapperFn(() async { - await dartSdkMemIndex.start(); - await flutterSdkMemIndex.start(); + registerDartSdkMemIndex(await createDartSdkMemIndex()); + registerFlutterSdkMemIndex(await createFlutterSdkMemIndex()); await indexUpdater.init(); final requestReceivePort = ReceivePort(); diff --git a/app/lib/service/services.dart b/app/lib/service/services.dart index 69d10a2db8..979bd85812 100644 --- a/app/lib/service/services.dart +++ b/app/lib/service/services.dart @@ -40,8 +40,6 @@ import '../publisher/backend.dart'; import '../publisher/domain_verifier.dart'; import '../scorecard/backend.dart'; import '../search/backend.dart'; -import '../search/dart_sdk_mem_index.dart'; -import '../search/flutter_sdk_mem_index.dart'; import '../search/search_client.dart'; import '../search/top_packages.dart'; import '../search/updater.dart'; @@ -232,9 +230,7 @@ Future _withPubServices(FutureOr Function() fn) async { registerAuditBackend(AuditBackend(dbService)); registerConsentBackend(ConsentBackend(dbService)); registerDartdocBackend(DartdocBackend()); - registerDartSdkMemIndex(DartSdkMemIndex()); registerEmailBackend(EmailBackend(dbService)); - registerFlutterSdkMemIndex(FlutterSdkMemIndex()); registerLikeBackend(LikeBackend(dbService)); registerNameTracker(NameTracker(dbService)); registerPackageIndexHolder(PackageIndexHolder()); @@ -275,8 +271,6 @@ Future _withPubServices(FutureOr Function() fn) async { registerScopeExitCallback(announcementBackend.close); registerScopeExitCallback(searchBackend.close); registerScopeExitCallback(() async => nameTracker.stopTracking()); - registerScopeExitCallback(dartSdkMemIndex.close); - registerScopeExitCallback(flutterSdkMemIndex.close); registerScopeExitCallback(popularityStorage.close); registerScopeExitCallback(scoreCardBackend.close); registerScopeExitCallback(searchClient.close); diff --git a/app/test/search/result_combiner_test.dart b/app/test/search/result_combiner_test.dart index 101300c9fe..a543cbb111 100644 --- a/app/test/search/result_combiner_test.dart +++ b/app/test/search/result_combiner_test.dart @@ -6,10 +6,9 @@ import 'dart:convert'; import 'package:_pub_shared/search/search_form.dart'; import 'package:pub_dartdoc_data/dartdoc_index.dart'; -import 'package:pub_dev/search/dart_sdk_mem_index.dart'; -import 'package:pub_dev/search/flutter_sdk_mem_index.dart'; import 'package:pub_dev/search/mem_index.dart'; import 'package:pub_dev/search/result_combiner.dart'; +import 'package:pub_dev/search/sdk_mem_index.dart'; import 'package:pub_dev/search/search_service.dart'; import 'package:test/test.dart'; @@ -28,58 +27,52 @@ void main() { ), ], ); - final dartSdkMemIndex = DartSdkMemIndex(); - final flutterSdkMemIndex = FlutterSdkMemIndex(); final combiner = SearchResultCombiner( primaryIndex: primaryIndex, - dartSdkMemIndex: dartSdkMemIndex, - flutterSdkMemIndex: flutterSdkMemIndex, + dartSdkMemIndex: SdkMemIndex.dart() + ..addDartdocIndex( + DartdocIndex.fromJsonList([ + { + 'name': 'dart:core', + 'qualifiedName': 'dart:core', + 'href': 'dart-core/dart-core-library.html', + 'kind': 8, + 'overriddenDepth': 0, + 'packageName': 'Dart' + }, + { + 'name': 'String', + 'qualifiedName': 'dart:core.String', + 'href': 'dart-core/String-class.html', + 'kind': 3, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'enclosedBy': {'name': 'dart:core', 'kind': 8} + }, + { + 'name': 'substring', + 'qualifiedName': 'dart:core.String.substring', + 'href': 'dart-core/String/substring.html', + 'kind': 9, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'enclosedBy': {'name': 'String', 'kind': 3} + }, + { + // fake method for checking the package name matches + 'name': 'stringutils', + 'qualifiedName': 'dart:core.String.stringutils', + 'href': 'dart-core/String/stringutils.html', + 'kind': 9, + 'overriddenDepth': 0, + 'packageName': 'Dart', + 'enclosedBy': {'name': 'String', 'kind': 3} + }, + ]), + ), + flutterSdkMemIndex: null, ); - setUpAll(() async { - dartSdkMemIndex.setDartdocIndex( - DartdocIndex.fromJsonList([ - { - 'name': 'dart:core', - 'qualifiedName': 'dart:core', - 'href': 'dart-core/dart-core-library.html', - 'kind': 8, - 'overriddenDepth': 0, - 'packageName': 'Dart' - }, - { - 'name': 'String', - 'qualifiedName': 'dart:core.String', - 'href': 'dart-core/String-class.html', - 'kind': 3, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'enclosedBy': {'name': 'dart:core', 'kind': 8} - }, - { - 'name': 'substring', - 'qualifiedName': 'dart:core.String.substring', - 'href': 'dart-core/String/substring.html', - 'kind': 9, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'enclosedBy': {'name': 'String', 'kind': 3} - }, - { - // fake method for checking the package name matches - 'name': 'stringutils', - 'qualifiedName': 'dart:core.String.stringutils', - 'href': 'dart-core/String/stringutils.html', - 'kind': 9, - 'overriddenDepth': 0, - 'packageName': 'Dart', - 'enclosedBy': {'name': 'String', 'kind': 3} - }, - ]), - version: '2.0.0', - ); - }); - test('non-text ranking', () async { final results = combiner .search(ServiceSearchQuery.parse(order: SearchOrder.popularity)); @@ -115,16 +108,14 @@ void main() { 'sdkLibraryHits': [ { 'sdk': 'dart', - 'version': '2.0.0', + 'version': isNotEmpty, 'library': 'dart:core', - 'url': - 'https://api.dart.dev/stable/2.0.0/dart-core/dart-core-library.html', + 'url': contains('dart-core-library.html'), 'score': closeTo(0.98, 0.01), 'apiPages': [ { 'path': 'dart-core/String/substring.html', - 'url': - 'https://api.dart.dev/stable/2.0.0/dart-core/String/substring.html' + 'url': contains('substring.html'), } ] },