Merge pull request #1063 from multiversx/UpdateCollectionIndexing

Fix collections retrieve

src/common/services/mx-communication/elastic-collection.model.ts

@@ -0,0 +1,21 @@
+export interface CollectionElastic {
+  token: string;
+  name: string;
+  type: string;
+  ticker: string;
+  currentOwner: string;
+  timestamp: number;
+  properties: CollectionElasticProperties;
+  api_nftCount: number;
+  api_isVerified?: boolean;
+}
+
+export interface CollectionElasticProperties {
+  canTransferNFTCreateRole: boolean;
+  canPause: boolean;
+  canFreeze: boolean;
+  canWipe: boolean;
+  canMint: boolean;
+  canBurn: boolean;
+  canAddQuantity: boolean;
+}

src/modules/nftCollections/collections-getter.service.ts

@@ -1,10 +1,10 @@
 import { Injectable } from '@nestjs/common';
 import { orderBy } from 'lodash';
 import { Address } from '@multiversx/sdk-core';
-import { genericDescriptions } from 'src/config';
+import { constants, genericDescriptions } from 'src/config';
 import { Collection, CollectionAsset } from './models';
 import { CollectionQuery } from './collection-query';
-import { CollectionApi, MxApiService, MxIdentityService } from 'src/common';
+import { MxApiService, MxElasticService, MxIdentityService } from 'src/common';
 import { CacheInfo } from 'src/common/services/caching/entities/cache.info';
 import { CollectionsNftsCountRedisHandler } from './collection-nfts-count.redis-handler';
 import { CollectionsNftsRedisHandler } from './collection-nfts.redis-handler';
@@ -17,6 +17,10 @@ import { CollectionNftTrait } from '../nft-traits/models/collection-traits.model
 import { DocumentDbService } from 'src/document-db/document-db.service';
 import { BlacklistedCollectionsService } from '../blacklist/blacklisted-collections.service';
 import { TrendingCollectionsService } from '../analytics/trending/trending-collections.service';
+import { ELASTIC_TOKENS_INDEX } from 'src/utils/constants';
+import { ElasticQuery, QueryType, ElasticSortOrder } from '@multiversx/sdk-nestjs-elastic';
+import { NftTypeEnum } from '../assets/models';
+import { CollectionElastic } from 'src/common/services/mx-communication/elastic-collection.model';
 
 @Injectable()
 export class CollectionsGetterService {
@@ -31,6 +35,7 @@ export class CollectionsGetterService {
     private analyticsService: TrendingCollectionsService,
     private documentDbService: DocumentDbService,
     private blacklistedCollectionsService: BlacklistedCollectionsService,
+    private elasticService: MxElasticService,
   ) {}
 
   async getCollections(
@@ -238,25 +243,19 @@ export class CollectionsGetterService {
   }
 
   public async getFullCollectionsRaw(): Promise<[Collection[], number]> {
-    const size = 25;
-    let from = 0;
-
-    const totalCount = await this.apiService.getCollectionsCount('?type=NonFungibleESDT,SemiFungibleESDT');
+    const query = this.getCollectionQuery();
     let collectionsResponse: Collection[] = [];
-    do {
-      let mappedCollections = await this.getMappedCollections(from, size);
+    await this.elasticService.getScrollableList(ELASTIC_TOKENS_INDEX, 'token', query, async (items: CollectionElastic[]) => {
+      let mappedCollections = await this.getMappedCollections(items);
 
       mappedCollections = await this.mapCollectionNftsCount(mappedCollections);
 
       mappedCollections = await this.mapCollectionNfts(mappedCollections);
 
       collectionsResponse.push(...mappedCollections);
+    });
 
-      from = from + size;
-    } while (from < totalCount && from <= 9975);
     let uniqueCollections = [...new Map(collectionsResponse.map((item) => [item.collection, item])).values()];
-    uniqueCollections = orderBy(uniqueCollections, ['verified', 'creationDate'], ['desc', 'desc']);
-
     return [uniqueCollections, uniqueCollections?.length];
   }
 
@@ -309,21 +308,20 @@ export class CollectionsGetterService {
     return [groupedCollections, groupedCollections.length];
   }
 
-  private async getMappedCollections(page: number, size: number): Promise<Collection[]> {
-    const collections = await this.apiService.getCollections(new CollectionQuery().addPageSize(page, size).build());
-    const promisesCollections = collections?.map((collection): Promise<Collection> => this.mapCollection(collection));
+  private async getMappedCollections(items: CollectionElastic[]): Promise<Collection[]> {
+    const promisesCollections = items?.map((collection): Promise<Collection> => this.mapCollection(collection));
     return await Promise.all(promisesCollections);
   }
 
-  private async mapCollection(collection: CollectionApi): Promise<Collection> {
-    const ownerAddress = new Address(collection.owner);
+  private async mapCollection(collection: CollectionElastic): Promise<Collection> {
+    const ownerAddress = new Address(collection.currentOwner);
     if (ownerAddress.isContractAddress()) {
-      const artist = await this.smartContractArtistService.getOrSetArtistForScAddress(collection.owner);
+      const artist = await this.smartContractArtistService.getOrSetArtistForScAddress(collection.currentOwner);
       const followersCount = await this.idService.getFollowersCount(artist?.value?.owner);
-      return Collection.fromCollectionApi(collection, artist?.value?.owner, followersCount?.count);
+      return Collection.fromCollectionElastic(collection, artist?.value?.owner, followersCount?.count);
     }
-    const followersCount = await this.idService.getFollowersCount(collection.owner);
-    return Collection.fromCollectionApi(collection, collection.owner, followersCount?.count);
+    const followersCount = await this.idService.getFollowersCount(collection.currentOwner);
+    return Collection.fromCollectionElastic(collection, collection.currentOwner, followersCount?.count);
   }
 
   private async mapCollectionNfts(localCollections: Collection[]) {
@@ -422,4 +420,15 @@ export class CollectionsGetterService {
     }
     return CollectionNftTrait.fromCollectionTraits(traitSummary.traitTypes);
   }
+
+  private getCollectionQuery(): ElasticQuery {
+    return ElasticQuery.create()
+      .withMustNotExistCondition('identifier')
+      .withMustMultiShouldCondition([NftTypeEnum.NonFungibleESDT, NftTypeEnum.SemiFungibleESDT], (type) => QueryType.Match('type', type))
+      .withPagination({ from: 0, size: constants.getCollectionsFromElasticBatchSize })
+      .withSort([
+        { name: 'api_isVerified', order: ElasticSortOrder.descending },
+        { name: 'timestamp', order: ElasticSortOrder.descending },
+      ]);
+  }
 }

src/modules/nftCollections/models/Collection.dto.ts

@@ -8,6 +8,7 @@ import { CollectionNftTrait } from 'src/modules/nft-traits/models/collection-tra
 import { CollectionAsset } from './CollectionAsset.dto';
 import { CollectionSocial } from './CollectionSocial.dto';
 import { CollectionVolumeLast24 } from 'src/modules/analytics/models/collection-volume';
+import { CollectionElastic } from 'src/common/services/mx-communication/elastic-collection.model';
 
 @ObjectType()
 export class Collection {
@@ -121,6 +122,32 @@ export class Collection {
       scamInfo: ScamInfo.fromScamInfoApi(collectionApi.scamInfo),
     });
   }
+
+  static fromCollectionElastic(collectionElastic: CollectionElastic, artistAddress?: string, followersCount?: number) {
+    if (!collectionElastic) {
+      return null;
+    }
+
+    return new Collection({
+      collection: collectionElastic.token,
+      artistAddress: artistAddress,
+      type: NftTypeEnum[collectionElastic.type],
+      ticker: collectionElastic.ticker,
+      ownerAddress: collectionElastic.currentOwner,
+      creationDate: collectionElastic.timestamp,
+      name: collectionElastic.name,
+      canTransferRole: collectionElastic.properties.canTransferNFTCreateRole,
+      canPause: collectionElastic.properties.canPause,
+      canBurn: collectionElastic.properties.canBurn,
+      canFreeze: collectionElastic.properties.canFreeze,
+      canWipe: collectionElastic.properties.canWipe,
+      canAddQuantity: collectionElastic.properties.canAddQuantity,
+      canCreate: collectionElastic.properties.canMint,
+      artistFollowersCount: followersCount,
+      nftsCount: collectionElastic.api_nftCount,
+      verified: collectionElastic.api_isVerified ?? false,
+    });
+  }
 }
 
 @ObjectType()