🐎 Optimize genesis asset query

LiskArchive · Jan 25, 2024 · 4336100 · 4336100
1 parent e7debd7
commit 4336100
Show file tree

Hide file tree

Showing 8 changed files with 46 additions and 87 deletions.
diff --git a/services/blockchain-connector/app.js b/services/blockchain-connector/app.js
@@ -14,7 +14,7 @@
  *
  */
 const path = require('path');
-const { Microservice, Logger, LoggerConfig } = require('lisk-service-framework');
+const { Signals, Microservice, Logger, LoggerConfig } = require('lisk-service-framework');
 
 const config = require('./config');
 
@@ -31,6 +31,12 @@ const app = Microservice({
 	transporter: config.transporter,
 	brokerTimeout: config.brokerTimeout, // in seconds
 	logger: config.log,
+	events: {
+		'update.index.status': async payload => {
+			logger.debug("Received a 'update.index.status' moleculer event from indexer.");
+			Signals.get('updateIndexStatus').dispatch(payload);
+		},
+	},
 });
 
 nodeStatus.waitForNode().then(async () => {

diff --git a/services/blockchain-connector/methods/token.js b/services/blockchain-connector/methods/token.js
@@ -22,6 +22,7 @@ const {
 	getTokenBalances,
 	getTokenInitializationFees,
 	tokenHasEscrowAccount,
+	getTokenBalanceAtGenesis,
 } = require('../shared/sdk');
 
 module.exports = [
@@ -84,4 +85,11 @@ module.exports = [
 		controller: async () => getTokenInitializationFees(),
 		params: {},
 	},
+	{
+		name: 'getTokenBalanceAtGenesis',
+		controller: async ({ address }) => getTokenBalanceAtGenesis(address),
+		params: {
+			address: { optional: false, type: 'string' },
+		},
+	},
 ];
diff --git a/services/blockchain-connector/shared/sdk/index.js b/services/blockchain-connector/shared/sdk/index.js
@@ -65,6 +65,7 @@ const {
 	getTotalSupply,
 	getTokenInitializationFees,
 	updateTokenInfo,
+	getTokenBalanceAtGenesis,
 } = require('./token');
 
 const {
@@ -178,7 +179,7 @@ module.exports = {
 	dryRunTransaction,
 	formatTransaction,
 
-	// Tokens
+	// Token
 	tokenHasUserAccount,
 	tokenHasEscrowAccount,
 	getTokenBalance,
@@ -187,6 +188,7 @@ module.exports = {
 	getSupportedTokens,
 	getTotalSupply,
 	getTokenInitializationFees,
+	getTokenBalanceAtGenesis,
 
 	// PoS
 	getAllPosValidators,

diff --git a/services/blockchain-connector/shared/sdk/token.js b/services/blockchain-connector/shared/sdk/token.js
@@ -15,6 +15,8 @@
  */
 const { invokeEndpoint } = require('./client');
 const { isMainchain } = require('./interoperability');
+const { getGenesisAssetByModule } = require('./genesisBlock');
+const { MODULE_NAME_TOKEN } = require('./constants/names');
 
 let escrowedAmounts;
 let supportedTokens;
@@ -73,6 +75,20 @@ const updateTokenInfo = async () => {
 	totalSupply = await getTotalSupply(true);
 };
 
+const getTokenBalanceAtGenesis = async address => {
+	const MODULE_TOKEN_SUBSTORE_USER = 'userSubstore';
+
+	const tokenModuleGenesisAssets = await getGenesisAssetByModule({
+		module: MODULE_NAME_TOKEN,
+		subStore: MODULE_TOKEN_SUBSTORE_USER,
+	});
+
+	const balancesAtGenesis = tokenModuleGenesisAssets[MODULE_TOKEN_SUBSTORE_USER];
+	const balancesByAddress = balancesAtGenesis.find(e => e.address === address);
+
+	return balancesByAddress;
+};
+
 module.exports = {
 	tokenHasUserAccount: hasUserAccount,
 	tokenHasEscrowAccount: hasEscrowAccount,
@@ -83,4 +99,5 @@ module.exports = {
 	getTotalSupply,
 	getTokenInitializationFees,
 	updateTokenInfo,
+	getTokenBalanceAtGenesis,
 };
diff --git a/services/export/config.js b/services/export/config.js
@@ -78,7 +78,7 @@ config.excel.sheets = {
 config.queue = {
 	scheduleTransactionExport: {
 		name: 'ScheduleTransactionExportQueue',
-		concurrency: 5, // TODO: Add env support
+		concurrency: 10,
 		options: {
 			defaultJobOptions: {
 				attempts: 5,

diff --git a/services/export/shared/requestAll.js b/services/export/shared/requestAll.js
@@ -13,9 +13,8 @@
  * Removal or modification of this copyright notice is prohibited.
  *
  */
-const { Utils } = require('lisk-service-framework');
 
-const requestAllStandard = async (fn, params, limit) => {
+const requestAll = async (fn, params, limit) => {
 	const defaultMaxAmount = limit || 1000;
 	const oneRequestLimit = params.limit || 100;
 	const firstRequest = await fn({
@@ -48,57 +47,4 @@ const requestAllStandard = async (fn, params, limit) => {
 	return data;
 };
 
-const requestAllCustom = async (fn, method, params, limit) => {
-	const maxAmount = limit || Number.MAX_SAFE_INTEGER;
-	const oneRequestLimit = params.limit || 100;
-	const firstRequest = await fn(method, {
-		...params,
-		...{
-			limit: oneRequestLimit,
-			offset: 0,
-		},
-	});
-	const totalResponse = firstRequest;
-	if (totalResponse && !totalResponse.error) {
-		if (maxAmount > oneRequestLimit) {
-			for (let page = 1; page < Math.ceil(maxAmount / oneRequestLimit); page++) {
-				const curOffset = oneRequestLimit * page;
-
-				const result = await fn(method, {
-					...params,
-					...{
-						limit: Math.min(oneRequestLimit, maxAmount - curOffset),
-						offset: curOffset,
-					},
-				});
-
-				// This check needs to be updated for dynamic exit based on emptiness of object properties
-				if (!result || Utils.isEmptyArray(result) || Utils.isEmptyObject(result)) {
-					break;
-				}
-
-				if (Array.isArray(totalResponse)) totalResponse.push(...result);
-				else if (Utils.isObject(totalResponse)) {
-					// When response is an object, we should traverse the properties and merge the values.
-					// We can safely assume that the properties would be of type array, so concatenation will
-					// result in the whole response. If property is not an array, the latest value is kept.
-					Object.entries(totalResponse).forEach(([dataKey, dataVal]) => {
-						if (Array.isArray(dataVal)) {
-							totalResponse[dataKey].push(...result[dataKey]);
-						} else if (Utils.isObject(dataVal)) {
-							totalResponse[dataKey] = { ...totalResponse[dataKey], ...result[dataKey] };
-						} else {
-							totalResponse[dataKey] = result[dataKey];
-						}
-					});
-				}
-			}
-		}
-	}
-	return totalResponse;
-};
-
-module.exports = {
-	requestAllStandard,
-	requestAllCustom,
-};
+module.exports = requestAll;
diff --git a/services/export/shared/transactionsExport.js b/services/export/shared/transactionsExport.js
@@ -29,8 +29,9 @@ const {
 } = require('lisk-service-framework');
 
 const config = require('../config');
-const FilesystemCache = require('./csvCache');
 const regex = require('./regex');
+const requestAll = require('./requestAll');
+const FilesystemCache = require('./csvCache');
 const fields = require('./excelFieldMappings');
 
 const {
@@ -41,7 +42,6 @@ const {
 	MODULE,
 	COMMAND,
 	EVENT,
-	MODULE_SUB_STORE,
 	requestIndexer,
 	requestConnector,
 	requestAppRegistry,
@@ -58,7 +58,6 @@ const {
 } = require('./helpers');
 
 const { checkIfIndexReadyForInterval } = require('./utils/ready');
-const { requestAllCustom, requestAllStandard } = require('./requestAll');
 
 const partials = FilesystemCache(config.cache.partials);
 const staticFiles = FilesystemCache(config.cache.exports);
@@ -71,7 +70,6 @@ const MAX_NUM_TRANSACTIONS = 10e5;
 
 const logger = Logger();
 
-let tokenModuleData;
 let feeTokenID;
 let defaultStartDate;
 
@@ -93,7 +91,7 @@ const validateIfAccountExists = async address => {
 };
 
 const getEvents = async params =>
-	requestAllStandard(requestIndexer.bind(null, 'events'), {
+	requestAll(requestIndexer.bind(null, 'events'), {
 		topic: params.address,
 		timestamp: params.timestamp,
 		module: params.module,
@@ -177,7 +175,7 @@ const getBlocksInAsc = async params => {
 		})
 	).meta.total;
 
-	const blocks = await requestAllStandard(
+	const blocks = await requestAll(
 		getBlocks,
 		{
 			generatorAddress: params.address,
@@ -221,26 +219,9 @@ const getChainInfo = async chainID => {
 };
 
 const getOpeningBalance = async address => {
-	if (!tokenModuleData) {
-		const genesisBlockAssetsLength = await requestConnector('getGenesisAssetsLength', {
-			module: MODULE.TOKEN,
-			subStore: MODULE_SUB_STORE.TOKEN.USER,
-		});
-		const totalUsers = genesisBlockAssetsLength[MODULE.TOKEN][MODULE_SUB_STORE.TOKEN.USER];
-
-		tokenModuleData = (
-			await requestAllCustom(
-				requestConnector,
-				'getGenesisAssetByModule',
-				{ module: MODULE.TOKEN, subStore: MODULE_SUB_STORE.TOKEN.USER },
-				totalUsers,
-			)
-		).userSubstore;
-	}
-
-	const filteredAccount = tokenModuleData.find(e => e.address === address);
-	const openingBalance = filteredAccount
-		? { tokenID: filteredAccount.tokenID, amount: filteredAccount.availableBalance }
+	const accountInfo = await requestConnector('getTokenBalanceAtGenesis', { address });
+	const openingBalance = accountInfo
+		? { tokenID: accountInfo.tokenID, amount: accountInfo.availableBalance }
 		: null;
 
 	return openingBalance;
@@ -445,7 +426,7 @@ const exportTransactions = async job => {
 			const fromTimestamp = moment(day, DATE_FORMAT).startOf('day').unix();
 			const toTimestamp = moment(day, DATE_FORMAT).endOf('day').unix();
 			const timestampRange = `${fromTimestamp}:${toTimestamp}`;
-			const transactions = await requestAllStandard(
+			const transactions = await requestAll(
 				getTransactionsInAsc,
 				{ ...params, timestamp: timestampRange },
 				MAX_NUM_TRANSACTIONS,

diff --git a/services/export/shared/utils/ready.js b/services/export/shared/utils/ready.js
@@ -41,7 +41,6 @@ const getIndexStatus = async () => {
 	return indexStatusCache;
 };
 
-// TODO: Write tests
 const checkIfIndexReadyForInterval = async interval => {
 	try {
 		// Blockchain fully indexed