🐛 Fix indexTokenBalances

docker-compose.yml

@@ -203,6 +203,7 @@ services:
       - MAINCHAIN_SERVICE_URL=${MAINCHAIN_SERVICE_URL}
       - LISK_STATIC=${LISK_STATIC}
       - DEVNET_MAINCHAIN_URL=${DEVNET_MAINCHAIN_URL}
+      - ACCOUNT_BALANCE_UPDATE_BATCH_SIZE=${ACCOUNT_BALANCE_UPDATE_BATCH_SIZE}
       - JOB_INTERVAL_DELETE_SERIALIZED_EVENTS=${JOB_INTERVAL_DELETE_SERIALIZED_EVENTS}
       - JOB_SCHEDULE_DELETE_SERIALIZED_EVENTS=${JOB_SCHEDULE_DELETE_SERIALIZED_EVENTS}
       - JOB_INTERVAL_REFRESH_VALIDATORS=${JOB_INTERVAL_REFRESH_VALIDATORS}

docker/example.env

@@ -58,6 +58,7 @@ ENABLE_PERSIST_EVENTS=false
 # DURABILITY_VERIFY_FREQUENCY=20
 # INDEX_SNAPSHOT_URL= ''
 # ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP=false
+# ACCOUNT_BALANCE_UPDATE_BATCH_SIZE=1000
 
 # Moleculer jobs configuration 
 # JOB_INTERVAL_DELETE_SERIALIZED_EVENTS=0

docs/antora/modules/ROOT/pages/configuration/index.adoc

@@ -698,6 +698,12 @@ By default, it is set to run every 15 minutes.
 By default, it is set to `0`.
 | 0
 
+| `ACCOUNT_BALANCE_UPDATE_BATCH_SIZE`
+| number
+| Number of accounts for which the balance index is updated at a time.
+By default, it is set to `1000`.
+| 1000
+
 | `MAINCHAIN_SERVICE_URL`
 | string
 | Mainchain service URL for custom deployments.
@@ -776,6 +782,7 @@ module.exports = {
         // LISK_STATIC: 'https://static-data.lisk.com',
         // DEVNET_MAINCHAIN_URL: 'http://devnet-service.liskdev.net:9901',
         // ESTIMATES_BUFFER_BYTES_LENGTH: 0,
+        // ACCOUNT_BALANCE_UPDATE_BATCH_SIZE: 1000,
         // JOB_INTERVAL_DELETE_SERIALIZED_EVENTS: 0,
         // JOB_SCHEDULE_DELETE_SERIALIZED_EVENTS: '*/5 * * * *',
         // JOB_INTERVAL_REFRESH_VALIDATORS: 0,

ecosystem.config.js

@@ -171,6 +171,7 @@ module.exports = {
 				// LISK_STATIC: 'https://static-data.lisk.com',
 				// DEVNET_MAINCHAIN_URL: 'http://devnet-service.liskdev.net:9901',
 				// ESTIMATES_BUFFER_BYTES_LENGTH: 0,
+				// ACCOUNT_BALANCE_UPDATE_BATCH_SIZE: 1000,
 				// JOB_INTERVAL_DELETE_SERIALIZED_EVENTS: 0,
 				// JOB_SCHEDULE_DELETE_SERIALIZED_EVENTS: '*/5 * * * *',
 				// JOB_INTERVAL_REFRESH_VALIDATORS: 0,

services/blockchain-indexer/README.md

@@ -45,6 +45,7 @@ A list of the most commonly used environment variables is presented below:
 - `ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP`: Boolean flag to enable downloading snapshot from an (unsecured) HTTP URL.
 - `LISK_STATIC`: URL of Lisk static assets.
 - `SERVICE_INDEXER_CACHE_REDIS`: URL of the cache storage (Redis).
+- `ACCOUNT_BALANCE_UPDATE_BATCH_SIZE`: Number of accounts for which the balance index is updated at a time. By default, it is set to 1000.
 - `JOB_INTERVAL_DELETE_SERIALIZED_EVENTS`: Job run interval to delete serialized events. By default, it is set to 0.
 - `JOB_SCHEDULE_DELETE_SERIALIZED_EVENTS`: Job run cron schedule to delete serialized events. By default, it is set to run every 5th minute (`*/5 * * * *`).
 - `JOB_INTERVAL_REFRESH_VALIDATORS`: Job run interval to refresh validators cache. By default, it is set to 0.

services/blockchain-indexer/config.js

@@ -110,6 +110,13 @@ config.queue = {
 	},
 };
 
+config.set = {
+	accountBalanceUpdate: {
+		name: 'AccountBalanceUpdate',
+		batchSize: Number(process.env.ACCOUNT_BALANCE_UPDATE_BATCH_SIZE) || 1000,
+	},
+};
+
 config.operations = {
 	isDataRetrievalModeEnabled: Boolean(
 		String(process.env.ENABLE_DATA_RETRIEVAL_MODE).toLowerCase() !== 'false',

services/blockchain-indexer/shared/indexer/accountBalanceIndex.js

@@ -32,9 +32,6 @@ const redis = new Redis(config.endpoints.cache);
 
 const MYSQL_ENDPOINT = config.endpoints.mysql;
 
-const ACCOUNTS_BALANCE_UPDATE_SET_NAME = 'AccountsBalanceUpdate';
-const MAX_ACCOUNT_COUNT_IN_ONE_EXECUTION = 1000; // 1e3
-
 const getAccountBalancesTable = () => getTableInstance(accountBalancesTableSchema, MYSQL_ENDPOINT);
 
 const updateAccountBalances = async address => {
@@ -53,7 +50,7 @@ const updateAccountBalances = async address => {
 
 const scheduleAddressesBalanceUpdate = async addresses => {
 	if (addresses.length) {
-		redis.sadd(ACCOUNTS_BALANCE_UPDATE_SET_NAME, addresses);
+		redis.sadd(config.set.accountBalanceUpdate.name, addresses);
 	}
 };
 
@@ -79,8 +76,8 @@ const getAddressesFromTokenEvents = events => {
 
 const triggerAccountsBalanceUpdate = async () => {
 	const addresses = await redis.spop(
-		ACCOUNTS_BALANCE_UPDATE_SET_NAME,
-		MAX_ACCOUNT_COUNT_IN_ONE_EXECUTION,
+		config.set.accountBalanceUpdate.name,
+		config.set.accountBalanceUpdate.batchSize,
 	);
 
 	try {

services/blockchain-indexer/shared/indexer/genesisBlock.js

@@ -18,19 +18,22 @@ const {
 		MySQL: { getTableInstance },
 	},
 	Signals,
+	Logger,
 } = require('lisk-service-framework');
 
 const { MODULE, MODULE_SUB_STORE, getGenesisHeight } = require('../constants');
 const { updateTotalStake, updateTotalSelfStake } = require('./transactionProcessor/pos/stake');
 const { requestConnector } = require('../utils/request');
-const { accountBalanceIndexQueue } = require('./accountBalanceIndex');
+const { updateAccountBalances } = require('./accountBalanceIndex');
 const { updateTotalLockedAmounts } = require('./utils/blockchainIndex');
 
 const requestAll = require('../utils/requestAll');
 const config = require('../../config');
 const commissionsTableSchema = require('../database/schema/commissions');
 const { getIndexStats } = require('./indexStatus');
 
+const logger = Logger();
+
 const MYSQL_ENDPOINT = config.endpoints.mysql;
 
 const getCommissionsTable = () => getTableInstance(commissionsTableSchema, MYSQL_ENDPOINT);
@@ -144,7 +147,16 @@ const indexGenesisBlockAssets = async dbTrx => {
 };
 
 const indexTokenBalances = async () => {
-	allAccountsAddresses.forEach(async address => accountBalanceIndexQueue.add({ address }));
+	// eslint-disable-next-line no-restricted-syntax
+	for (const address of allAccountsAddresses) {
+		await updateAccountBalances(address).catch(err => {
+			const errorMessage = `Updating account balance for ${address} failed. Retrying.\nError: ${err.message}.`;
+			logger.warn(errorMessage);
+			logger.debug(err.stack);
+
+			allAccountsAddresses.push(address);
+		});
+	}
 	isTokensBalanceIndexed = true;
 };