diff --git a/docker-compose.yml b/docker-compose.yml index bedff993fa..4ad9ff8cc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -203,6 +203,7 @@ services: - MAINCHAIN_SERVICE_URL=${MAINCHAIN_SERVICE_URL} - LISK_STATIC=${LISK_STATIC} - DEVNET_MAINCHAIN_URL=${DEVNET_MAINCHAIN_URL} + - ACCOUNT_BALANCE_UPDATE_BATCH_SIZE=${ACCOUNT_BALANCE_UPDATE_BATCH_SIZE} - JOB_INTERVAL_DELETE_SERIALIZED_EVENTS=${JOB_INTERVAL_DELETE_SERIALIZED_EVENTS} - JOB_SCHEDULE_DELETE_SERIALIZED_EVENTS=${JOB_SCHEDULE_DELETE_SERIALIZED_EVENTS} - JOB_INTERVAL_REFRESH_VALIDATORS=${JOB_INTERVAL_REFRESH_VALIDATORS} diff --git a/docker/example.env b/docker/example.env index cd749c2450..82323cf5f5 100644 --- a/docker/example.env +++ b/docker/example.env @@ -58,6 +58,7 @@ ENABLE_PERSIST_EVENTS=false # DURABILITY_VERIFY_FREQUENCY=20 # INDEX_SNAPSHOT_URL= '' # ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP=false +# ACCOUNT_BALANCE_UPDATE_BATCH_SIZE=1000 # Moleculer jobs configuration # JOB_INTERVAL_DELETE_SERIALIZED_EVENTS=0 diff --git a/docs/antora/modules/ROOT/pages/configuration/index.adoc b/docs/antora/modules/ROOT/pages/configuration/index.adoc index ace8bc46ad..2038a68383 100644 --- a/docs/antora/modules/ROOT/pages/configuration/index.adoc +++ b/docs/antora/modules/ROOT/pages/configuration/index.adoc @@ -698,6 +698,12 @@ By default, it is set to run every 15 minutes. By default, it is set to `0`. | 0 +| `ACCOUNT_BALANCE_UPDATE_BATCH_SIZE` +| number +| Number of accounts for which the balance index is updated at a time. +By default, it is set to `1000`. +| 1000 + | `MAINCHAIN_SERVICE_URL` | string | Mainchain service URL for custom deployments. @@ -776,6 +782,7 @@ module.exports = { // LISK_STATIC: 'https://static-data.lisk.com', // DEVNET_MAINCHAIN_URL: 'http://devnet-service.liskdev.net:9901', // ESTIMATES_BUFFER_BYTES_LENGTH: 0, + // ACCOUNT_BALANCE_UPDATE_BATCH_SIZE: 1000, // JOB_INTERVAL_DELETE_SERIALIZED_EVENTS: 0, // JOB_SCHEDULE_DELETE_SERIALIZED_EVENTS: '*/5 * * * *', // JOB_INTERVAL_REFRESH_VALIDATORS: 0, diff --git a/ecosystem.config.js b/ecosystem.config.js index 5e1dce5072..5d7ab49c39 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -171,6 +171,7 @@ module.exports = { // LISK_STATIC: 'https://static-data.lisk.com', // DEVNET_MAINCHAIN_URL: 'http://devnet-service.liskdev.net:9901', // ESTIMATES_BUFFER_BYTES_LENGTH: 0, + // ACCOUNT_BALANCE_UPDATE_BATCH_SIZE: 1000, // JOB_INTERVAL_DELETE_SERIALIZED_EVENTS: 0, // JOB_SCHEDULE_DELETE_SERIALIZED_EVENTS: '*/5 * * * *', // JOB_INTERVAL_REFRESH_VALIDATORS: 0, diff --git a/services/blockchain-indexer/README.md b/services/blockchain-indexer/README.md index 3b8db31d7f..4177e44663 100644 --- a/services/blockchain-indexer/README.md +++ b/services/blockchain-indexer/README.md @@ -45,6 +45,7 @@ A list of the most commonly used environment variables is presented below: - `ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP`: Boolean flag to enable downloading snapshot from an (unsecured) HTTP URL. - `LISK_STATIC`: URL of Lisk static assets. - `SERVICE_INDEXER_CACHE_REDIS`: URL of the cache storage (Redis). +- `ACCOUNT_BALANCE_UPDATE_BATCH_SIZE`: Number of accounts for which the balance index is updated at a time. By default, it is set to 1000. - `JOB_INTERVAL_DELETE_SERIALIZED_EVENTS`: Job run interval to delete serialized events. By default, it is set to 0. - `JOB_SCHEDULE_DELETE_SERIALIZED_EVENTS`: Job run cron schedule to delete serialized events. By default, it is set to run every 5th minute (`*/5 * * * *`). - `JOB_INTERVAL_REFRESH_VALIDATORS`: Job run interval to refresh validators cache. By default, it is set to 0. diff --git a/services/blockchain-indexer/config.js b/services/blockchain-indexer/config.js index d6abc66576..7487ceac3b 100644 --- a/services/blockchain-indexer/config.js +++ b/services/blockchain-indexer/config.js @@ -110,6 +110,13 @@ config.queue = { }, }; +config.set = { + accountBalanceUpdate: { + name: 'AccountBalanceUpdate', + batchSize: Number(process.env.ACCOUNT_BALANCE_UPDATE_BATCH_SIZE) || 1000, + }, +}; + config.operations = { isDataRetrievalModeEnabled: Boolean( String(process.env.ENABLE_DATA_RETRIEVAL_MODE).toLowerCase() !== 'false', diff --git a/services/blockchain-indexer/shared/indexer/accountBalanceIndex.js b/services/blockchain-indexer/shared/indexer/accountBalanceIndex.js index 48ff5291a0..52ce7ad70c 100644 --- a/services/blockchain-indexer/shared/indexer/accountBalanceIndex.js +++ b/services/blockchain-indexer/shared/indexer/accountBalanceIndex.js @@ -32,9 +32,6 @@ const redis = new Redis(config.endpoints.cache); const MYSQL_ENDPOINT = config.endpoints.mysql; -const ACCOUNTS_BALANCE_UPDATE_SET_NAME = 'AccountsBalanceUpdate'; -const MAX_ACCOUNT_COUNT_IN_ONE_EXECUTION = 1000; // 1e3 - const getAccountBalancesTable = () => getTableInstance(accountBalancesTableSchema, MYSQL_ENDPOINT); const updateAccountBalances = async address => { @@ -53,7 +50,7 @@ const updateAccountBalances = async address => { const scheduleAddressesBalanceUpdate = async addresses => { if (addresses.length) { - redis.sadd(ACCOUNTS_BALANCE_UPDATE_SET_NAME, addresses); + redis.sadd(config.set.accountBalanceUpdate.name, addresses); } }; @@ -79,8 +76,8 @@ const getAddressesFromTokenEvents = events => { const triggerAccountsBalanceUpdate = async () => { const addresses = await redis.spop( - ACCOUNTS_BALANCE_UPDATE_SET_NAME, - MAX_ACCOUNT_COUNT_IN_ONE_EXECUTION, + config.set.accountBalanceUpdate.name, + config.set.accountBalanceUpdate.batchSize, ); try { diff --git a/services/blockchain-indexer/shared/indexer/genesisBlock.js b/services/blockchain-indexer/shared/indexer/genesisBlock.js index 824a323b66..b38475b957 100644 --- a/services/blockchain-indexer/shared/indexer/genesisBlock.js +++ b/services/blockchain-indexer/shared/indexer/genesisBlock.js @@ -18,12 +18,13 @@ const { MySQL: { getTableInstance }, }, Signals, + Logger, } = require('lisk-service-framework'); const { MODULE, MODULE_SUB_STORE, getGenesisHeight } = require('../constants'); const { updateTotalStake, updateTotalSelfStake } = require('./transactionProcessor/pos/stake'); const { requestConnector } = require('../utils/request'); -const { accountBalanceIndexQueue } = require('./accountBalanceIndex'); +const { updateAccountBalances } = require('./accountBalanceIndex'); const { updateTotalLockedAmounts } = require('./utils/blockchainIndex'); const requestAll = require('../utils/requestAll'); @@ -31,6 +32,8 @@ const config = require('../../config'); const commissionsTableSchema = require('../database/schema/commissions'); const { getIndexStats } = require('./indexStatus'); +const logger = Logger(); + const MYSQL_ENDPOINT = config.endpoints.mysql; const getCommissionsTable = () => getTableInstance(commissionsTableSchema, MYSQL_ENDPOINT); @@ -144,7 +147,16 @@ const indexGenesisBlockAssets = async dbTrx => { }; const indexTokenBalances = async () => { - allAccountsAddresses.forEach(async address => accountBalanceIndexQueue.add({ address })); + // eslint-disable-next-line no-restricted-syntax + for (const address of allAccountsAddresses) { + await updateAccountBalances(address).catch(err => { + const errorMessage = `Updating account balance for ${address} failed. Retrying.\nError: ${err.message}.`; + logger.warn(errorMessage); + logger.debug(err.stack); + + allAccountsAddresses.push(address); + }); + } isTokensBalanceIndexed = true; };