From 5001c668369fab20b95cde239925a9114698cda7 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi Date: Mon, 30 Oct 2023 21:39:53 +0100 Subject: [PATCH] :wrench: Tweak getApiClient retry settings --- docker-compose.yml | 4 +++ docker/example.env | 6 +++- .../ROOT/pages/configuration/index.adoc | 28 +++++++++++++++++-- ecosystem.config.js | 6 +++- services/blockchain-connector/README.md | 6 +++- services/blockchain-connector/config.js | 11 ++++++++ .../blockchain-connector/shared/sdk/client.js | 9 +++--- services/blockchain-indexer/README.md | 2 +- services/blockchain-indexer/config.js | 2 +- .../shared/indexer/blockchainIndex.js | 12 ++------ .../shared/indexer/utils/events.js | 2 +- 11 files changed, 67 insertions(+), 21 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4ad9ff8cc9..0be9ef4e08 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -153,6 +153,10 @@ services: - ENABLE_TESTING_MODE=${ENABLE_TESTING_MODE} - ENABLE_BLOCK_CACHING=${ENABLE_BLOCK_CACHING} - EXPIRY_IN_HOURS=${EXPIRY_IN_HOURS} + - CLIENT_INSTANTIATION_MAX_WAIT_TIME=${CLIENT_INSTANTIATION_MAX_WAIT_TIME} + - CLIENT_INSTANTIATION_RETRY_INTERVAL=${CLIENT_INSTANTIATION_RETRY_INTERVAL} + - ENDPOINT_INVOKE_MAX_RETRIES=${ENDPOINT_INVOKE_MAX_RETRIES} + - ENDPOINT_INVOKE_RETRY_DELAY=${ENDPOINT_INVOKE_RETRY_DELAY} - JOB_INTERVAL_CACHE_CLEANUP=${JOB_INTERVAL_CACHE_CLEANUP} - JOB_SCHEDULE_CACHE_CLEANUP=${JOB_SCHEDULE_CACHE_CLEANUP} - JOB_INTERVAL_REFRESH_PEERS=${JOB_INTERVAL_REFRESH_PEERS} diff --git a/docker/example.env b/docker/example.env index 82323cf5f5..cb06b6793b 100644 --- a/docker/example.env +++ b/docker/example.env @@ -28,6 +28,10 @@ ## Lisk Service Blockchain Connector # GENESIS_BLOCK_URL='https://downloads.lisk.com/lisk/mainnet/genesis_block.json.tar.gz' +# CLIENT_INSTANTIATION_MAX_WAIT_TIME=100, +# CLIENT_INSTANTIATION_RETRY_INTERVAL=5, +# ENDPOINT_INVOKE_MAX_RETRIES=5, +# ENDPOINT_INVOKE_RETRY_DELAY=10, # Moleculer jobs configuration # JOB_INTERVAL_CACHE_CLEANUP=0 @@ -55,7 +59,7 @@ ENABLE_PERSIST_EVENTS=false # DEVNET_MAINCHAIN_URL='http://devnet-service.liskdev.net:9901' # ESTIMATES_BUFFER_BYTES_LENGTH=0 # ENABLE_APPLY_SNAPSHOT=false -# DURABILITY_VERIFY_FREQUENCY=20 +# DURABILITY_VERIFY_FREQUENCY=10 # INDEX_SNAPSHOT_URL= '' # ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP=false # ACCOUNT_BALANCE_UPDATE_BATCH_SIZE=1000 diff --git a/docs/antora/modules/ROOT/pages/configuration/index.adoc b/docs/antora/modules/ROOT/pages/configuration/index.adoc index 2038a68383..793afc8e3c 100644 --- a/docs/antora/modules/ROOT/pages/configuration/index.adoc +++ b/docs/antora/modules/ROOT/pages/configuration/index.adoc @@ -401,6 +401,30 @@ To enable it, set it to `true`. By default, it is set to `12` hours. | 12 +| `CLIENT_INSTANTIATION_MAX_WAIT_TIME` +| number +| Maximum wait time (in milliseconds) for the API client instantiation before forcefully instantiating a new client when getApiClient is invoked. +By default, it is set to `100`. +| 100 + +| `CLIENT_INSTANTIATION_RETRY_INTERVAL` +| number +| Retry interval (in milliseconds) to invoke instantiate API client when getApiClient is invoked. +By default, it is set to `5`. +| 5 + +| `ENDPOINT_INVOKE_MAX_RETRIES` +| number +| Maximum number of endpoint invocation request retries to the node. +By default, it is set to `5`. +| 5 + +| `ENDPOINT_INVOKE_RETRY_DELAY` +| number +| Delay (in milliseconds) between each endpoint invocation request retry. +By default, it is set to `10`. +| 10 + | `JOB_INTERVAL_CACHE_CLEANUP` | number | Job run interval to clean up block cache. @@ -730,7 +754,7 @@ To accelerate the indexing process, the blockchain-indexer microservice also sup |`DURABILITY_VERIFY_FREQUENCY` |number -|Frequency in milliseconds to verify if a block is indexed or rolled-back successfully. By default, it is set to 20. +|Frequency in milliseconds to verify if a block is indexed or rolled-back successfully. By default, it is set to 10. |false |`INDEX_SNAPSHOT_URL` @@ -767,7 +791,7 @@ module.exports = { ENABLE_INDEXING_MODE: 'true', ENABLE_PERSIST_EVENTS: 'false', // ENABLE_APPLY_SNAPSHOT: 'false', - // DURABILITY_VERIFY_FREQUENCY: 20, + // DURABILITY_VERIFY_FREQUENCY: 10, // INDEX_SNAPSHOT_URL: '', // ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP: 'true', // SERVICE_INDEXER_MYSQL_READ_REPLICA: 'mysql://lisk:password@127.0.0.1:3306/lisk', diff --git a/ecosystem.config.js b/ecosystem.config.js index 5d7ab49c39..dbed98750d 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -127,6 +127,10 @@ module.exports = { // SERVICE_LOG_FILE: false, // DOCKER_HOST: 'local', // GENESIS_BLOCK_URL: 'https://downloads.lisk.com/lisk/mainnet/genesis_block.json.tar.gz', + // CLIENT_INSTANTIATION_MAX_WAIT_TIME: 100, + // CLIENT_INSTANTIATION_RETRY_INTERVAL: 5, + // ENDPOINT_INVOKE_MAX_RETRIES: 5, + // ENDPOINT_INVOKE_RETRY_DELAY: 10, // JOB_INTERVAL_CACHE_CLEANUP: 0, // JOB_SCHEDULE_CACHE_CLEANUP: '0 */12 * * *', // JOB_INTERVAL_REFRESH_PEERS: 60, @@ -156,7 +160,7 @@ module.exports = { ENABLE_INDEXING_MODE: true, ENABLE_PERSIST_EVENTS: false, // ENABLE_APPLY_SNAPSHOT: false, - // DURABILITY_VERIFY_FREQUENCY: 20, + // DURABILITY_VERIFY_FREQUENCY: 10, // INDEX_SNAPSHOT_URL: '', // ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP: true, // SERVICE_INDEXER_MYSQL_READ_REPLICA: 'mysql://lisk:password@127.0.0.1:3306/lisk', diff --git a/services/blockchain-connector/README.md b/services/blockchain-connector/README.md index 8b48613154..6d7a965443 100644 --- a/services/blockchain-connector/README.md +++ b/services/blockchain-connector/README.md @@ -31,9 +31,13 @@ A list of the most commonly used environment variables is presented below: - `USE_LISK_IPC_CLIENT`: Boolean flag to enable IPC-based connection to the Lisk SDK-based application node. Not applicable to a docker-based setup. - `LISK_APP_DATA_PATH`: Data path to connect with the Lisk SDK-based application node over IPC. Not applicable to a docker-based setup. - `GENESIS_BLOCK_URL`: URL of the Lisk SDK-based application' genesis block. Only to be used when the genesis block is large enough to be transmitted over API calls within the timeout. -- `GEOIP_JSON`: URL of GeoIP server +- `GEOIP_JSON`: URL of GeoIP server. - `ENABLE_BLOCK_CACHING`: Boolean flag to enable the block caching. Disabled by default. To enable, set it to `true`. - `EXPIRY_IN_HOURS`: Expiry time (in hours) for block cache. By default, it is set to 12. +- `CLIENT_INSTANTIATION_MAX_WAIT_TIME`: Maximum wait time (in milliseconds) for the API client instantiation before forcefully instantiating a new client when getApiClient is invoked. By default, it is set to 100. +- `CLIENT_INSTANTIATION_RETRY_INTERVAL`: Retry interval (in milliseconds) to invoke instantiate API client when getApiClient is invoked. By default, it is set to 5. +- `ENDPOINT_INVOKE_MAX_RETRIES`: Maximum number of endpoint invocation request retries to the node. By default, it is set to 5. +- `ENDPOINT_INVOKE_RETRY_DELAY`: Delay (in milliseconds) between each endpoint invocation request retry. By default, it is set to 10. - `JOB_INTERVAL_CACHE_CLEANUP`: Job run interval to cleanup block cache. By default, it is set to 0. - `JOB_SCHEDULE_CACHE_CLEANUP`: Job run cron schedule to cleanup block cache. By default, it is set to run every 12 hours (`0 */12 * * *`). - `JOB_INTERVAL_REFRESH_PEERS`: Job run interval to refresh the peers list. By default, it is set to run every 60 seconds. diff --git a/services/blockchain-connector/config.js b/services/blockchain-connector/config.js index 0702194eda..eb08f82462 100644 --- a/services/blockchain-connector/config.js +++ b/services/blockchain-connector/config.js @@ -113,4 +113,15 @@ config.job = { // Every n number of blocks, verify if client connection is alive config.connectionVerifyBlockInterval = process.env.CONN_VERIFY_BLOCK_INTERVAL || 10; +config.apiClient = { + instantiation: { + maxWaitTime: Number(process.env.CLIENT_INSTANTIATION_MAX_WAIT_TIME || 100), // in millisecs + retryInterval: Number(process.env.CLIENT_INSTANTIATION_RETRY_INTERVAL || 5), // in millisecs + }, + request: { + maxRetries: Number(process.env.ENDPOINT_INVOKE_MAX_RETRIES || 5), + retryDelay: Number(process.env.ENDPOINT_INVOKE_RETRY_DELAY || 10), // in millisecs + }, +}; + module.exports = config; diff --git a/services/blockchain-connector/shared/sdk/client.js b/services/blockchain-connector/shared/sdk/client.js index 6e2efa0cf8..e9491472ef 100644 --- a/services/blockchain-connector/shared/sdk/client.js +++ b/services/blockchain-connector/shared/sdk/client.js @@ -28,9 +28,10 @@ const logger = Logger(); // Constants const timeoutMessage = 'Response not received in'; const liskAddress = config.endpoints.liskWs; -const NUM_REQUEST_RETRIES = 5; -const RETRY_INTERVAL = 500; // ms -const MAX_INSTANTIATION_WAIT_TIME = 100; // in ms +const RETRY_INTERVAL = config.apiClient.instantiation.retryInterval; +const MAX_INSTANTIATION_WAIT_TIME = config.apiClient.instantiation.maxWaitTime; +const NUM_REQUEST_RETRIES = config.apiClient.request.maxRetries; +const ENDPOINT_INVOKE_RETRY_DELAY = config.apiClient.request.retryDelay; // Caching and flags let clientCache; @@ -101,7 +102,7 @@ const invokeEndpoint = async (endpoint, params = {}, numRetries = NUM_REQUEST_RE return response; } catch (err) { if (retries && err.message.includes(timeoutMessage)) { - await delay(10); + await delay(ENDPOINT_INVOKE_RETRY_DELAY); } else { throw err; } diff --git a/services/blockchain-indexer/README.md b/services/blockchain-indexer/README.md index 4177e44663..eb6f38b690 100644 --- a/services/blockchain-indexer/README.md +++ b/services/blockchain-indexer/README.md @@ -40,7 +40,7 @@ A list of the most commonly used environment variables is presented below: - `ENABLE_INDEXING_MODE`: Boolean flag to enable the Data Indexing mode. - `ENABLE_PERSIST_EVENTS`: Boolean flag to permanently maintain the events in the MySQL database. - `ENABLE_APPLY_SNAPSHOT`: Boolean flag to enable initialization of the index with the Lisk Service DB snapshot. -- `DURABILITY_VERIFY_FREQUENCY`: Frequency in milliseconds to verify if a block is indexed or rolled-back successfully. By default, it is set to 20. +- `DURABILITY_VERIFY_FREQUENCY`: Frequency in milliseconds to verify if a block is indexed or rolled-back successfully. By default, it is set to 10. - `INDEX_SNAPSHOT_URL`: URL from where the Lisk Service DB snapshot will be downloaded. - `ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP`: Boolean flag to enable downloading snapshot from an (unsecured) HTTP URL. - `LISK_STATIC`: URL of Lisk static assets. diff --git a/services/blockchain-indexer/config.js b/services/blockchain-indexer/config.js index 7487ceac3b..f17b7424fb 100644 --- a/services/blockchain-indexer/config.js +++ b/services/blockchain-indexer/config.js @@ -156,7 +156,7 @@ config.networks = Object.freeze({ config.db = { isPersistEvents: Boolean(String(process.env.ENABLE_PERSIST_EVENTS).toLowerCase() === 'true'), - durabilityVerifyFrequency: Number(process.env.DURABILITY_VERIFY_FREQUENCY) || 20, // In millisecs + durabilityVerifyFrequency: Number(process.env.DURABILITY_VERIFY_FREQUENCY) || 10, // In millisecs }; config.snapshot = { diff --git a/services/blockchain-indexer/shared/indexer/blockchainIndex.js b/services/blockchain-indexer/shared/indexer/blockchainIndex.js index e978eb0d8f..11c35bc4c2 100644 --- a/services/blockchain-indexer/shared/indexer/blockchainIndex.js +++ b/services/blockchain-indexer/shared/indexer/blockchainIndex.js @@ -248,7 +248,7 @@ const indexBlock = async job => { const eventsTable = await getEventsTable(); const eventTopicsTable = await getEventTopicsTable(); - const { eventsInfo, eventTopicsInfo } = await getEventsInfoToIndex(block, events); + const { eventsInfo, eventTopicsInfo } = getEventsInfoToIndex(block, events); await eventsTable.upsert(eventsInfo, dbTrx); await eventTopicsTable.upsert(eventTopicsInfo, dbTrx); @@ -459,7 +459,7 @@ const deleteIndexedBlocks = async job => { const eventsTable = await getEventsTable(); const eventTopicsTable = await getEventTopicsTable(); - const { eventsInfo, eventTopicsInfo } = await getEventsInfoToIndex(block, events); + const { eventsInfo, eventTopicsInfo } = getEventsInfoToIndex(block, events); await eventsTable.delete(eventsInfo, dbTrx); await eventTopicsTable.delete(eventTopicsInfo, dbTrx); @@ -663,13 +663,7 @@ const findMissingBlocksInRange = async (fromHeight, toHeight) => { ); const blocksTable = await getBlocksTable(); - const propBetweens = [ - { - property: 'height', - from: fromHeight, - to: toHeight, - }, - ]; + const propBetweens = [{ property: 'height', from: fromHeight, to: toHeight }]; const indexedBlockCount = await blocksTable.count({ propBetweens }); // This block helps determine empty index diff --git a/services/blockchain-indexer/shared/indexer/utils/events.js b/services/blockchain-indexer/shared/indexer/utils/events.js index 4410772a01..8da50dd665 100644 --- a/services/blockchain-indexer/shared/indexer/utils/events.js +++ b/services/blockchain-indexer/shared/indexer/utils/events.js @@ -42,7 +42,7 @@ const keyValueTable = getKeyValueTable(); const getEventsTable = () => getTableInstance(eventsTableSchema, MYSQL_ENDPOINT); -const getEventsInfoToIndex = async (block, events) => { +const getEventsInfoToIndex = (block, events) => { const eventsInfoToIndex = { eventsInfo: [], eventTopicsInfo: [],