Optimize retry mechanism in invokeEndpoint

docker-compose.yml

@@ -153,6 +153,10 @@ services:
       - ENABLE_TESTING_MODE=${ENABLE_TESTING_MODE}
       - ENABLE_BLOCK_CACHING=${ENABLE_BLOCK_CACHING}
       - EXPIRY_IN_HOURS=${EXPIRY_IN_HOURS}
+      - CLIENT_INSTANTIATION_MAX_WAIT_TIME=${CLIENT_INSTANTIATION_MAX_WAIT_TIME}
+      - CLIENT_INSTANTIATION_RETRY_INTERVAL=${CLIENT_INSTANTIATION_RETRY_INTERVAL}
+      - ENDPOINT_INVOKE_MAX_RETRIES=${ENDPOINT_INVOKE_MAX_RETRIES}
+      - ENDPOINT_INVOKE_RETRY_DELAY=${ENDPOINT_INVOKE_RETRY_DELAY}
       - JOB_INTERVAL_CACHE_CLEANUP=${JOB_INTERVAL_CACHE_CLEANUP}
       - JOB_SCHEDULE_CACHE_CLEANUP=${JOB_SCHEDULE_CACHE_CLEANUP}
       - JOB_INTERVAL_REFRESH_PEERS=${JOB_INTERVAL_REFRESH_PEERS}

docker/example.env

@@ -28,6 +28,10 @@
 ## Lisk Service Blockchain Connector
 
 # GENESIS_BLOCK_URL='https://downloads.lisk.com/lisk/mainnet/genesis_block.json.tar.gz'
+# CLIENT_INSTANTIATION_MAX_WAIT_TIME=100,
+# CLIENT_INSTANTIATION_RETRY_INTERVAL=5,
+# ENDPOINT_INVOKE_MAX_RETRIES=5,
+# ENDPOINT_INVOKE_RETRY_DELAY=10,
 
 # Moleculer jobs configuration 
 # JOB_INTERVAL_CACHE_CLEANUP=0
@@ -55,7 +59,7 @@ ENABLE_PERSIST_EVENTS=false
 # DEVNET_MAINCHAIN_URL='http://devnet-service.liskdev.net:9901'
 # ESTIMATES_BUFFER_BYTES_LENGTH=0
 # ENABLE_APPLY_SNAPSHOT=false
-# DURABILITY_VERIFY_FREQUENCY=20
+# DURABILITY_VERIFY_FREQUENCY=10
 # INDEX_SNAPSHOT_URL= ''
 # ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP=false
 # ACCOUNT_BALANCE_UPDATE_BATCH_SIZE=1000

docs/antora/modules/ROOT/pages/configuration/index.adoc

@@ -401,6 +401,30 @@ To enable it, set it to `true`.
 By default, it is set to `12` hours.
 | 12
 
+| `CLIENT_INSTANTIATION_MAX_WAIT_TIME`
+| number
+| Maximum wait time (in milliseconds) for the API client instantiation before forcefully instantiating a new client when getApiClient is invoked.
+By default, it is set to `100`.
+| 100
+
+| `CLIENT_INSTANTIATION_RETRY_INTERVAL`
+| number
+| Retry interval (in milliseconds) to invoke instantiate API client when getApiClient is invoked.
+By default, it is set to `5`.
+| 5
+
+| `ENDPOINT_INVOKE_MAX_RETRIES`
+| number
+| Maximum number of endpoint invocation request retries to the node.
+By default, it is set to `5`.
+| 5
+
+| `ENDPOINT_INVOKE_RETRY_DELAY`
+| number
+| Delay (in milliseconds) between each endpoint invocation request retry.
+By default, it is set to `10`.
+| 10
+
 | `JOB_INTERVAL_CACHE_CLEANUP`
 | number
 | Job run interval to clean up block cache.
@@ -730,7 +754,7 @@ To accelerate the indexing process, the blockchain-indexer microservice also sup
 
 |`DURABILITY_VERIFY_FREQUENCY`
 |number
-|Frequency in milliseconds to verify if a block is indexed or rolled-back successfully. By default, it is set to 20.
+|Frequency in milliseconds to verify if a block is indexed or rolled-back successfully. By default, it is set to 10.
 |false
 
 |`INDEX_SNAPSHOT_URL`
@@ -767,7 +791,7 @@ module.exports = {
         ENABLE_INDEXING_MODE: 'true',
         ENABLE_PERSIST_EVENTS: 'false',
         // ENABLE_APPLY_SNAPSHOT: 'false',
-        // DURABILITY_VERIFY_FREQUENCY: 20,
+        // DURABILITY_VERIFY_FREQUENCY: 10,
         // INDEX_SNAPSHOT_URL: '',
         // ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP: 'true',
         // SERVICE_INDEXER_MYSQL_READ_REPLICA: 'mysql://lisk:[email protected]:3306/lisk',

ecosystem.config.js

@@ -127,6 +127,10 @@ module.exports = {
 				// SERVICE_LOG_FILE: false,
 				// DOCKER_HOST: 'local',
 				// GENESIS_BLOCK_URL: 'https://downloads.lisk.com/lisk/mainnet/genesis_block.json.tar.gz',
+				// CLIENT_INSTANTIATION_MAX_WAIT_TIME: 100,
+				// CLIENT_INSTANTIATION_RETRY_INTERVAL: 5,
+				// ENDPOINT_INVOKE_MAX_RETRIES: 5,
+				// ENDPOINT_INVOKE_RETRY_DELAY: 10,
 				// JOB_INTERVAL_CACHE_CLEANUP: 0,
 				// JOB_SCHEDULE_CACHE_CLEANUP: '0 */12 * * *',
 				// JOB_INTERVAL_REFRESH_PEERS: 60,
@@ -156,7 +160,7 @@ module.exports = {
 				ENABLE_INDEXING_MODE: true,
 				ENABLE_PERSIST_EVENTS: false,
 				// ENABLE_APPLY_SNAPSHOT: false,
-				// DURABILITY_VERIFY_FREQUENCY: 20,
+				// DURABILITY_VERIFY_FREQUENCY: 10,
 				// INDEX_SNAPSHOT_URL: '',
 				// ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP: true,
 				// SERVICE_INDEXER_MYSQL_READ_REPLICA: 'mysql://lisk:[email protected]:3306/lisk',

services/blockchain-connector/README.md

@@ -31,9 +31,13 @@ A list of the most commonly used environment variables is presented below:
 - `USE_LISK_IPC_CLIENT`: Boolean flag to enable IPC-based connection to the Lisk SDK-based application node. Not applicable to a docker-based setup.
 - `LISK_APP_DATA_PATH`: Data path to connect with the Lisk SDK-based application node over IPC. Not applicable to a docker-based setup.
 - `GENESIS_BLOCK_URL`: URL of the Lisk SDK-based application' genesis block. Only to be used when the genesis block is large enough to be transmitted over API calls within the timeout.
-- `GEOIP_JSON`: URL of GeoIP server
+- `GEOIP_JSON`: URL of GeoIP server.
 - `ENABLE_BLOCK_CACHING`: Boolean flag to enable the block caching. Disabled by default. To enable, set it to `true`.
 - `EXPIRY_IN_HOURS`: Expiry time (in hours) for block cache. By default, it is set to 12.
+- `CLIENT_INSTANTIATION_MAX_WAIT_TIME`: Maximum wait time (in milliseconds) for the API client instantiation before forcefully instantiating a new client when getApiClient is invoked. By default, it is set to 100.
+- `CLIENT_INSTANTIATION_RETRY_INTERVAL`: Retry interval (in milliseconds) to invoke instantiate API client when getApiClient is invoked. By default, it is set to 5.
+- `ENDPOINT_INVOKE_MAX_RETRIES`: Maximum number of endpoint invocation request retries to the node. By default, it is set to 5.
+- `ENDPOINT_INVOKE_RETRY_DELAY`: Delay (in milliseconds) between each endpoint invocation request retry. By default, it is set to 10.
 - `JOB_INTERVAL_CACHE_CLEANUP`: Job run interval to cleanup block cache. By default, it is set to 0.
 - `JOB_SCHEDULE_CACHE_CLEANUP`: Job run cron schedule to cleanup block cache. By default, it is set to run every 12 hours (`0 */12 * * *`).
 - `JOB_INTERVAL_REFRESH_PEERS`: Job run interval to refresh the peers list. By default, it is set to run every 60 seconds.

services/blockchain-connector/config.js

@@ -113,4 +113,15 @@ config.job = {
 // Every n number of blocks, verify if client connection is alive
 config.connectionVerifyBlockInterval = process.env.CONN_VERIFY_BLOCK_INTERVAL || 10;
 
+config.apiClient = {
+	instantiation: {
+		maxWaitTime: Number(process.env.CLIENT_INSTANTIATION_MAX_WAIT_TIME || 100), // in millisecs
+		retryInterval: Number(process.env.CLIENT_INSTANTIATION_RETRY_INTERVAL || 5), // in millisecs
+	},
+	request: {
+		maxRetries: Number(process.env.ENDPOINT_INVOKE_MAX_RETRIES || 5),
+		retryDelay: Number(process.env.ENDPOINT_INVOKE_RETRY_DELAY || 10), // in millisecs
+	},
+};
+
 module.exports = config;

services/blockchain-connector/shared/sdk/client.js

@@ -28,9 +28,10 @@ const logger = Logger();
 // Constants
 const timeoutMessage = 'Response not received in';
 const liskAddress = config.endpoints.liskWs;
-const NUM_REQUEST_RETRIES = 5;
-const RETRY_INTERVAL = 500; // ms
-const MAX_INSTANTIATION_WAIT_TIME = 100; // in ms
+const RETRY_INTERVAL = config.apiClient.instantiation.retryInterval;
+const MAX_INSTANTIATION_WAIT_TIME = config.apiClient.instantiation.maxWaitTime;
+const NUM_REQUEST_RETRIES = config.apiClient.request.maxRetries;
+const ENDPOINT_INVOKE_RETRY_DELAY = config.apiClient.request.retryDelay;
 
 // Caching and flags
 let clientCache;
@@ -101,7 +102,7 @@ const invokeEndpoint = async (endpoint, params = {}, numRetries = NUM_REQUEST_RE
 			return response;
 		} catch (err) {
 			if (retries && err.message.includes(timeoutMessage)) {
-				await delay(10);
+				await delay(ENDPOINT_INVOKE_RETRY_DELAY);
 			} else {
 				throw err;
 			}

services/blockchain-indexer/README.md

@@ -40,7 +40,7 @@ A list of the most commonly used environment variables is presented below:
 - `ENABLE_INDEXING_MODE`: Boolean flag to enable the Data Indexing mode.
 - `ENABLE_PERSIST_EVENTS`: Boolean flag to permanently maintain the events in the MySQL database.
 - `ENABLE_APPLY_SNAPSHOT`: Boolean flag to enable initialization of the index with the Lisk Service DB snapshot.
-- `DURABILITY_VERIFY_FREQUENCY`: Frequency in milliseconds to verify if a block is indexed or rolled-back successfully. By default, it is set to 20.
+- `DURABILITY_VERIFY_FREQUENCY`: Frequency in milliseconds to verify if a block is indexed or rolled-back successfully. By default, it is set to 10.
 - `INDEX_SNAPSHOT_URL`: URL from where the Lisk Service DB snapshot will be downloaded.
 - `ENABLE_SNAPSHOT_ALLOW_INSECURE_HTTP`: Boolean flag to enable downloading snapshot from an (unsecured) HTTP URL.
 - `LISK_STATIC`: URL of Lisk static assets.

services/blockchain-indexer/config.js

@@ -156,7 +156,7 @@ config.networks = Object.freeze({
 
 config.db = {
 	isPersistEvents: Boolean(String(process.env.ENABLE_PERSIST_EVENTS).toLowerCase() === 'true'),
-	durabilityVerifyFrequency: Number(process.env.DURABILITY_VERIFY_FREQUENCY) || 20, // In millisecs
+	durabilityVerifyFrequency: Number(process.env.DURABILITY_VERIFY_FREQUENCY) || 10, // In millisecs
 };
 
 config.snapshot = {

services/blockchain-indexer/shared/indexer/blockchainIndex.js

@@ -248,7 +248,7 @@ const indexBlock = async job => {
 			const eventsTable = await getEventsTable();
 			const eventTopicsTable = await getEventTopicsTable();
 
-			const { eventsInfo, eventTopicsInfo } = await getEventsInfoToIndex(block, events);
+			const { eventsInfo, eventTopicsInfo } = getEventsInfoToIndex(block, events);
 			await eventsTable.upsert(eventsInfo, dbTrx);
 			await eventTopicsTable.upsert(eventTopicsInfo, dbTrx);
 
@@ -459,7 +459,7 @@ const deleteIndexedBlocks = async job => {
 					const eventsTable = await getEventsTable();
 					const eventTopicsTable = await getEventTopicsTable();
 
-					const { eventsInfo, eventTopicsInfo } = await getEventsInfoToIndex(block, events);
+					const { eventsInfo, eventTopicsInfo } = getEventsInfoToIndex(block, events);
 					await eventsTable.delete(eventsInfo, dbTrx);
 					await eventTopicsTable.delete(eventTopicsInfo, dbTrx);
 
@@ -663,13 +663,7 @@ const findMissingBlocksInRange = async (fromHeight, toHeight) => {
 	);
 
 	const blocksTable = await getBlocksTable();
-	const propBetweens = [
-		{
-			property: 'height',
-			from: fromHeight,
-			to: toHeight,
-		},
-	];
+	const propBetweens = [{ property: 'height', from: fromHeight, to: toHeight }];
 	const indexedBlockCount = await blocksTable.count({ propBetweens });
 
 	// This block helps determine empty index

services/blockchain-indexer/shared/indexer/utils/events.js

@@ -42,7 +42,7 @@ const keyValueTable = getKeyValueTable();
 
 const getEventsTable = () => getTableInstance(eventsTableSchema, MYSQL_ENDPOINT);
 
-const getEventsInfoToIndex = async (block, events) => {
+const getEventsInfoToIndex = (block, events) => {
 	const eventsInfoToIndex = {
 		eventsInfo: [],
 		eventTopicsInfo: [],