From 2516d12b0a36024935243465effa94d75851b08c Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi Date: Wed, 25 Oct 2023 10:23:56 +0200 Subject: [PATCH 1/5] :hammer: Index transactionID as a topic for events where unavailable --- .../blockchain-indexer/shared/constants.js | 12 +++++-- .../business/transactionsEstimateFees.js | 4 +-- .../shared/indexer/utils/events.js | 36 +++++++++++++++++-- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/services/blockchain-indexer/shared/constants.js b/services/blockchain-indexer/shared/constants.js index 2b75d772c7..42e62ac8ef 100644 --- a/services/blockchain-indexer/shared/constants.js +++ b/services/blockchain-indexer/shared/constants.js @@ -153,8 +153,9 @@ const PATTERN_ANY_CHAIN_TOKEN_ID = '*'.repeat(LENGTH_TOKEN_LOCAL_ID); const LENGTH_TOKEN_ID = LENGTH_CHAIN_ID + LENGTH_TOKEN_LOCAL_ID; const LENGTH_NETWORK_ID = 1 * 2; // Each byte is represented with 2 nibbles const LENGTH_BYTE_SIGNATURE = 64; -const LENGTH_BYTE_ID = 32; +const LENGTH_BYTE_TRANSACTION_ID = 32; const DEFAULT_NUM_OF_SIGNATURES = 1; +const LENGTH_TRANSACTION_ID = LENGTH_BYTE_TRANSACTION_ID * 2; // Each byte is represented with 2 nibbles const MAX_COMMISSION = BigInt('10000'); @@ -180,6 +181,11 @@ const EVENT = Object.freeze({ CCM_SEND_SUCCESS: 'ccmSendSuccess', }); +const EVENT_TOPIC_PREFIX = Object.freeze({ + TX_ID: '04', + CCM_ID: '05', +}); + const TRANSACTION_VERIFY_RESULT = { INVALID: -1, PENDING: 0, @@ -221,11 +227,13 @@ module.exports = { MODULE_SUB_STORE, COMMAND, EVENT, + EVENT_TOPIC_PREFIX, MAX_COMMISSION, KV_STORE_KEY, TRANSACTION_STATUS, TRANSACTION_VERIFY_RESULT, LENGTH_BYTE_SIGNATURE, - LENGTH_BYTE_ID, + LENGTH_BYTE_TRANSACTION_ID, + LENGTH_TRANSACTION_ID, DEFAULT_NUM_OF_SIGNATURES, }; diff --git a/services/blockchain-indexer/shared/dataService/business/transactionsEstimateFees.js b/services/blockchain-indexer/shared/dataService/business/transactionsEstimateFees.js index a9c0113a05..fc1e030dde 100644 --- a/services/blockchain-indexer/shared/dataService/business/transactionsEstimateFees.js +++ b/services/blockchain-indexer/shared/dataService/business/transactionsEstimateFees.js @@ -40,7 +40,7 @@ const { COMMAND, EVENT, LENGTH_BYTE_SIGNATURE, - LENGTH_BYTE_ID, + LENGTH_BYTE_TRANSACTION_ID, DEFAULT_NUM_OF_SIGNATURES, } = require('../../constants'); @@ -74,7 +74,7 @@ const OPTIONAL_TRANSACTION_PROPERTIES = Object.freeze({ }, ID: { propName: 'id', - defaultValue: () => getRandomBytes(LENGTH_BYTE_ID).toString('hex'), + defaultValue: () => getRandomBytes(LENGTH_BYTE_TRANSACTION_ID).toString('hex'), }, }); diff --git a/services/blockchain-indexer/shared/indexer/utils/events.js b/services/blockchain-indexer/shared/indexer/utils/events.js index 438b2e1e46..adf861e186 100644 --- a/services/blockchain-indexer/shared/indexer/utils/events.js +++ b/services/blockchain-indexer/shared/indexer/utils/events.js @@ -27,7 +27,12 @@ const { }, } = require('lisk-service-framework'); -const { getGenesisHeight } = require('../../constants'); +const { + getGenesisHeight, + EVENT, + EVENT_TOPIC_PREFIX, + LENGTH_TRANSACTION_ID, +} = require('../../constants'); const config = require('../../../config'); const eventsTableSchema = require('../../database/schema/events'); @@ -48,7 +53,7 @@ const getEventsInfoToIndex = async (block, events) => { eventTopicsInfo: [], }; - events.forEach(event => { + events.forEach((event, eventIndex) => { const eventInfo = { id: event.id, name: event.name, @@ -60,7 +65,7 @@ const getEventsInfoToIndex = async (block, events) => { }; // Store whole event when persistence is enabled or block is not finalized yet - // Storing event of non-finalized block is required to fetch events of a dropped block + // Storing event of non-finalized block is required to fetch events of a deleted block if (!block.isFinal || config.db.isPersistEvents) { eventInfo.eventStr = JSON.stringify(event); } @@ -73,6 +78,31 @@ const getEventsInfoToIndex = async (block, events) => { topic, }; eventsInfoToIndex.eventTopicsInfo.push(eventTopicInfo); + + // Add the corresponding transactionID as a topic when not present in the topics list + // i.e. topic starts with the CCM ID prefix + // Useful to fetch the relevant events when queried by transactionID + if ( + topic.startsWith(EVENT_TOPIC_PREFIX.CCM_ID) && + topic.length === EVENT_TOPIC_PREFIX.CCM_ID.length + LENGTH_TRANSACTION_ID + ) { + const commandExecResultEvent = events + .slice(eventIndex) + .find(e => e.name === EVENT.COMMAND_EXECUTION_RESULT); + + const [topicTransactionID] = commandExecResultEvent.topics; + + const transactionID = // Remove the topic prefix from transactionID before indexing + topicTransactionID.length === EVENT_TOPIC_PREFIX.TX_ID.length + LENGTH_TRANSACTION_ID + ? topicTransactionID.slice(EVENT_TOPIC_PREFIX.TX_ID.length) + : topicTransactionID; + + const eventTopicAdditionalInfo = { + eventID: event.id, + topic: transactionID, + }; + eventsInfoToIndex.eventTopicsInfo.push(eventTopicAdditionalInfo); + } }); }); From e86618c7cfb866de2b411cb2047ed97bed7a64cb Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi Date: Wed, 25 Oct 2023 11:10:49 +0200 Subject: [PATCH 2/5] :hammer: Add support to handle transactionID/ccmID with topic prefixes --- .../blockchain-indexer/shared/constants.js | 8 +++---- .../shared/dataService/business/events.js | 23 +++++++++++++++++-- .../business/transactionsEstimateFees.js | 4 ++-- .../shared/indexer/utils/events.js | 11 +++------ 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/services/blockchain-indexer/shared/constants.js b/services/blockchain-indexer/shared/constants.js index 42e62ac8ef..6115913416 100644 --- a/services/blockchain-indexer/shared/constants.js +++ b/services/blockchain-indexer/shared/constants.js @@ -153,9 +153,9 @@ const PATTERN_ANY_CHAIN_TOKEN_ID = '*'.repeat(LENGTH_TOKEN_LOCAL_ID); const LENGTH_TOKEN_ID = LENGTH_CHAIN_ID + LENGTH_TOKEN_LOCAL_ID; const LENGTH_NETWORK_ID = 1 * 2; // Each byte is represented with 2 nibbles const LENGTH_BYTE_SIGNATURE = 64; -const LENGTH_BYTE_TRANSACTION_ID = 32; +const LENGTH_BYTE_ID = 32; const DEFAULT_NUM_OF_SIGNATURES = 1; -const LENGTH_TRANSACTION_ID = LENGTH_BYTE_TRANSACTION_ID * 2; // Each byte is represented with 2 nibbles +const LENGTH_ID = LENGTH_BYTE_ID * 2; // Each byte is represented with 2 nibbles const MAX_COMMISSION = BigInt('10000'); @@ -233,7 +233,7 @@ module.exports = { TRANSACTION_STATUS, TRANSACTION_VERIFY_RESULT, LENGTH_BYTE_SIGNATURE, - LENGTH_BYTE_TRANSACTION_ID, - LENGTH_TRANSACTION_ID, + LENGTH_BYTE_ID, + LENGTH_ID, DEFAULT_NUM_OF_SIGNATURES, }; diff --git a/services/blockchain-indexer/shared/dataService/business/events.js b/services/blockchain-indexer/shared/dataService/business/events.js index c42fd5af78..e2b4a7df1c 100644 --- a/services/blockchain-indexer/shared/dataService/business/events.js +++ b/services/blockchain-indexer/shared/dataService/business/events.js @@ -32,6 +32,7 @@ const eventTopicsTableSchema = require('../../database/schema/eventTopics'); const { requestConnector } = require('../../utils/request'); const { normalizeRangeParam } = require('../../utils/param'); const { parseToJSONCompatObj } = require('../../utils/parser'); +const { LENGTH_ID, EVENT_TOPIC_PREFIX } = require('../../constants'); const MYSQL_ENDPOINT = config.endpoints.mysqlReplica; @@ -122,6 +123,7 @@ const getEvents = async params => { const { transactionID, ...remParams } = params; params = remParams; + // Special handling of transaction topic prefix is unnecessary here because of the handling for topic param below if (!params.topic) { params.topic = transactionID; } else { @@ -173,11 +175,28 @@ const getEvents = async params => { const { topic, ...remParams } = params; params = remParams; + const topics = topic.split(','); + topics.forEach(t => { + if ( + t.startsWith(EVENT_TOPIC_PREFIX.TX_ID) && + t.length === EVENT_TOPIC_PREFIX.TX_ID.length + LENGTH_ID + ) { + // Check for the transaction ID both with and without the topic prefix + topics.push(t.slice(EVENT_TOPIC_PREFIX.TX_ID.length)); + } else if ( + t.startsWith(EVENT_TOPIC_PREFIX.CCM_ID) && + t.length === EVENT_TOPIC_PREFIX.CCM_ID.length + LENGTH_ID + ) { + // Check for CCM ID both with and without the topic prefix + topics.push(t.slice(EVENT_TOPIC_PREFIX.CCM_ID.length)); + } + }); + const response = await eventTopicsTable.find( { - whereIn: { property: 'topic', values: topic.split(',') }, + whereIn: { property: 'topic', values: topics }, groupBy: 'eventID', - havingRaw: `COUNT(DISTINCT topic) = ${topic.split(',').length}`, + havingRaw: `COUNT(DISTINCT topic) = ${topics.length}`, }, ['eventID'], ); diff --git a/services/blockchain-indexer/shared/dataService/business/transactionsEstimateFees.js b/services/blockchain-indexer/shared/dataService/business/transactionsEstimateFees.js index fc1e030dde..a9c0113a05 100644 --- a/services/blockchain-indexer/shared/dataService/business/transactionsEstimateFees.js +++ b/services/blockchain-indexer/shared/dataService/business/transactionsEstimateFees.js @@ -40,7 +40,7 @@ const { COMMAND, EVENT, LENGTH_BYTE_SIGNATURE, - LENGTH_BYTE_TRANSACTION_ID, + LENGTH_BYTE_ID, DEFAULT_NUM_OF_SIGNATURES, } = require('../../constants'); @@ -74,7 +74,7 @@ const OPTIONAL_TRANSACTION_PROPERTIES = Object.freeze({ }, ID: { propName: 'id', - defaultValue: () => getRandomBytes(LENGTH_BYTE_TRANSACTION_ID).toString('hex'), + defaultValue: () => getRandomBytes(LENGTH_BYTE_ID).toString('hex'), }, }); diff --git a/services/blockchain-indexer/shared/indexer/utils/events.js b/services/blockchain-indexer/shared/indexer/utils/events.js index adf861e186..60223369ee 100644 --- a/services/blockchain-indexer/shared/indexer/utils/events.js +++ b/services/blockchain-indexer/shared/indexer/utils/events.js @@ -27,12 +27,7 @@ const { }, } = require('lisk-service-framework'); -const { - getGenesisHeight, - EVENT, - EVENT_TOPIC_PREFIX, - LENGTH_TRANSACTION_ID, -} = require('../../constants'); +const { getGenesisHeight, EVENT, EVENT_TOPIC_PREFIX, LENGTH_ID } = require('../../constants'); const config = require('../../../config'); const eventsTableSchema = require('../../database/schema/events'); @@ -84,7 +79,7 @@ const getEventsInfoToIndex = async (block, events) => { // Useful to fetch the relevant events when queried by transactionID if ( topic.startsWith(EVENT_TOPIC_PREFIX.CCM_ID) && - topic.length === EVENT_TOPIC_PREFIX.CCM_ID.length + LENGTH_TRANSACTION_ID + topic.length === EVENT_TOPIC_PREFIX.CCM_ID.length + LENGTH_ID ) { const commandExecResultEvent = events .slice(eventIndex) @@ -93,7 +88,7 @@ const getEventsInfoToIndex = async (block, events) => { const [topicTransactionID] = commandExecResultEvent.topics; const transactionID = // Remove the topic prefix from transactionID before indexing - topicTransactionID.length === EVENT_TOPIC_PREFIX.TX_ID.length + LENGTH_TRANSACTION_ID + topicTransactionID.length === EVENT_TOPIC_PREFIX.TX_ID.length + LENGTH_ID ? topicTransactionID.slice(EVENT_TOPIC_PREFIX.TX_ID.length) : topicTransactionID; From 012ffaa81113dd097fcd71e32fa73f5acdea23e7 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi Date: Wed, 25 Oct 2023 11:25:29 +0200 Subject: [PATCH 3/5] :hammer: Update gateway/docs to allow querying of events with special topic prefixes --- docs/api/version3.md | 2 +- services/gateway/apis/http-version3/methods/events.js | 2 +- services/gateway/shared/regex.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/version3.md b/docs/api/version3.md index e19c8c286d..1b9b0da304 100644 --- a/docs/api/version3.md +++ b/docs/api/version3.md @@ -757,7 +757,7 @@ _Supports pagination._ | --------- | ---- | ---------- | ------- | ------- | | transactionID | String | `/^\b(?:[A-Fa-f0-9]){64}\b$/` | *(empty)* | | | senderAddress | String | `/^lsk[a-hjkm-z2-9]{38}$/` | *(empty)* | | -| topic | String | `/^\b(?:[0-9a-fA-F]{2,64}\|lsk[a-hjkm-z2-9]{38})(?:,(?:[0-9a-fA-F]{2,64}\|lsk[a-hjkm-z2-9]{38}))*\b$/` | *(empty)* | Can be expressed as a CSV. | +| topic | String | `/^\b(?:(?:04\|05)?[0-9a-fA-F]{2,64}\|lsk[a-hjkm-z2-9]{38})(?:,(?:(?:04\|05)?[0-9a-fA-F]{2,64}\|lsk[a-hjkm-z2-9]{38}))*\b$/` | *(empty)* | Can be expressed as a CSV. | | blockID | String | `/^\b(?:[A-Fa-f0-9]){64}\b$/` | *(empty)* | | | height | String | `/^(?:(?:\d+)\|(?::(?:\d+))\|(?:(?:\d+):(?:\d+)?))$/` | *(empty)* | Query by height or a height range. Can be expressed as an interval i.e. `1:20` or `1:` or `:20`. Specified values are inclusive. | | timestamp | String | `/^(?:(?:\d+)\|(?::(?:\d+))\|(?:(?:\d+):(?:\d+)?))$/` | *(empty)* | Query by timestamp or a timestamp range. Can be expressed as an interval i.e. `1000000:2000000` or `1000000:` or `:2000000`. Specified values are inclusive. | diff --git a/services/gateway/apis/http-version3/methods/events.js b/services/gateway/apis/http-version3/methods/events.js index 53d8241f2d..624b5b30c7 100644 --- a/services/gateway/apis/http-version3/methods/events.js +++ b/services/gateway/apis/http-version3/methods/events.js @@ -32,7 +32,7 @@ module.exports = { max: 41, pattern: regex.ADDRESS_LISK32, }, - topic: { optional: true, type: 'string', min: 1, pattern: regex.TOPIC_CSV }, + topic: { optional: true, type: 'string', min: 2, pattern: regex.TOPIC_CSV }, module: { optional: true, type: 'string', min: 1, pattern: regex.MODULE }, name: { optional: true, diff --git a/services/gateway/shared/regex.js b/services/gateway/shared/regex.js index 1897b4db4d..1a103a85c5 100644 --- a/services/gateway/shared/regex.js +++ b/services/gateway/shared/regex.js @@ -53,7 +53,7 @@ const COMMAND = MODULE; const MODULE_CSV = /^(?:[0-9a-zA-Z]{1,32})(?:,[0-9a-zA-Z]{1,32})*$/; const COMMAND_CSV = MODULE_CSV; const TOPIC_CSV = - /^\b(?:[0-9a-fA-F]{2,64}|lsk[a-hjkm-z2-9]{38})(?:,(?:[0-9a-fA-F]{2,64}|lsk[a-hjkm-z2-9]{38}))*\b$/; + /^\b(?:(?:04|05)?[0-9a-fA-F]{2,64}|lsk[a-hjkm-z2-9]{38})(?:,(?:(?:04|05)?[0-9a-fA-F]{2,64}|lsk[a-hjkm-z2-9]{38}))*\b$/; const HEX_STRING = /^\b[a-fA-F0-9]+\b$/; const EXCEL_EXPORT_FILENAME = /^\btransactions_([a-fA-F0-9]{8})_(lsk[a-hjkm-z2-9]{38})_((\d{4})-((1[012])|(0?[1-9]))-(([012][1-9])|([123]0)|31))_((\d{4})-((1[012])|(0?[1-9]))-(([012][1-9])|([123]0)|31))\.xlsx\b$/; From 8508f57e34fab258bc793ffdd069d5b582d3756d Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi Date: Wed, 25 Oct 2023 12:34:35 +0200 Subject: [PATCH 4/5] :ok_hand: Apply review feedback --- .../blockchain-indexer/shared/dataService/business/events.js | 3 ++- services/blockchain-indexer/shared/indexer/utils/events.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/blockchain-indexer/shared/dataService/business/events.js b/services/blockchain-indexer/shared/dataService/business/events.js index e2b4a7df1c..dcb747eee5 100644 --- a/services/blockchain-indexer/shared/dataService/business/events.js +++ b/services/blockchain-indexer/shared/dataService/business/events.js @@ -196,7 +196,8 @@ const getEvents = async params => { { whereIn: { property: 'topic', values: topics }, groupBy: 'eventID', - havingRaw: `COUNT(DISTINCT topic) = ${topics.length}`, + // Must be the length of topic.split(',') instead of topics list to ensure that the DB response returns correct number of eventIDs + havingRaw: `COUNT(DISTINCT topic) = ${topic.split(',').length}`, }, ['eventID'], ); diff --git a/services/blockchain-indexer/shared/indexer/utils/events.js b/services/blockchain-indexer/shared/indexer/utils/events.js index 60223369ee..4410772a01 100644 --- a/services/blockchain-indexer/shared/indexer/utils/events.js +++ b/services/blockchain-indexer/shared/indexer/utils/events.js @@ -75,7 +75,7 @@ const getEventsInfoToIndex = async (block, events) => { eventsInfoToIndex.eventTopicsInfo.push(eventTopicInfo); // Add the corresponding transactionID as a topic when not present in the topics list - // i.e. topic starts with the CCM ID prefix + // i.e. only when the topic starts with the CCM ID prefix // Useful to fetch the relevant events when queried by transactionID if ( topic.startsWith(EVENT_TOPIC_PREFIX.CCM_ID) && From d12240f71cd53044283eda2aa738086f5ff767be Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi Date: Wed, 25 Oct 2023 14:25:46 +0200 Subject: [PATCH 5/5] :bug: Ensure the HAVING clause uses number of unique topics --- .../shared/dataService/business/events.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/blockchain-indexer/shared/dataService/business/events.js b/services/blockchain-indexer/shared/dataService/business/events.js index dcb747eee5..95afe22bfe 100644 --- a/services/blockchain-indexer/shared/dataService/business/events.js +++ b/services/blockchain-indexer/shared/dataService/business/events.js @@ -33,6 +33,7 @@ const { requestConnector } = require('../../utils/request'); const { normalizeRangeParam } = require('../../utils/param'); const { parseToJSONCompatObj } = require('../../utils/parser'); const { LENGTH_ID, EVENT_TOPIC_PREFIX } = require('../../constants'); +const { dropDuplicates } = require('../../utils/array'); const MYSQL_ENDPOINT = config.endpoints.mysqlReplica; @@ -176,6 +177,7 @@ const getEvents = async params => { params = remParams; const topics = topic.split(','); + const numUniqueTopics = dropDuplicates(topics).length; topics.forEach(t => { if ( t.startsWith(EVENT_TOPIC_PREFIX.TX_ID) && @@ -196,8 +198,9 @@ const getEvents = async params => { { whereIn: { property: 'topic', values: topics }, groupBy: 'eventID', - // Must be the length of topic.split(',') instead of topics list to ensure that the DB response returns correct number of eventIDs - havingRaw: `COUNT(DISTINCT topic) = ${topic.split(',').length}`, + // Must be the numUniqueTopics from params.topic instead of the length from the updated topics list + // This is to ensure that the DB response returns correct number of eventIDs + havingRaw: `COUNT(DISTINCT topic) = ${numUniqueTopics}`, }, ['eventID'], );