Merge pull request #1897 from LiskHQ/1893-update-events-indexing-logic

Update events indexing logic

docs/api/version3.md

@@ -757,7 +757,7 @@ _Supports pagination._
 | --------- | ---- | ---------- | ------- | ------- |
 | transactionID | String | `/^\b(?:[A-Fa-f0-9]){64}\b$/` | *(empty)* |  |
 | senderAddress | String | `/^lsk[a-hjkm-z2-9]{38}$/` | *(empty)* |  |
-| topic | String | `/^\b(?:[0-9a-fA-F]{2,64}\|lsk[a-hjkm-z2-9]{38})(?:,(?:[0-9a-fA-F]{2,64}\|lsk[a-hjkm-z2-9]{38}))*\b$/` | *(empty)* | Can be expressed as a CSV. |
+| topic | String | `/^\b(?:(?:04\|05)?[0-9a-fA-F]{2,64}\|lsk[a-hjkm-z2-9]{38})(?:,(?:(?:04\|05)?[0-9a-fA-F]{2,64}\|lsk[a-hjkm-z2-9]{38}))*\b$/` | *(empty)* | Can be expressed as a CSV. |
 | blockID | String | `/^\b(?:[A-Fa-f0-9]){64}\b$/` | *(empty)* |  |
 | height | String | `/^(?:(?:\d+)\|(?::(?:\d+))\|(?:(?:\d+):(?:\d+)?))$/` | *(empty)* | Query by height or a height range. Can be expressed as an interval i.e. `1:20` or `1:` or `:20`. Specified values are inclusive. |
 | timestamp | String | `/^(?:(?:\d+)\|(?::(?:\d+))\|(?:(?:\d+):(?:\d+)?))$/` | *(empty)* | Query by timestamp or a timestamp range. Can be expressed as an interval i.e. `1000000:2000000` or `1000000:` or `:2000000`. Specified values are inclusive. |

services/blockchain-indexer/shared/constants.js

@@ -155,6 +155,7 @@ const LENGTH_NETWORK_ID = 1 * 2; // Each byte is represented with 2 nibbles
 const LENGTH_BYTE_SIGNATURE = 64;
 const LENGTH_BYTE_ID = 32;
 const DEFAULT_NUM_OF_SIGNATURES = 1;
+const LENGTH_ID = LENGTH_BYTE_ID * 2; // Each byte is represented with 2 nibbles
 
 const MAX_COMMISSION = BigInt('10000');
 
@@ -180,6 +181,11 @@ const EVENT = Object.freeze({
 	CCM_SEND_SUCCESS: 'ccmSendSuccess',
 });
 
+const EVENT_TOPIC_PREFIX = Object.freeze({
+	TX_ID: '04',
+	CCM_ID: '05',
+});
+
 const TRANSACTION_VERIFY_RESULT = {
 	INVALID: -1,
 	PENDING: 0,
@@ -221,11 +227,13 @@ module.exports = {
 	MODULE_SUB_STORE,
 	COMMAND,
 	EVENT,
+	EVENT_TOPIC_PREFIX,
 	MAX_COMMISSION,
 	KV_STORE_KEY,
 	TRANSACTION_STATUS,
 	TRANSACTION_VERIFY_RESULT,
 	LENGTH_BYTE_SIGNATURE,
 	LENGTH_BYTE_ID,
+	LENGTH_ID,
 	DEFAULT_NUM_OF_SIGNATURES,
 };

services/blockchain-indexer/shared/dataService/business/events.js

@@ -32,6 +32,8 @@ const eventTopicsTableSchema = require('../../database/schema/eventTopics');
 const { requestConnector } = require('../../utils/request');
 const { normalizeRangeParam } = require('../../utils/param');
 const { parseToJSONCompatObj } = require('../../utils/parser');
+const { LENGTH_ID, EVENT_TOPIC_PREFIX } = require('../../constants');
+const { dropDuplicates } = require('../../utils/array');
 
 const MYSQL_ENDPOINT = config.endpoints.mysqlReplica;
 
@@ -122,6 +124,7 @@ const getEvents = async params => {
 		const { transactionID, ...remParams } = params;
 		params = remParams;
 
+		// Special handling of transaction topic prefix is unnecessary here because of the handling for topic param below
 		if (!params.topic) {
 			params.topic = transactionID;
 		} else {
@@ -173,11 +176,31 @@ const getEvents = async params => {
 		const { topic, ...remParams } = params;
 		params = remParams;
 
+		const topics = topic.split(',');
+		const numUniqueTopics = dropDuplicates(topics).length;
+		topics.forEach(t => {
+			if (
+				t.startsWith(EVENT_TOPIC_PREFIX.TX_ID) &&
+				t.length === EVENT_TOPIC_PREFIX.TX_ID.length + LENGTH_ID
+			) {
+				// Check for the transaction ID both with and without the topic prefix
+				topics.push(t.slice(EVENT_TOPIC_PREFIX.TX_ID.length));
+			} else if (
+				t.startsWith(EVENT_TOPIC_PREFIX.CCM_ID) &&
+				t.length === EVENT_TOPIC_PREFIX.CCM_ID.length + LENGTH_ID
+			) {
+				// Check for CCM ID both with and without the topic prefix
+				topics.push(t.slice(EVENT_TOPIC_PREFIX.CCM_ID.length));
+			}
+		});
+
 		const response = await eventTopicsTable.find(
 			{
-				whereIn: { property: 'topic', values: topic.split(',') },
+				whereIn: { property: 'topic', values: topics },
 				groupBy: 'eventID',
-				havingRaw: `COUNT(DISTINCT topic) = ${topic.split(',').length}`,
+				// Must be the numUniqueTopics from params.topic instead of the length from the updated topics list
+				// This is to ensure that the DB response returns correct number of eventIDs
+				havingRaw: `COUNT(DISTINCT topic) = ${numUniqueTopics}`,
 			},
 			['eventID'],
 		);

services/blockchain-indexer/shared/indexer/utils/events.js

@@ -27,7 +27,7 @@ const {
 	},
 } = require('lisk-service-framework');
 
-const { getGenesisHeight } = require('../../constants');
+const { getGenesisHeight, EVENT, EVENT_TOPIC_PREFIX, LENGTH_ID } = require('../../constants');
 
 const config = require('../../../config');
 const eventsTableSchema = require('../../database/schema/events');
@@ -48,7 +48,7 @@ const getEventsInfoToIndex = async (block, events) => {
 		eventTopicsInfo: [],
 	};
 
-	events.forEach(event => {
+	events.forEach((event, eventIndex) => {
 		const eventInfo = {
 			id: event.id,
 			name: event.name,
@@ -60,7 +60,7 @@ const getEventsInfoToIndex = async (block, events) => {
 		};
 
 		// Store whole event when persistence is enabled or block is not finalized yet
-		// Storing event of non-finalized block is required to fetch events of a dropped block
+		// Storing event of non-finalized block is required to fetch events of a deleted block
 		if (!block.isFinal || config.db.isPersistEvents) {
 			eventInfo.eventStr = JSON.stringify(event);
 		}
@@ -73,6 +73,31 @@ const getEventsInfoToIndex = async (block, events) => {
 				topic,
 			};
 			eventsInfoToIndex.eventTopicsInfo.push(eventTopicInfo);
+
+			// Add the corresponding transactionID as a topic when not present in the topics list
+			// i.e. only when the topic starts with the CCM ID prefix
+			// Useful to fetch the relevant events when queried by transactionID
+			if (
+				topic.startsWith(EVENT_TOPIC_PREFIX.CCM_ID) &&
+				topic.length === EVENT_TOPIC_PREFIX.CCM_ID.length + LENGTH_ID
+			) {
+				const commandExecResultEvent = events
+					.slice(eventIndex)
+					.find(e => e.name === EVENT.COMMAND_EXECUTION_RESULT);
+
+				const [topicTransactionID] = commandExecResultEvent.topics;
+
+				const transactionID = // Remove the topic prefix from transactionID before indexing
+					topicTransactionID.length === EVENT_TOPIC_PREFIX.TX_ID.length + LENGTH_ID
+						? topicTransactionID.slice(EVENT_TOPIC_PREFIX.TX_ID.length)
+						: topicTransactionID;
+
+				const eventTopicAdditionalInfo = {
+					eventID: event.id,
+					topic: transactionID,
+				};
+				eventsInfoToIndex.eventTopicsInfo.push(eventTopicAdditionalInfo);
+			}
 		});
 	});
 

services/gateway/apis/http-version3/methods/events.js

@@ -32,7 +32,7 @@ module.exports = {
 			max: 41,
 			pattern: regex.ADDRESS_LISK32,
 		},
-		topic: { optional: true, type: 'string', min: 1, pattern: regex.TOPIC_CSV },
+		topic: { optional: true, type: 'string', min: 2, pattern: regex.TOPIC_CSV },
 		module: { optional: true, type: 'string', min: 1, pattern: regex.MODULE },
 		name: {
 			optional: true,

services/gateway/shared/regex.js

@@ -53,7 +53,7 @@ const COMMAND = MODULE;
 const MODULE_CSV = /^(?:[0-9a-zA-Z]{1,32})(?:,[0-9a-zA-Z]{1,32})*$/;
 const COMMAND_CSV = MODULE_CSV;
 const TOPIC_CSV =
-	/^\b(?:[0-9a-fA-F]{2,64}|lsk[a-hjkm-z2-9]{38})(?:,(?:[0-9a-fA-F]{2,64}|lsk[a-hjkm-z2-9]{38}))*\b$/;
+	/^\b(?:(?:04|05)?[0-9a-fA-F]{2,64}|lsk[a-hjkm-z2-9]{38})(?:,(?:(?:04|05)?[0-9a-fA-F]{2,64}|lsk[a-hjkm-z2-9]{38}))*\b$/;
 const HEX_STRING = /^\b[a-fA-F0-9]+\b$/;
 const EXCEL_EXPORT_FILENAME =
 	/^\btransactions_([a-fA-F0-9]{8})_(lsk[a-hjkm-z2-9]{38})_((\d{4})-((1[012])|(0?[1-9]))-(([012][1-9])|([123]0)|31))_((\d{4})-((1[012])|(0?[1-9]))-(([012][1-9])|([123]0)|31))\.xlsx\b$/;