From b5202dbd51212684db2892561698431d937a5cd4 Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Fri, 1 Mar 2024 21:24:28 +1100 Subject: [PATCH 01/16] Update BlockchainApiRoute.scala --- .../http/api/BlockchainApiRoute.scala | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 0be225d035..8da9f2c26f 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -242,37 +242,45 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting validateAndGetBoxesByAddress(address, offset, limit) } - private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Int, limit: Int, sortDir: Direction, unconfirmed: Boolean): Future[Seq[IndexedErgoBox]] = + private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Int, limit: Int, sortDir: Direction, unconfirmed: Boolean, excludeMempoolSpent: Boolean): Future[Seq[IndexedErgoBox]] = getHistoryWithMempool.map { case (history, mempool) => - getAddress(addr)(history) - .getOrElse(IndexedErgoAddress(hashErgoTree(addr.script))) - .retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed) + val addressUtxos = getAddress(addr)(history) + .getOrElse(IndexedErgoAddress(hashErgoTree(addr.script))) + .retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed) + if (excludeMempoolSpent) { + // If excluding boxes spent in mempool, filter out those boxes + val spentBoxesIdsInMempool = mempool.spentInputs.toSet + addressUtxos.filterNot(box => spentBoxesIdsInMempool.contains(box.id)) + } else { + addressUtxos } + } private def validateAndGetBoxesByAddressUnspent(address: ErgoAddress, offset: Int, limit: Int, dir: Direction, - unconfirmed: Boolean): Route = { + unconfirmed: Boolean, + excludeMempoolSpent: Boolean): Route = { if (limit > MaxItems) { BadRequest(s"No more than $MaxItems boxes can be requested") } else if (dir == SortDirection.INVALID) { BadRequest("Invalid parameter for sort direction, valid values are \"ASC\" and \"DESC\"") } else { - ApiResponse(getBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed)) + ApiResponse(getBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed, excludeMempoolSpent)) } } private def getBoxesByAddressUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging & sortDir & unconfirmed) { - (address, offset, limit, dir, unconfirmed) => - validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed) + (address, offset, limit, dir, unconfirmed, excludeMempoolSpent) => + validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed, excludeMempoolSpent) } private def getBoxesByAddressUnspentGetRoute: Route = (pathPrefix("box" / "unspent" / "byAddress") & get & addressPass & paging & sortDir & unconfirmed) { - (address, offset, limit, dir, unconfirmed) => - validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed) + (address, offset, limit, dir, unconfirmed, excludeMempoolSpent) => + validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed, excludeMempoolSpent) } private def getBoxRange(offset: Int, limit: Int): Future[Seq[ModifierId]] = From ed8acdfb65acaea266ea88ff4312222e42d41b25 Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Fri, 1 Mar 2024 21:26:26 +1100 Subject: [PATCH 02/16] Update openapi.yaml --- src/main/resources/api/openapi.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 24062ad56d..914cc3815f 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -6413,6 +6413,13 @@ paths: schema: type: boolean default: false + - in: query + name: exludeMempoolSpent + required: false + description: if true exclude spent inputs from mempool + schema: + type: boolean + default: false responses: '200': description: unspent boxes associated with wanted address From 2669890837c900326c9bfe2cad103994cc4dc0eb Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Fri, 1 Mar 2024 21:26:39 +1100 Subject: [PATCH 03/16] Update openapi-ai.yaml --- src/main/resources/api/openapi-ai.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/resources/api/openapi-ai.yaml b/src/main/resources/api/openapi-ai.yaml index 1e562ff0f5..41696443fe 100644 --- a/src/main/resources/api/openapi-ai.yaml +++ b/src/main/resources/api/openapi-ai.yaml @@ -1390,6 +1390,13 @@ paths: schema: type: string default: desc + - in: query + name: exludeMempoolSpent + required: false + description: if true exclude spent inputs from mempool + schema: + type: boolean + default: false responses: '200': description: unspent boxes associated with wanted address @@ -1479,4 +1486,4 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ApiError' \ No newline at end of file + $ref: '#/components/schemas/ApiError' From f211055c128c3e61e3be5634a18c37528ca0367d Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Fri, 15 Mar 2024 23:05:29 +1100 Subject: [PATCH 04/16] Fix typo for param name --- src/main/resources/api/openapi-ai.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/api/openapi-ai.yaml b/src/main/resources/api/openapi-ai.yaml index 41696443fe..61dd8bb9b2 100644 --- a/src/main/resources/api/openapi-ai.yaml +++ b/src/main/resources/api/openapi-ai.yaml @@ -1391,7 +1391,7 @@ paths: type: string default: desc - in: query - name: exludeMempoolSpent + name: excludeMempoolSpent required: false description: if true exclude spent inputs from mempool schema: From e1f82868a5a78d814aeec95f51fde41a86e80dfb Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Fri, 15 Mar 2024 23:05:52 +1100 Subject: [PATCH 05/16] Fix typo for param name --- src/main/resources/api/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 914cc3815f..5384aa8124 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -6414,7 +6414,7 @@ paths: type: boolean default: false - in: query - name: exludeMempoolSpent + name: excludeMempoolSpent required: false description: if true exclude spent inputs from mempool schema: From 1aef809c372b645253615a3c78b46e6045f5c62c Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Wed, 20 Mar 2024 01:28:42 +1100 Subject: [PATCH 06/16] Add recursive search for utxos --- .../http/api/BlockchainApiRoute.scala | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 8da9f2c26f..09ed609c0f 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -256,6 +256,43 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } } + private def getBoxesByAddressUnspent( + addr: ErgoAddress, + offset: Int, + limit: Int, + sortDir: Direction, + unconfirmed: Boolean, + excludeMempoolSpent: Boolean +): Future[Seq[IndexedErgoBox]] = { + + def fetchAndFilter(limit: Int, accumulated: Seq[IndexedErgoBox] = Seq.empty): Future[Seq[IndexedErgoBox]] = { + getHistoryWithMempool.flatMap { case (history, mempool) => + val addressUtxos = getAddress(addr)(history) + .getOrElse(IndexedErgoAddress(hashErgoTree(addr.script))) + .retrieveUtxos(history, mempool, offset + accumulated.length, limit, sortDir, unconfirmed) + + val newUtxos = if (excludeMempoolSpent) { + val spentBoxesIdsInMempool = mempool.spentInputs.toSet + addressUtxos.filterNot(box => spentBoxesIdsInMempool.contains(box.id)) + } else { + addressUtxos + } + + val updatedAccumulated = accumulated ++ newUtxos + // If reached limit OR we have obtained the maximumm available UTXOS (returned amount < limit), return successful. + if (updatedAccumulated.length >= originalLimit || addressUtxos.length < limit) { + Future.successful(updatedAccumulated.take(originalLimit)) // Just to ensure that no more than limit is taken + } else { + val maxLimit = 200; + val newLimit = Math.min(limit * 2, maxLimit); // Prevents limit becoming too large + fetchAndFilter(newLimit, updatedAccumulated) + } + } + } + val originalLimit = limit + fetchAndFilter(originalLimit) +} + private def validateAndGetBoxesByAddressUnspent(address: ErgoAddress, offset: Int, limit: Int, From 7ee480c4ae78a4e32095fdcb9f725dd7bb679754 Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Wed, 20 Mar 2024 01:30:49 +1100 Subject: [PATCH 07/16] Remove old implementation --- .../ergoplatform/http/api/BlockchainApiRoute.scala | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 09ed609c0f..727d7e902b 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -242,20 +242,6 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting validateAndGetBoxesByAddress(address, offset, limit) } - private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Int, limit: Int, sortDir: Direction, unconfirmed: Boolean, excludeMempoolSpent: Boolean): Future[Seq[IndexedErgoBox]] = - getHistoryWithMempool.map { case (history, mempool) => - val addressUtxos = getAddress(addr)(history) - .getOrElse(IndexedErgoAddress(hashErgoTree(addr.script))) - .retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed) - if (excludeMempoolSpent) { - // If excluding boxes spent in mempool, filter out those boxes - val spentBoxesIdsInMempool = mempool.spentInputs.toSet - addressUtxos.filterNot(box => spentBoxesIdsInMempool.contains(box.id)) - } else { - addressUtxos - } - } - private def getBoxesByAddressUnspent( addr: ErgoAddress, offset: Int, From d97effd08666e004afce6dd8105ae474cb0b964f Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Mon, 25 Mar 2024 18:33:30 +1100 Subject: [PATCH 08/16] Attempt fix for bad signatures --- .../ergoplatform/http/api/BlockchainApiRoute.scala | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 727d7e902b..aacd62fc7e 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -295,17 +295,20 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } private def getBoxesByAddressUnspentR: Route = - (post & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging & sortDir & unconfirmed) { - (address, offset, limit, dir, unconfirmed, excludeMempoolSpent) => + (post & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging & sortDir & unconfirmed & parameter('excludeMempoolSpent.as[Boolean].?)) { + (address, offset, limit, dir, unconfirmed, excludeMempoolSpentOption) => + val excludeMempoolSpent = excludeMempoolSpentOption.getOrElse(false) validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed, excludeMempoolSpent) } private def getBoxesByAddressUnspentGetRoute: Route = - (pathPrefix("box" / "unspent" / "byAddress") & get & addressPass & paging & sortDir & unconfirmed) { - (address, offset, limit, dir, unconfirmed, excludeMempoolSpent) => + (pathPrefix("box" / "unspent" / "byAddress") & get & addressPass & paging & sortDir & unconfirmed & parameter('excludeMempoolSpent.as[Boolean].?)) { + (address, offset, limit, dir, unconfirmed, excludeMempoolSpentOption) => + val excludeMempoolSpent = excludeMempoolSpentOption.getOrElse(false) validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed, excludeMempoolSpent) } + private def getBoxRange(offset: Int, limit: Int): Future[Seq[ModifierId]] = getHistory.map { history => val base: Long = getIndex(GlobalBoxIndexKey, history).getLong - offset From 6783dc53f58ba8a07a622b5df29153059b40c8d0 Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Mon, 25 Mar 2024 18:46:38 +1100 Subject: [PATCH 09/16] Attempt to fix type mismatch --- .../org/ergoplatform/http/api/BlockchainApiRoute.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index aacd62fc7e..0bfbca0f03 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -257,13 +257,13 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting .getOrElse(IndexedErgoAddress(hashErgoTree(addr.script))) .retrieveUtxos(history, mempool, offset + accumulated.length, limit, sortDir, unconfirmed) + val spentBoxesIdsInMempool: Set[Array[Byte]] = mempool.spentInputs.map(idToBytes).toSet val newUtxos = if (excludeMempoolSpent) { - val spentBoxesIdsInMempool = mempool.spentInputs.toSet - addressUtxos.filterNot(box => spentBoxesIdsInMempool.contains(box.id)) + addressUtxos.filterNot(box => spentBoxesIdsInMempool.contains(idToBytes(box.id))) } else { addressUtxos - } - + } + val updatedAccumulated = accumulated ++ newUtxos // If reached limit OR we have obtained the maximumm available UTXOS (returned amount < limit), return successful. if (updatedAccumulated.length >= originalLimit || addressUtxos.length < limit) { From 3f9cb459a8e13b9085e33bf4e20be1c9a497e4c9 Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:08:41 +1100 Subject: [PATCH 10/16] Fix filtering of spent boxes in mempool using ModifierId --- .../org/ergoplatform/http/api/BlockchainApiRoute.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 0bfbca0f03..48a87bfe6f 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -251,15 +251,17 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting excludeMempoolSpent: Boolean ): Future[Seq[IndexedErgoBox]] = { + val originalLimit = limit + def fetchAndFilter(limit: Int, accumulated: Seq[IndexedErgoBox] = Seq.empty): Future[Seq[IndexedErgoBox]] = { getHistoryWithMempool.flatMap { case (history, mempool) => val addressUtxos = getAddress(addr)(history) .getOrElse(IndexedErgoAddress(hashErgoTree(addr.script))) .retrieveUtxos(history, mempool, offset + accumulated.length, limit, sortDir, unconfirmed) - val spentBoxesIdsInMempool: Set[Array[Byte]] = mempool.spentInputs.map(idToBytes).toSet + val spentBoxesIdsInMempool: Set[ModifierId] = mempool.spentInputs.map(bytesToId).toSet val newUtxos = if (excludeMempoolSpent) { - addressUtxos.filterNot(box => spentBoxesIdsInMempool.contains(idToBytes(box.id))) + addressUtxos.filterNot(box => spentBoxesIdsInMempool.contains(box.id)) } else { addressUtxos } @@ -271,11 +273,11 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } else { val maxLimit = 200; val newLimit = Math.min(limit * 2, maxLimit); // Prevents limit becoming too large - fetchAndFilter(newLimit, updatedAccumulated) + fetchAndFilter(newLimit, updatedAccumulated) } } } - val originalLimit = limit + fetchAndFilter(originalLimit) } From beac0a223fb3445ffba3195a66f8242cbc729c80 Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Fri, 19 Apr 2024 22:28:23 +1000 Subject: [PATCH 11/16] Pass excludeMempoolSpent into retrieveUtxos --- .../http/api/BlockchainApiRoute.scala | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 48a87bfe6f..8ef0b5f035 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -255,24 +255,18 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting def fetchAndFilter(limit: Int, accumulated: Seq[IndexedErgoBox] = Seq.empty): Future[Seq[IndexedErgoBox]] = { getHistoryWithMempool.flatMap { case (history, mempool) => + val spentBoxesIdsInMempool = if (excludeMempoolSpent) mempool.spentInputs.map(bytesToId).toSet else Set.empty[ModifierId] + val addressUtxos = getAddress(addr)(history) .getOrElse(IndexedErgoAddress(hashErgoTree(addr.script))) - .retrieveUtxos(history, mempool, offset + accumulated.length, limit, sortDir, unconfirmed) + .retrieveUtxos(history, mempool, offset + accumulated.length, limit, sortDir, unconfirmed, spentBoxesIdsInMempool) - val spentBoxesIdsInMempool: Set[ModifierId] = mempool.spentInputs.map(bytesToId).toSet - val newUtxos = if (excludeMempoolSpent) { - addressUtxos.filterNot(box => spentBoxesIdsInMempool.contains(box.id)) - } else { - addressUtxos - } - - val updatedAccumulated = accumulated ++ newUtxos - // If reached limit OR we have obtained the maximumm available UTXOS (returned amount < limit), return successful. + val updatedAccumulated = accumulated ++ addressUtxos if (updatedAccumulated.length >= originalLimit || addressUtxos.length < limit) { - Future.successful(updatedAccumulated.take(originalLimit)) // Just to ensure that no more than limit is taken + Future.successful(updatedAccumulated.take(originalLimit)) } else { - val maxLimit = 200; - val newLimit = Math.min(limit * 2, maxLimit); // Prevents limit becoming too large + val maxLimit = 200 + val newLimit = Math.min(limit * 2, maxLimit) fetchAndFilter(newLimit, updatedAccumulated) } } From 3a2ab2bf2634fd11faf6b9926cb4f41c5019c6e0 Mon Sep 17 00:00:00 2001 From: anon-yum2 <161828021+anon-yum2@users.noreply.github.com> Date: Fri, 19 Apr 2024 22:29:29 +1000 Subject: [PATCH 12/16] Add overloaded retrieveUtxos --- .../nodeView/history/extra/Segment.scala | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala index c45a5c9595..c42c5c1ec3 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala @@ -287,6 +287,60 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId, confirmedBoxes } + + /** + * Overloaded retrieveUtxos for mempool filtering + * Get a range of the boxes associated with the parent that are NOT spent + * + * @param history - history to use + * @param mempool - mempool to use, if unconfirmed is true + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @param sortDir - whether to start retreival from newest box ([[DESC]]) or oldest box ([[ASC]]) + * @param unconfirmed - whether to include unconfirmed boxes + * @param spentBoxesIdsInMempool - Set of box IDs that are spent in the mempool (to be excluded if necessary) + * @return array of unspent boxes + */ + def retrieveUtxos(history: ErgoHistoryReader, + mempool: ErgoMemPoolReader, + offset: Int, + limit: Int, + sortDir: Direction, + unconfirmed: Boolean, + spentBoxesIdsInMempool: Set[ModifierId]): Seq[IndexedErgoBox] = { + val data: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] + val confirmedBoxes: Seq[IndexedErgoBox] = sortDir match { + case DESC => + data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id)) + var segment: Int = boxSegmentCount + while(data.length < (limit + offset) && segment > 0) { + segment -= 1 + history.typedExtraIndexById[T](idMod(boxSegmentId(parentId, segment))).get.boxes + .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id)) ++=: data + } + data.reverse.slice(offset, offset + limit) + case ASC => + var segment: Int = 0 + while(data.length < (limit + offset) && segment < boxSegmentCount) { + data ++= history.typedExtraIndexById[T](idMod(boxSegmentId(parentId, segment))).get.boxes + .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id)) + segment += 1 + } + if (data.length < (limit + offset)) + data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id)) + data.slice(offset, offset + limit) + } + if(unconfirmed) { + val mempoolBoxes = filterMempool(mempool.getAll.flatMap(_.transaction.outputs)) + val unconfirmedBoxes = mempoolBoxes.map(new IndexedErgoBox(0, None, None, _, 0)).filterNot(box => spentBoxesIdsInMempool.contains(box.id)) + sortDir match { + case DESC => unconfirmedBoxes ++ confirmedBoxes + case ASC => confirmedBoxes ++ unconfirmedBoxes + } + } else + confirmedBoxes + } + /** * Logic for [[Segment.rollback]] * From f49c2d9fc05ada972f0d7bbe67f9d6005fcd0ead Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Sat, 20 Jul 2024 22:33:53 +0300 Subject: [PATCH 13/16] 5.0.22 version set --- src/main/resources/api/openapi-ai.yaml | 2 +- src/main/resources/api/openapi.yaml | 2 +- src/main/resources/application.conf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/resources/api/openapi-ai.yaml b/src/main/resources/api/openapi-ai.yaml index f6cbea204d..1cadbe01e6 100644 --- a/src/main/resources/api/openapi-ai.yaml +++ b/src/main/resources/api/openapi-ai.yaml @@ -1,7 +1,7 @@ openapi: "3.0.2" info: - version: "5.0.21" + version: "5.0.22" title: Ergo Node API description: Specification of Ergo Node API for ChatGPT plugin. The following endpoints supported diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index a49f06f81c..d77c0d0f5f 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.2" info: - version: "5.0.21" + version: "5.0.22" title: Ergo Node API description: API docs for Ergo Node. Models are shared between all Ergo products contact: diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 7d30c0ea97..37c2406a53 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -436,7 +436,7 @@ scorex { nodeName = "ergo-node" # Network protocol version to be sent in handshakes - appVersion = 5.0.21 + appVersion = 5.0.22 # Network agent name. May contain information about client code # stack, starting from core code-base up to the end graphical interface. From b581a5b4e5a585c79677f949fe9db6386c1f82eb Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Fri, 26 Jul 2024 13:02:48 +0300 Subject: [PATCH 14/16] simplifying #2131 --- .../http/api/BlockchainApiRoute.scala | 41 +++------- .../nodeView/history/extra/Segment.scala | 80 ++++--------------- 2 files changed, 28 insertions(+), 93 deletions(-) diff --git a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala index 8ef0b5f035..aa8e6aa7df 100644 --- a/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala @@ -243,38 +243,23 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting } private def getBoxesByAddressUnspent( - addr: ErgoAddress, - offset: Int, - limit: Int, - sortDir: Direction, - unconfirmed: Boolean, - excludeMempoolSpent: Boolean -): Future[Seq[IndexedErgoBox]] = { - - val originalLimit = limit - - def fetchAndFilter(limit: Int, accumulated: Seq[IndexedErgoBox] = Seq.empty): Future[Seq[IndexedErgoBox]] = { - getHistoryWithMempool.flatMap { case (history, mempool) => + addr: ErgoAddress, + offset: Int, + limit: Int, + sortDir: Direction, + unconfirmed: Boolean, + excludeMempoolSpent: Boolean + ): Future[Seq[IndexedErgoBox]] = { + + getHistoryWithMempool.map { case (history, mempool) => val spentBoxesIdsInMempool = if (excludeMempoolSpent) mempool.spentInputs.map(bytesToId).toSet else Set.empty[ModifierId] - val addressUtxos = getAddress(addr)(history) + getAddress(addr)(history) .getOrElse(IndexedErgoAddress(hashErgoTree(addr.script))) - .retrieveUtxos(history, mempool, offset + accumulated.length, limit, sortDir, unconfirmed, spentBoxesIdsInMempool) - - val updatedAccumulated = accumulated ++ addressUtxos - if (updatedAccumulated.length >= originalLimit || addressUtxos.length < limit) { - Future.successful(updatedAccumulated.take(originalLimit)) - } else { - val maxLimit = 200 - val newLimit = Math.min(limit * 2, maxLimit) - fetchAndFilter(newLimit, updatedAccumulated) - } + .retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed, spentBoxesIdsInMempool) } } - fetchAndFilter(originalLimit) -} - private def validateAndGetBoxesByAddressUnspent(address: ErgoAddress, offset: Int, limit: Int, @@ -343,7 +328,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting getHistoryWithMempool.map { case (history, mempool) => getAddress(tree)(history) .getOrElse(IndexedErgoAddress(hashErgoTree(tree))) - .retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed) + .retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed, Set.empty) } private def getBoxesByErgoTreeUnspentR: Route = (post & pathPrefix("box" / "unspent" / "byErgoTree") & ergoTree & paging & sortDir & unconfirmed) { (tree, offset, limit, dir, unconfirmed) => @@ -382,7 +367,7 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting getHistoryWithMempool.map { case (history, mempool) => history.typedExtraIndexById[IndexedToken](uniqueId(id)) .getOrElse(IndexedToken(id)) - .retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed) + .retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed, Set.empty) } private def getBoxesByTokenIdUnspentR: Route = (get & pathPrefix("box" / "unspent" / "byTokenId") & modifierId & paging & sortDir & unconfirmed) { (id, offset, limit, dir, unconfirmed) => diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala index c42c5c1ec3..98e9ff9660 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala @@ -237,70 +237,20 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId, def retrieveBoxes(history: ErgoHistoryReader, offset: Int, limit: Int)(implicit segmentTreshold: Int): Array[IndexedErgoBox] = getFromSegments(history, offset, limit, boxSegmentCount, boxes, boxSegmentId, _.boxes, getBoxes) - /** - * Get a range of the boxes associated with the parent that are NOT spent - * - * @param history - history to use - * @param mempool - mempool to use, if unconfirmed is true - * @param offset - items to skip from the start - * @param limit - items to retrieve - * @param sortDir - whether to start retreival from newest box ([[DESC]]) or oldest box ([[ASC]]) - * @param unconfirmed - whether to include unconfirmed boxes - * @return array of unspent boxes - */ - def retrieveUtxos(history: ErgoHistoryReader, - mempool: ErgoMemPoolReader, - offset: Int, - limit: Int, - sortDir: Direction, - unconfirmed: Boolean): Seq[IndexedErgoBox] = { - val data: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox] - val confirmedBoxes: Seq[IndexedErgoBox] = sortDir match { - case DESC => - data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get) - var segment: Int = boxSegmentCount - while(data.length < (limit + offset) && segment > 0) { - segment -= 1 - history.typedExtraIndexById[T](idMod(boxSegmentId(parentId, segment))).get.boxes - .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get) ++=: data - } - data.reverse.slice(offset, offset + limit) - case ASC => - var segment: Int = 0 - while(data.length < (limit + offset) && segment < boxSegmentCount) { - data ++= history.typedExtraIndexById[T](idMod(boxSegmentId(parentId, segment))).get.boxes - .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get) - segment += 1 - } - if (data.length < (limit + offset)) - data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get) - data.slice(offset, offset + limit) - } - if(unconfirmed) { - val mempoolBoxes = filterMempool(mempool.getAll.flatMap(_.transaction.outputs)) - val unconfirmedBoxes = mempoolBoxes.map(new IndexedErgoBox(0, None, None, _, 0)) - sortDir match { - case DESC => unconfirmedBoxes ++ confirmedBoxes - case ASC => confirmedBoxes ++ unconfirmedBoxes - } - } else - confirmedBoxes - } - /** - * Overloaded retrieveUtxos for mempool filtering - * Get a range of the boxes associated with the parent that are NOT spent - * - * @param history - history to use - * @param mempool - mempool to use, if unconfirmed is true - * @param offset - items to skip from the start - * @param limit - items to retrieve - * @param sortDir - whether to start retreival from newest box ([[DESC]]) or oldest box ([[ASC]]) - * @param unconfirmed - whether to include unconfirmed boxes - * @param spentBoxesIdsInMempool - Set of box IDs that are spent in the mempool (to be excluded if necessary) - * @return array of unspent boxes - */ + * Overloaded retrieveUtxos for mempool filtering + * Get a range of the boxes associated with the parent that are NOT spent + * + * @param history - history to use + * @param mempool - mempool to use, if unconfirmed is true + * @param offset - items to skip from the start + * @param limit - items to retrieve + * @param sortDir - whether to start retrieval from newest box ([[DESC]]) or oldest box ([[ASC]]) + * @param unconfirmed - whether to include unconfirmed boxes + * @param spentBoxesIdsInMempool - Set of box IDs that are spent in the mempool (to be excluded if necessary) + * @return array of unspent boxes + */ def retrieveUtxos(history: ErgoHistoryReader, mempool: ErgoMemPoolReader, offset: Int, @@ -313,7 +263,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId, case DESC => data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id)) var segment: Int = boxSegmentCount - while(data.length < (limit + offset) && segment > 0) { + while (data.length < (limit + offset) && segment > 0) { segment -= 1 history.typedExtraIndexById[T](idMod(boxSegmentId(parentId, segment))).get.boxes .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id)) ++=: data @@ -321,7 +271,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId, data.reverse.slice(offset, offset + limit) case ASC => var segment: Int = 0 - while(data.length < (limit + offset) && segment < boxSegmentCount) { + while (data.length < (limit + offset) && segment < boxSegmentCount) { data ++= history.typedExtraIndexById[T](idMod(boxSegmentId(parentId, segment))).get.boxes .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id)) segment += 1 @@ -330,7 +280,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId, data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id)) data.slice(offset, offset + limit) } - if(unconfirmed) { + if (unconfirmed) { val mempoolBoxes = filterMempool(mempool.getAll.flatMap(_.transaction.outputs)) val unconfirmedBoxes = mempoolBoxes.map(new IndexedErgoBox(0, None, None, _, 0)).filterNot(box => spentBoxesIdsInMempool.contains(box.id)) sortDir match { From 0821f4b4c6baec262f45a6d951e9943e2d3f5468 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Fri, 26 Jul 2024 13:32:58 +0300 Subject: [PATCH 15/16] merging #2167 --- README.md | 2 +- src/main/resources/mainnet.conf | 4 +- .../nodeView/history/extra/Segment.scala | 111 +++++++---- .../history/storage/HistoryStorage.scala | 10 +- .../nodeView/wallet/ErgoWalletService.scala | 2 +- .../nodeView/history/extra/SegmentSpec.scala | 187 ++++++++++++++++++ 6 files changed, 264 insertions(+), 52 deletions(-) create mode 100644 src/test/scala/org/ergoplatform/nodeView/history/extra/SegmentSpec.scala diff --git a/README.md b/README.md index 9efcc702f2..b49d4aecea 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ By default, the node processes all blocks from the genesis block. However, there ergo { ... - node.utxoBootstrap = true + node.utxo.utxoBootstrap = true ... } ``` diff --git a/src/main/resources/mainnet.conf b/src/main/resources/mainnet.conf index 594044789e..dfc6850e84 100644 --- a/src/main/resources/mainnet.conf +++ b/src/main/resources/mainnet.conf @@ -70,8 +70,8 @@ ergo { # # To validate all the scripts for all the blocks, set checkpoint = null. checkpoint = { - height = 1020454 - blockId = "7829c6513c5b319b86e87253ed29d51fed61597c65e32f5197434461ccc4c905" + height = 1231454 + blockId = "ca5aa96a2d560f49cd5652eae4b9e16bbf410ee32365313dc16544ee5fda1e6d" } # List with hex-encoded identifiers of transactions banned from getting into memory pool diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala index 98e9ff9660..af6e501f88 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala @@ -22,8 +22,8 @@ import scala.reflect.ClassTag * This mechanism is used to prevent excessive serialization/deserialization delays caused by objects with a lot of transaction/box indexes. * @param parentId - identifier of parent object * @param factory - parent object factory - * @param txs - list of numberic transaction indexes - * @param boxes - list of numberic box indexes, negative values indicate the box is spent + * @param txs - list of numeric transaction indexes + * @param boxes - list of numeric box indexes, negative values indicate the box is spent * @param idMod - function to apply to ids during segmentation and db lookup * @tparam T - type of parent object */ @@ -55,12 +55,12 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId, /** * @return total number of boxes associated with the parent object */ - def boxCount(implicit segmentTreshold: Int): Long = segmentTreshold * boxSegmentCount + boxes.length + def boxCount(implicit segmentThreshold: Int): Long = segmentThreshold * boxSegmentCount + boxes.length /** * @return total number of transactions associated with the parent object */ - def txCount(implicit segmentTreshold: Int): Long = segmentTreshold * txSegmentCount + txs.length + def txCount(implicit segmentThreshold: Int): Long = segmentThreshold * txSegmentCount + txs.length /** * Locate which segment the given box number is in and change its sign, meaning it spends unspent boxes and vice versa. @@ -141,20 +141,11 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId, * @param limit - items to retrieve * @return array of offsets */ - private def getSegmentsForRange(offset: Int, limit: Int)(implicit segmentTreshold: Int): Array[Int] = - (math.max(math.floor(offset * 1F / segmentTreshold).toInt, 1) to math.ceil((offset + limit) * 1F / segmentTreshold).toInt).toArray - - /** - * Get a range of elements from an ArrayBuffer by removing the last "offset" elements, - * then getting the last "limit" elements reversed. - * - * @param arr - array to get range from - * @param offset - number of items to skip from the end - * @param limit - number of items to retrieve - * @return a reversed range in "arr" ArrayBuffer - */ - private def sliceReversed(arr: ArrayBuffer[Long], offset: Int, limit: Int): ArrayBuffer[Long] = - arr.slice(arr.length - limit - offset, arr.length - offset).reverse + private[extra] def getSegmentsForRange(offset: Int, limit: Int)(implicit segmentTreshold: Int): Array[Int] = { + val floor = math.max(math.floor(offset * 1F / segmentTreshold).toInt, 0) + val ceil = math.ceil((offset + limit) * 1F / segmentTreshold).toInt + (floor to ceil).toArray + } /** * Get an array of transactions with full bodies from an array of numeric transaction indexes @@ -163,8 +154,9 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId, * @param history - database handle * @return array of transactions with full bodies */ - private def getTxs(arr: ArrayBuffer[Long], history: ErgoHistoryReader): Array[IndexedErgoTransaction] = // sorted to match explorer - arr.map(n => NumericTxIndex.getTxByNumber(history, n).get.retrieveBody(history)).toArray.sortBy(tx => (-tx.height, tx.id)) + private def getTxs(arr: ArrayBuffer[Long], history: ErgoHistoryReader): Array[IndexedErgoTransaction] = { + arr.map(n => NumericTxIndex.getTxByNumber(history, n).get.retrieveBody(history)).toArray + } /** * Get an array of boxes from an array of numeric box indexes @@ -186,33 +178,66 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId, * @param array - the indexes already in memory * @param idOf - function to calculate segment ids, either [[txSegmentId]] or [[boxSegmentId]] * @param arraySelector - function to select index array from retreived segments - * @param retreive - function to retreive indexes from database + * @param retrieve - function to retrieve indexes from database * @tparam B - type of desired indexes, either [[IndexedErgoTransaction]] or [[IndexedErgoBox]] * @return */ - private def getFromSegments[B: ClassTag](history: ErgoHistoryReader, - offset: Int, - limit: Int, - segmentCount: Int, - array: ArrayBuffer[Long], - idOf: (ModifierId, Int) => ModifierId, - arraySelector: T => ArrayBuffer[Long], - retreive: (ArrayBuffer[Long], ErgoHistoryReader) => Array[B]) - (implicit segmentTreshold: Int): Array[B] = { + private[extra] def getFromSegments[B: ClassTag](history: ErgoHistoryReader, + offset: Int, + limit: Int, + segmentCount: Int, + arr: ArrayBuffer[Long], + idOf: (ModifierId, Int) => ModifierId, + arraySelector: T => ArrayBuffer[Long], + retrieve: (ArrayBuffer[Long], ErgoHistoryReader) => Array[B], + txsFlag: Boolean) + (implicit segmentTreshold: Int): Array[B] = { + val array = if(txsFlag) { + arr.reverse + } else { + arr + } + val total: Int = segmentTreshold * segmentCount + array.length - if(offset >= total) + if (offset >= total) return Array.empty[B] // return empty array if all elements are skipped - if(offset + limit > array.length && segmentCount > 0) { - val data: ArrayBuffer[Long] = ArrayBuffer.empty[Long] - getSegmentsForRange(offset, limit).map(n => math.max(segmentCount - n, 0)).distinct.foreach { num => - arraySelector( - history.typedExtraIndexById[T](idMod(idOf(parentId, num))).get - ) ++=: data + if (offset + limit > array.length && segmentCount > 0) { + + val target = offset + limit + + val collected: ArrayBuffer[Long] = ArrayBuffer.empty[Long] + collected ++= (if (offset < array.length) array.slice(offset, Math.min(offset + limit, array.length)) else Nil) + val segments = getSegmentsForRange(offset - array.length, limit).map(n => math.min(segmentCount - 1, n)).distinct + segments.foreach { num => + val lowerBound = array.length + num * segmentTreshold + val upperBound = lowerBound + segmentTreshold + + if (collected.length < limit && target > lowerBound) { + val arr = if(txsFlag) { + arraySelector( + history.typedExtraIndexById[T](idMod(idOf(parentId, (segmentCount - 1) - num))).get + ).reverse + } else { + arraySelector( + history.typedExtraIndexById[T](idMod(idOf(parentId, num))).get + ).reverse + } + if (target > upperBound) { + collected ++= arr.slice(offset - lowerBound, arr.size) + } else { + if (offset > lowerBound) { + collected ++= arr.slice(offset - lowerBound, offset - lowerBound + limit) + } else { + collected ++= arr.slice(0, target - lowerBound) + } + } + } } - data ++= (if(offset < array.length) array else Nil) - retreive(sliceReversed(data, offset % segmentTreshold, math.min(total - offset, limit)), history) - } else - retreive(sliceReversed(array, offset, limit), history) + + retrieve(collected, history) + } else { + retrieve(array.slice(offset, offset + limit), history) + } } /** @@ -224,7 +249,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId, * @return array of transactions with full bodies */ def retrieveTxs(history: ErgoHistoryReader, offset: Int, limit: Int)(implicit segmentTreshold: Int): Array[IndexedErgoTransaction] = - getFromSegments(history, offset, limit, txSegmentCount, txs, txSegmentId, _.txs, getTxs) + getFromSegments(history, offset, limit, txSegmentCount, txs, txSegmentId, _.txs, getTxs, txsFlag = true) /** * Get a range of the boxes associated with the parent object @@ -235,7 +260,7 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId, * @return array of boxes */ def retrieveBoxes(history: ErgoHistoryReader, offset: Int, limit: Int)(implicit segmentTreshold: Int): Array[IndexedErgoBox] = - getFromSegments(history, offset, limit, boxSegmentCount, boxes, boxSegmentId, _.boxes, getBoxes) + getFromSegments(history, offset, limit, boxSegmentCount, boxes, boxSegmentId, _.boxes, getBoxes, txsFlag = false) /** diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 53e06b3738..88c4a1cd30 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -26,27 +26,27 @@ import scala.jdk.CollectionConverters.asScalaIteratorConverter * @param extraStore - key-value store, where key is id of Index and value is it's bytes * @param config - cache configs */ -class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, extraStore: LDBKVStore, config: CacheSettings) +class HistoryStorage(indexStore: LDBKVStore, objectsStore: LDBKVStore, extraStore: LDBKVStore, config: CacheSettings) extends ScorexLogging with AutoCloseable with ScorexEncoding { - private val headersCache = + private lazy val headersCache = Caffeine.newBuilder() .maximumSize(config.history.headersCacheSize) .build[String, BlockSection]() - private val blockSectionsCache = + private lazy val blockSectionsCache = Caffeine.newBuilder() .maximumSize(config.history.blockSectionsCacheSize) .build[String, BlockSection]() - private val extraCache = + private lazy val extraCache = Caffeine.newBuilder() .maximumSize(config.history.extraCacheSize) .build[String, ExtraIndex]() - private val indexCache = + private lazy val indexCache = Caffeine.newBuilder() .maximumSize(config.history.indexesCacheSize) .build[ByteArrayWrapper, Array[Byte]] diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala index b84402e288..04173ca03c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala @@ -677,4 +677,4 @@ class ErgoWalletServiceImpl(override val ergoSettings: ErgoSettings) extends Erg .map(ErgoTransaction.apply) } -} \ No newline at end of file +} diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/SegmentSpec.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/SegmentSpec.scala new file mode 100644 index 0000000000..4ab6dd4e82 --- /dev/null +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/SegmentSpec.scala @@ -0,0 +1,187 @@ +package org.ergoplatform.nodeView.history.extra + +import org.ergoplatform.utils.ErgoCorePropertyTest +import scorex.util.ModifierId +import scorex.util.encode.Base16 + +import scala.collection.mutable.ArrayBuffer +import org.ergoplatform.consensus.ProgressInfo +import org.ergoplatform.mining.AutolykosPowScheme +import org.ergoplatform.modifiers.{BlockSection, NonHeaderBlockSection} +import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.nodeView.history.storage.HistoryStorage +import org.ergoplatform.settings.ErgoSettings + +import scala.util.Try + +class SegmentSpec extends ErgoCorePropertyTest { + + property("correct slicing in getFromSegments") { + implicit val segmentTreshold: Int = 512 + + val hash = ModifierId @@ Base16.encode(Array.fill(32)(0.toByte)) + val boxes = new ArrayBuffer[Long]() + (1 to 1706).foreach{i => + boxes.append(i) + } + + val segment0 = IndexedErgoAddress(hash, new ArrayBuffer[Long](), boxes.take(segmentTreshold)) + val segment1 = IndexedErgoAddress(hash, new ArrayBuffer[Long](), boxes.slice(segmentTreshold, segmentTreshold * 2)) + val segment2 = IndexedErgoAddress(hash, new ArrayBuffer[Long](), boxes.slice(segmentTreshold * 2, segmentTreshold * 3)) + val ia = IndexedErgoAddress(hash, new ArrayBuffer[Long](), boxes.slice(segmentTreshold * 3, boxes.size).reverse) + + val hr = new ErgoHistoryReader { + override protected[history] val historyStorage: HistoryStorage = new HistoryStorage(null, null, null ,null) { + override def getExtraIndex(id: ModifierId): Option[ExtraIndex] = { + val b = Base16.decode(id).get.head + if(b == 0) { + Some(segment0) + } else if(b == 1) { + Some(segment1) + } else if(b == 2) { + Some(segment2) + } else { + None + } + } + } + + override protected val settings: ErgoSettings = null + + /** + * Whether state requires to download adProofs before full block application + */ + override protected def requireProofs: Boolean = ??? + + /** + * @param m - modifier to process + * @return ProgressInfo - info required for State to be consistent with History + */ + override protected def process(m: NonHeaderBlockSection): Try[ProgressInfo[BlockSection]] = ??? + + /** + * @param m - modifier to validate + * @return Success() if modifier is valid from History point of view, Failure(error) otherwise + */ + override protected def validate(m: NonHeaderBlockSection): Try[Unit] = ??? + + override val powScheme: AutolykosPowScheme = null + } + + def getFromSegments(offset: Int, limit: Int) = { + ia.getFromSegments( + history = hr, + offset, + limit, + segmentCount = 3, + ia.boxes, + idOf = { + case (_, i) => ModifierId @@ Base16.encode(Array.fill(32)(i.toByte)) + }, + arraySelector = { ai => ai.boxes }, + retrieve = { + case (arr, _) => arr.toArray + }, + txsFlag = false + ) + } + + val lim = 200 + + val a1 = getFromSegments(0, lim) + val a2 = getFromSegments(lim, lim) + val a3 = getFromSegments(2 * lim, lim) + val a4 = getFromSegments(3 * lim, lim) + val a5 = getFromSegments(4 * lim, lim) + val a6 = getFromSegments(5 * lim, lim) + val a7 = getFromSegments(6 * lim, lim) + val a8 = getFromSegments(7 * lim, lim) + val a9 = getFromSegments(8 * lim, lim) + val a10 = getFromSegments(9 * lim, lim) + + a1.distinct.length shouldBe lim + a2.distinct.length shouldBe lim + a3.distinct.length shouldBe lim + a4.distinct.length shouldBe lim + + (a2 ++ a4).distinct.length shouldBe lim*2 + + a5.distinct.length shouldBe lim + + (a3 ++ a5).distinct.length shouldBe lim*2 + + a6.distinct.length shouldBe lim + a7.distinct.length shouldBe lim + + (a1 ++ a2 ++ a3 ++ a4 ++ a5 ++ a6 ++ a7).distinct.length shouldBe 1400 + + a8.distinct.length shouldBe lim + a9.distinct.length shouldBe 106 + a10.distinct.length shouldBe 0 + + + (a1 ++ a2 ++ a3 ++ a4 ++ a5 ++ a6 ++ a7 ++ a8 ++ a9).distinct.length shouldBe 1706 + + val lim2 = 100 + + val b1 = getFromSegments(0, lim2) + val b2 = getFromSegments(lim2, lim2) + val b3 = getFromSegments(2 * lim2, lim2) + val b4 = getFromSegments(3 * lim2, lim2) + val b5 = getFromSegments(4 * lim2, lim2) + val b6 = getFromSegments(5 * lim2, lim2) + val b7 = getFromSegments(6 * lim2, lim2) + val b8 = getFromSegments(7 * lim2, lim2) + val b9 = getFromSegments(8 * lim2, lim2) + val b10 = getFromSegments(9 * lim2, lim2) + val b11 = getFromSegments(10 * lim2, lim2) + val b12 = getFromSegments(11 * lim2, lim2) + val b13 = getFromSegments(12 * lim2, lim2) + val b14 = getFromSegments(13 * lim2, lim2) + val b15 = getFromSegments(14 * lim2, lim2) + val b16 = getFromSegments(15 * lim2, lim2) + val b17 = getFromSegments(16 * lim2, lim2) + val b18 = getFromSegments(17 * lim2, lim2) + val b19 = getFromSegments(18 * lim2, lim2) + + b1.distinct.length shouldBe lim2 + b2.distinct.length shouldBe lim2 + b3.distinct.length shouldBe lim2 + b4.distinct.length shouldBe lim2 + b5.distinct.length shouldBe lim2 + b6.distinct.length shouldBe lim2 + b7.distinct.length shouldBe lim2 + b8.distinct.length shouldBe lim2 + b9.distinct.length shouldBe lim2 + b10.distinct.length shouldBe lim2 + b11.distinct.length shouldBe lim2 + b12.distinct.length shouldBe lim2 + b13.distinct.length shouldBe lim2 + b14.distinct.length shouldBe lim2 + b15.distinct.length shouldBe lim2 + b16.distinct.length shouldBe lim2 + b17.distinct.length shouldBe lim2 + b18.distinct.length shouldBe 6 + b19.distinct.length shouldBe 0 + + (b1 ++ b2).distinct.length shouldBe(2 * lim2) + (b2 ++ b3).distinct.length shouldBe(2 * lim2) + (b1 ++ b2 ++ b3).distinct.length shouldBe(3 * lim2) + + (b1 ++ b2 ++ b3 ++ b4 ++ b5 ++ b6 ++ b7 ++ b8 ++ b9 ++ + b10 ++ b11 ++ b12 ++ b13 ++ b14 ++ b15 ++ b16 ++ b17 ++ b18).distinct.length shouldBe 1706 + + val lim3 = 682 // array + segment threshold + val c1 = getFromSegments(0, lim3) + val c2 = getFromSegments(lim3, lim3) + val c3 = getFromSegments(2 * lim3, lim3) + val c4 = getFromSegments(3 * lim3, lim3) + + c1.distinct.length shouldBe lim3 + c2.distinct.length shouldBe lim3 + c3.distinct.length shouldBe 342 + c4.distinct.length shouldBe 0 + (c1 ++ c2 ++ c3).distinct.length shouldBe 1706 + } + +} From 73065a3a17d65501c8df59e7ca1c9f4f7434a962 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Thu, 1 Aug 2024 23:37:25 +0300 Subject: [PATCH 16/16] IndexedErgoTransaction format fix, double quotes in examples fix --- src/main/resources/api/openapi.yaml | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 3de95cc170..f3f3545e72 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -445,7 +445,7 @@ components: description: Transaction inputs type: array items: - $ref: '#/components/schemas/ErgoTransactionInput' + $ref: '#/components/schemas/IndexedErgoBox' dataInputs: description: Transaction data inputs type: array @@ -455,7 +455,7 @@ components: description: Transaction outputs type: array items: - $ref: '#/components/schemas/ErgoTransactionOutput' + $ref: '#/components/schemas/IndexedErgoBox' inclusionHeight: description: Height of a block the transaction was included in type: integer @@ -3096,7 +3096,7 @@ paths: application/json: schema: type: string - example: '"02c9e71790399816b3e40b2207e9ade19a9b7fe0600186cfb8e2b115bfdb34b57f38cd3c9f2890d11720eb3bb993993f00ededf812a590d2993df094a7ca4f0213e4820e1ab831eed5dc5c72665396d3a01d2a12900f1c3ab77700b284ae24fa8e8f7754f86f2282c795db6b0b17df1c29cc0552e59d01f7d777c638a813333277271c2f8b4d99d01ff0e6ee8695697bdd5b568089395620d7198c6093ce8bc59b928611b1b12452c05addaa42f4beff6a0a6fe90000000380d0dbc3f40210090402040005c801040205c8010500040004000e2003faf2cb329f2e90d6d23b58d91bbb6c046aa143261cc21f52fbe2824bfcbf04d807d601e4c6a70408d602b2a5730000d603e4c6a70601d604e4c6a7080ed605e4c6a70505d606e4c6a70705d60795720399c1a7c1720299c17202c1a7eb027201d1ededededededededed93c27202c2a793e4c672020408720193e4c6720205059572039d9c72057eb272047301000573029d9c72057eb2720473030005730494e4c672020601720393e4c672020705720693e4c67202080e720493e4c67202090ec5a79572039072079c720672059272079c72067205917207730595ef720393b1db630872027306d801d608b2db63087202730700ed938c7208017308938c7208027206c8df35000508cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba0405c8a8c105010105dc8b020e0266608cdea8baf0380008cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba04c8df350000c0843d1005040004000e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a701730073011001020402d19683030193a38cc7b2a57300000193c2b2a57301007473027303830108cdeeac93b1a57304c8df350000"' + example: '02c9e71790399816b3e40b2207e9ade19a9b7fe0600186cfb8e2b115bfdb34b57f38cd3c9f2890d11720eb3bb993993f00ededf812a590d2993df094a7ca4f0213e4820e1ab831eed5dc5c72665396d3a01d2a12900f1c3ab77700b284ae24fa8e8f7754f86f2282c795db6b0b17df1c29cc0552e59d01f7d777c638a813333277271c2f8b4d99d01ff0e6ee8695697bdd5b568089395620d7198c6093ce8bc59b928611b1b12452c05addaa42f4beff6a0a6fe90000000380d0dbc3f40210090402040005c801040205c8010500040004000e2003faf2cb329f2e90d6d23b58d91bbb6c046aa143261cc21f52fbe2824bfcbf04d807d601e4c6a70408d602b2a5730000d603e4c6a70601d604e4c6a7080ed605e4c6a70505d606e4c6a70705d60795720399c1a7c1720299c17202c1a7eb027201d1ededededededededed93c27202c2a793e4c672020408720193e4c6720205059572039d9c72057eb272047301000573029d9c72057eb2720473030005730494e4c672020601720393e4c672020705720693e4c67202080e720493e4c67202090ec5a79572039072079c720672059272079c72067205917207730595ef720393b1db630872027306d801d608b2db63087202730700ed938c7208017308938c7208027206c8df35000508cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba0405c8a8c105010105dc8b020e0266608cdea8baf0380008cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba04c8df350000c0843d1005040004000e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a701730073011001020402d19683030193a38cc7b2a57300000193c2b2a57301007473027303830108cdeeac93b1a57304c8df350000' responses: '200': description: JSON with ID of the new transaction @@ -3152,7 +3152,7 @@ paths: application/json: schema: type: string - example: '"02c9e71790399816b3e40b2207e9ade19a9b7fe0600186cfb8e2b115bfdb34b57f38cd3c9f2890d11720eb3bb993993f00ededf812a590d2993df094a7ca4f0213e4820e1ab831eed5dc5c72665396d3a01d2a12900f1c3ab77700b284ae24fa8e8f7754f86f2282c795db6b0b17df1c29cc0552e59d01f7d777c638a813333277271c2f8b4d99d01ff0e6ee8695697bdd5b568089395620d7198c6093ce8bc59b928611b1b12452c05addaa42f4beff6a0a6fe90000000380d0dbc3f40210090402040005c801040205c8010500040004000e2003faf2cb329f2e90d6d23b58d91bbb6c046aa143261cc21f52fbe2824bfcbf04d807d601e4c6a70408d602b2a5730000d603e4c6a70601d604e4c6a7080ed605e4c6a70505d606e4c6a70705d60795720399c1a7c1720299c17202c1a7eb027201d1ededededededededed93c27202c2a793e4c672020408720193e4c6720205059572039d9c72057eb272047301000573029d9c72057eb2720473030005730494e4c672020601720393e4c672020705720693e4c67202080e720493e4c67202090ec5a79572039072079c720672059272079c72067205917207730595ef720393b1db630872027306d801d608b2db63087202730700ed938c7208017308938c7208027206c8df35000508cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba0405c8a8c105010105dc8b020e0266608cdea8baf0380008cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba04c8df350000c0843d1005040004000e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a701730073011001020402d19683030193a38cc7b2a57300000193c2b2a57301007473027303830108cdeeac93b1a57304c8df350000"' + example: '02c9e71790399816b3e40b2207e9ade19a9b7fe0600186cfb8e2b115bfdb34b57f38cd3c9f2890d11720eb3bb993993f00ededf812a590d2993df094a7ca4f0213e4820e1ab831eed5dc5c72665396d3a01d2a12900f1c3ab77700b284ae24fa8e8f7754f86f2282c795db6b0b17df1c29cc0552e59d01f7d777c638a813333277271c2f8b4d99d01ff0e6ee8695697bdd5b568089395620d7198c6093ce8bc59b928611b1b12452c05addaa42f4beff6a0a6fe90000000380d0dbc3f40210090402040005c801040205c8010500040004000e2003faf2cb329f2e90d6d23b58d91bbb6c046aa143261cc21f52fbe2824bfcbf04d807d601e4c6a70408d602b2a5730000d603e4c6a70601d604e4c6a7080ed605e4c6a70505d606e4c6a70705d60795720399c1a7c1720299c17202c1a7eb027201d1ededededededededed93c27202c2a793e4c672020408720193e4c6720205059572039d9c72057eb272047301000573029d9c72057eb2720473030005730494e4c672020601720393e4c672020705720693e4c67202080e720493e4c67202090ec5a79572039072079c720672059272079c72067205917207730595ef720393b1db630872027306d801d608b2db63087202730700ed938c7208017308938c7208027206c8df35000508cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba0405c8a8c105010105dc8b020e0266608cdea8baf0380008cd030c8f9c4dc08f3c006fa85a47c9156dedbede000a8b764c6e374fd097e873ba04c8df350000c0843d1005040004000e36100204a00b08cd0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798ea02d192a39a8cc7a701730073011001020402d19683030193a38cc7b2a57300000193c2b2a57301007473027303830108cdeeac93b1a57304c8df350000' responses: '200': description: JSON with ID of the new transaction @@ -3283,7 +3283,7 @@ paths: application/json: schema: type: string - example: '"100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301"' + example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' responses: '200': description: Ergo transaction @@ -3383,7 +3383,7 @@ paths: application/json: schema: type: string - example: '"100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301"' + example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' responses: '200': description: Unconfirmed transaction output boxes that correspond to given ErgoTree hex @@ -3656,7 +3656,7 @@ paths: application/json: schema: type: string - example: '"127.0.0.1:5673"' + example: '127.0.0.1:5673' responses: '200': description: Attempt to connect to the peer @@ -3767,7 +3767,7 @@ paths: application/json: schema: type: string - example: '"7e1e79dd4936bdc7d09f4ba9212849136b589fba4bcf4263a0961a95b65d08cb16"' + example: '7e1e79dd4936bdc7d09f4ba9212849136b589fba4bcf4263a0961a95b65d08cb16' default: description: Error content: @@ -3816,7 +3816,7 @@ paths: schema: description: Encoded Ergo Address type: string - example: '"3WwbzW6u8hKWBcL1W7kNVMr25s2UHfSBnYtwSHvrRQt7DdPuoXrt"' + example: '3WwbzW6u8hKWBcL1W7kNVMr25s2UHfSBnYtwSHvrRQt7DdPuoXrt' responses: '200': description: Address validity with validation error @@ -3928,7 +3928,7 @@ paths: application/json: schema: type: string - example: '"100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301"' + example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' responses: '200': description: Ergo address @@ -3963,7 +3963,7 @@ paths: application/json: schema: type: string - example: '"83375fd213cfd7dfd984ce1901d62c302a1db53160b416674c8da1a393a6bbc316"' + example: '83375fd213cfd7dfd984ce1901d62c302a1db53160b416674c8da1a393a6bbc316' default: description: Error content: @@ -3983,7 +3983,7 @@ paths: application/json: schema: type: string - example: '"7yaASMijGEGTbttYHg1MrXnWB8EbzjJnFLSWvmNoHrXV"' + example: '7yaASMijGEGTbttYHg1MrXnWB8EbzjJnFLSWvmNoHrXV' responses: '200': description: Base16-encoded 32 byte hash @@ -3991,7 +3991,7 @@ paths: application/json: schema: type: string - example: '"6ed54addddaf10fe8fcda330bd443a57914fbce38a9fa27248b07e361cc76a41"' + example: '6ed54addddaf10fe8fcda330bd443a57914fbce38a9fa27248b07e361cc76a41' default: description: Error content: @@ -6028,10 +6028,10 @@ paths: required: true content: application/json: - description: adderess associated with transactions + description: address associated with transactions schema: type: string - example: '"3WwbzW6u8hKWBcL1W7kNVMr25s2UHfSBnYtwSHvrRQt7DdPuoXrt"' + example: '3WwbzW6u8hKWBcL1W7kNVMr25s2UHfSBnYtwSHvrRQt7DdPuoXrt' parameters: - in: query name: offset @@ -6064,7 +6064,7 @@ paths: $ref: '#/components/schemas/IndexedErgoTransaction' total: type: integer - description: Total count of retreived transactions + description: Total count of retrieved transactions '404': description: No transactions found for wanted address content: @@ -6321,7 +6321,7 @@ paths: description: adderess associated with boxes schema: type: string - example: '"3WwbzW6u8hKWBcL1W7kNVMr25s2UHfSBnYtwSHvrRQt7DdPuoXrt"' + example: '3WwbzW6u8hKWBcL1W7kNVMr25s2UHfSBnYtwSHvrRQt7DdPuoXrt' parameters: - in: query name: offset @@ -6381,7 +6381,7 @@ paths: description: adderess associated with unspent boxes schema: type: string - example: '"3WwbzW6u8hKWBcL1W7kNVMr25s2UHfSBnYtwSHvrRQt7DdPuoXrt"' + example: '3WwbzW6u8hKWBcL1W7kNVMr25s2UHfSBnYtwSHvrRQt7DdPuoXrt' parameters: - in: query name: offset @@ -6496,7 +6496,7 @@ paths: description: hex encoded ergotree schema: type: string - example: '"100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301"' + example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' parameters: - in: query name: offset @@ -6550,7 +6550,7 @@ paths: description: hex encoded ergotree schema: type: string - example: '"100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301"' + example: '100204a00b08cd021cf943317b0fdb50f60892a46b9132b9ced337c7de79248b104b293d9f1f078eea02d192a39a8cc7a70173007301' parameters: - in: query name: offset