From 874f17ef3e9cb7c7c024e8ca9be44a8a994a6ac4 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Fri, 9 Aug 2024 23:56:42 +0300 Subject: [PATCH] object ScorexEncoder, removing unused arg from reportInvalidModifier --- .../ergoplatform/consensus/ProgressInfo.scala | 9 +- .../scala/org/ergoplatform/core/core.scala | 8 +- .../org/ergoplatform/http/api/ApiCodecs.scala | 4 +- .../org/ergoplatform/settings/Algos.scala | 16 +- .../ergoplatform/utils/ScorexEncoder.scala | 14 +- .../ergoplatform/utils/ScorexEncoding.scala | 4 +- .../validation/ModifierValidator.scala | 7 +- .../http/api/ErgoUtilsApiRoute.scala | 11 +- .../network/ErgoNodeViewSynchronizer.scala | 10 +- .../nodeView/ErgoNodeViewHolder.scala | 7 +- .../nodeView/history/ErgoHistory.scala | 10 +- .../nodeView/history/ErgoHistoryReader.scala | 4 +- .../history/storage/HistoryStorage.scala | 8 +- .../BlockSectionProcessor.scala | 3 +- .../EmptyBlockSectionProcessor.scala | 2 +- .../FullBlockProcessor.scala | 2 +- .../FullBlockSectionProcessor.scala | 2 +- .../modifierprocessors/HeadersProcessor.scala | 3 +- .../nodeView/state/DigestState.scala | 9 +- .../nodeView/state/UtxoState.scala | 187 +++++++++--------- .../VerifyADHistorySpecification.scala | 17 +- .../VerifyNonADHistorySpecification.scala | 4 +- 22 files changed, 157 insertions(+), 184 deletions(-) diff --git a/ergo-core/src/main/scala/org/ergoplatform/consensus/ProgressInfo.scala b/ergo-core/src/main/scala/org/ergoplatform/consensus/ProgressInfo.scala index 96efe2ec47..15cdbe3459 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/consensus/ProgressInfo.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/consensus/ProgressInfo.scala @@ -16,8 +16,7 @@ import scorex.util.ModifierId case class ProgressInfo[PM <: BlockSection](branchPoint: Option[ModifierId], toRemove: Seq[PM], toApply: Seq[PM], - toDownload: Seq[(NetworkObjectTypeId.Value, ModifierId)]) - (implicit encoder: ScorexEncoder) { + toDownload: Seq[(NetworkObjectTypeId.Value, ModifierId)]) { if (toRemove.nonEmpty) require(branchPoint.isDefined, s"Branch point should be defined for non-empty `toRemove`") @@ -25,7 +24,11 @@ case class ProgressInfo[PM <: BlockSection](branchPoint: Option[ModifierId], lazy val chainSwitchingNeeded: Boolean = toRemove.nonEmpty override def toString: String = { - s"ProgressInfo(BranchPoint: ${branchPoint.map(encoder.encodeId)}, " + + s"ProgressInfo(BranchPoint: ${branchPoint.map(ScorexEncoder.encodeId)}, " + s" to remove: ${toRemove.map(_.encodedId)}, to apply: ${toApply.map(_.encodedId)})" } } + +object ProgressInfo { + val empty = ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) +} diff --git a/ergo-core/src/main/scala/org/ergoplatform/core/core.scala b/ergo-core/src/main/scala/org/ergoplatform/core/core.scala index 59d6d0ffdd..15fa1a1f8f 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/core/core.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/core/core.scala @@ -13,18 +13,18 @@ package object core { type VersionTag = VersionTag.Type - def idsToString(ids: Seq[(NetworkObjectTypeId.Value, util.ModifierId)])(implicit enc: ScorexEncoder): String = { + def idsToString(ids: Seq[(NetworkObjectTypeId.Value, util.ModifierId)]): String = { List(ids.headOption, ids.lastOption) .flatten - .map { case (typeId, id) => s"($typeId,${enc.encodeId(id)})" } + .map { case (typeId, id) => s"($typeId,${ScorexEncoder.encodeId(id)})" } .mkString("[", "..", "]") } - def idsToString(modifierType: NetworkObjectTypeId.Value, ids: Seq[util.ModifierId])(implicit encoder: ScorexEncoder): String = { + def idsToString(modifierType: NetworkObjectTypeId.Value, ids: Seq[util.ModifierId]): String = { idsToString(ids.map(id => (modifierType, id))) } - def idsToString(invData: InvData)(implicit encoder: ScorexEncoder): String = idsToString(invData.typeId, invData.ids) + def idsToString(invData: InvData): String = idsToString(invData.typeId, invData.ids) def bytesToId: Array[Byte] => util.ModifierId = scorex.util.bytesToId diff --git a/ergo-core/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/ergo-core/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index 1b6c9b2214..cdd5552635 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -1,7 +1,7 @@ package org.ergoplatform.http.api -import cats.syntax.either._ -import io.circe._ +import cats.syntax.either._ // needed for Scala 2.11 +import io.circe._ // needed for Scala 2.11 import io.circe.syntax._ import org.bouncycastle.util.BigIntegers import org.ergoplatform.ErgoBox.RegisterId diff --git a/ergo-core/src/main/scala/org/ergoplatform/settings/Algos.scala b/ergo-core/src/main/scala/org/ergoplatform/settings/Algos.scala index ac80a7001d..ed3843a873 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/settings/Algos.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/settings/Algos.scala @@ -1,24 +1,24 @@ package org.ergoplatform.settings import org.ergoplatform.utils -import org.ergoplatform.utils.ScorexEncoder import scorex.crypto.authds.LeafData import scorex.crypto.authds.merkle.MerkleTree import scorex.crypto.hash.Digest32 -import scorex.util._ +import scorex.util.encode.BytesEncoder object Algos extends ErgoAlgos with utils.ScorexEncoding { - // ErgoAlgos in sigmastate extends scorex.util.ScorexEncoding where encoder is BytesEncoder - // but here we use scorex.core.utils.ScorexEncoding where encoder is ScorexEncoder - // After ScorexEncoder is moved (there is even a todo for that) from scorex.core to scorex.util - // we can fix this ugliness. - override implicit val encoder: ScorexEncoder = utils.ScorexEncoder.default + override implicit val encoder: BytesEncoder = utils.ScorexEncoder lazy val emptyMerkleTreeRoot: Digest32 = Algos.hash(LeafData @@ Array[Byte]()) - @inline def encode(id: ModifierId): String = encoder.encode(id) + /** + * This method might be useful and reimplemented, if encoding of ModifierId and VersionTag + * is different form default bytes encoding, e.g. this method should be reimplemented together + * with encode() and decode methods + */ + @inline def encode(id: String): String = id /** * A method to build a Merkle tree over binary objects (leafs of the tree) diff --git a/ergo-core/src/main/scala/org/ergoplatform/utils/ScorexEncoder.scala b/ergo-core/src/main/scala/org/ergoplatform/utils/ScorexEncoder.scala index e437be18f6..3911490e10 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/utils/ScorexEncoder.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/utils/ScorexEncoder.scala @@ -6,7 +6,7 @@ import scorex.util.encode.{Base16, BytesEncoder} import scala.util.Try -class ScorexEncoder extends BytesEncoder { +object ScorexEncoder extends BytesEncoder { @inline override val Alphabet: String = Base16.Alphabet @@ -16,14 +16,6 @@ class ScorexEncoder extends BytesEncoder { @inline override def decode(input: String): Try[Array[Byte]] = Base16.decode(input) - /** - * This method might be useful and reimplemented, if encoding of ModifierId and VersionTag - * is different form default bytes encoding, e.g. this method should be reimplemented together - * with encode() and decode methods - */ - @inline - def encode(input: String): String = input - /** * This method might be useful and reimplemented, if encoding of ModifierId and VersionTag * is different form default bytes encoding, e.g. this method should be reimplemented together @@ -41,7 +33,3 @@ class ScorexEncoder extends BytesEncoder { def encodeId(input: ModifierId): String = input } - -object ScorexEncoder { - val default: ScorexEncoder = new ScorexEncoder() -} diff --git a/ergo-core/src/main/scala/org/ergoplatform/utils/ScorexEncoding.scala b/ergo-core/src/main/scala/org/ergoplatform/utils/ScorexEncoding.scala index 089d00b640..916c92dac5 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/utils/ScorexEncoding.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/utils/ScorexEncoding.scala @@ -1,9 +1,11 @@ package org.ergoplatform.utils +import scorex.util.encode.BytesEncoder + /** * Trait with bytes to string encoder * TODO extract to ScorexUtils project */ trait ScorexEncoding { - implicit val encoder: ScorexEncoder = ScorexEncoder.default + val encoder: BytesEncoder = ScorexEncoder } diff --git a/ergo-core/src/main/scala/org/ergoplatform/validation/ModifierValidator.scala b/ergo-core/src/main/scala/org/ergoplatform/validation/ModifierValidator.scala index dab7a2db33..3f6fb3518c 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/validation/ModifierValidator.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/validation/ModifierValidator.scala @@ -25,8 +25,8 @@ import scala.util.{Failure, Success, Try} */ object ModifierValidator { - def apply(settings: ValidationSettings)(implicit e: ScorexEncoder): ValidationState[Unit] = { - ValidationState(ModifierValidator.success, settings)(e) + def apply(settings: ValidationSettings): ValidationState[Unit] = { + ValidationState(ModifierValidator.success, settings) } /** report recoverable modifier error that could be fixed by later retries */ @@ -65,7 +65,7 @@ object ModifierValidator { } /** This is the place where all the validation DSL lives */ -case class ValidationState[T](result: ValidationResult[T], settings: ValidationSettings)(implicit e: ScorexEncoder) { +case class ValidationState[T](result: ValidationResult[T], settings: ValidationSettings) { /** Create the next validation state as the result of given `operation` */ def pass[R](operation: => ValidationResult[R]): ValidationState[R] = { @@ -115,6 +115,7 @@ case class ValidationState[T](result: ValidationResult[T], settings: ValidationS /** Validate the `id`s are equal. The `error` callback will be provided with detail on argument values */ def validateEqualIds(id: Short, `given`: => ModifierId, expected: => ModifierId, modifierTypeId: NetworkObjectTypeId.Value): ValidationState[T] = { + val e = ScorexEncoder pass { if (!settings.isActive(id) || given == expected) result else settings.getError(id, InvalidModifier(s"Given: ${e.encodeId(given)}, expected ${e.encodeId(expected)}", given, modifierTypeId)) diff --git a/src/main/scala/org/ergoplatform/http/api/ErgoUtilsApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/ErgoUtilsApiRoute.scala index 87775dba7a..e98e82c488 100644 --- a/src/main/scala/org/ergoplatform/http/api/ErgoUtilsApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/ErgoUtilsApiRoute.scala @@ -9,7 +9,7 @@ import org.ergoplatform.http.api.ApiError.BadRequest import org.ergoplatform.settings.{ErgoSettings, RESTApiSettings} import org.ergoplatform.{ErgoAddressEncoder, P2PKAddress} import scorex.core.api.http.{ApiResponse, ApiRoute} -import org.ergoplatform.utils.ScorexEncoding +import org.ergoplatform.utils.ScorexEncoder import scorex.crypto.hash.Blake2b256 import scorex.util.encode.Base16 import sigmastate.crypto.DLogProtocol.ProveDlog @@ -18,10 +18,7 @@ import java.security.SecureRandom import scala.util.Failure import sigmastate.serialization.{ErgoTreeSerializer, GroupElementSerializer, SigmaSerializer} -class ErgoUtilsApiRoute(val ergoSettings: ErgoSettings)( - implicit val context: ActorRefFactory -) extends ApiRoute - with ScorexEncoding { +class ErgoUtilsApiRoute(val ergoSettings: ErgoSettings)(implicit val context: ActorRefFactory) extends ApiRoute { private val SeedSize = 32 private val treeSerializer: ErgoTreeSerializer = new ErgoTreeSerializer @@ -46,7 +43,7 @@ class ErgoUtilsApiRoute(val ergoSettings: ErgoSettings)( private def seed(length: Int): String = { val seed = new Array[Byte](length) new SecureRandom().nextBytes(seed) //seed mutated here! - encoder.encode(seed) + ScorexEncoder.encode(seed) } def seedRoute: Route = (get & path("seed")) { @@ -60,7 +57,7 @@ class ErgoUtilsApiRoute(val ergoSettings: ErgoSettings)( def hashBlake2b: Route = { (post & path("hash" / "blake2b") & entity(as[Json])) { json => json.as[String] match { - case Right(message) => ApiResponse(encoder.encode(Blake2b256(message))) + case Right(message) => ApiResponse(ScorexEncoder.encode(Blake2b256(message))) case Left(ex) => ApiError(StatusCodes.BadRequest, ex.getMessage()) } } diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index e2a550282a..2feec87f97 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -22,7 +22,7 @@ import org.ergoplatform.network.message.{InvSpec, MessageSpec, ModifiersSpec, Re import scorex.core.network._ import scorex.core.network.{ConnectedPeer, ModifiersStatus, SendToPeer, SendToPeers} import org.ergoplatform.network.message.{InvData, Message, ModifiersData} -import org.ergoplatform.utils.ScorexEncoding +import org.ergoplatform.utils.ScorexEncoder import org.ergoplatform.validation.MalformedModifierError import scorex.util.{ModifierId, ScorexLogging} import scorex.core.network.DeliveryTracker @@ -53,7 +53,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, settings: ErgoSettings, syncTracker: ErgoSyncTracker, deliveryTracker: DeliveryTracker)(implicit ex: ExecutionContext) - extends Actor with Synchronizer with ScorexLogging with ScorexEncoding { + extends Actor with Synchronizer with ScorexLogging { import org.ergoplatform.network.ErgoNodeViewSynchronizer._ @@ -777,7 +777,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, case _ => // Penalize peer and do nothing - it will be switched to correct state on CheckDelivery penalizeMisbehavingPeer(remote) - log.warn(s"Failed to parse transaction with declared id ${encoder.encodeId(id)} from ${remote.toString}") + log.warn(s"Failed to parse transaction with declared id ${ScorexEncoder.encodeId(id)} from ${remote.toString}") } } } @@ -801,7 +801,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, // Forget about block section, so it will be redownloaded if announced again only deliveryTracker.setUnknown(id, modifierTypeId) penalizeMisbehavingPeer(remote) - log.warn(s"Failed to parse modifier with declared id ${encoder.encodeId(id)} from ${remote.toString}") + log.warn(s"Failed to parse modifier with declared id ${ScorexEncoder.encodeId(id)} from ${remote.toString}") None } } @@ -1230,7 +1230,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, } else { // A block section is not delivered on time. log.info(s"Peer ${peer.toString} has not delivered network object " + - s"$modifierTypeId : ${encoder.encodeId(modifierId)} on time") + s"$modifierTypeId : ${ScorexEncoder.encodeId(modifierId)} on time") // Number of delivery checks for a block section, utxo set snapshot chunk or manifest // increased or initialized, except the case where we can have issues with connectivity, diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index e68b9d4693..f955cf61f9 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -19,7 +19,6 @@ import org.ergoplatform.network.ErgoNodeViewSynchronizerMessages._ import org.ergoplatform.nodeView.ErgoNodeViewHolder.{BlockAppliedTransactions, CurrentView, DownloadRequest} import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages._ import org.ergoplatform.modifiers.history.{ADProofs, HistoryModifierSerializer} -import org.ergoplatform.utils.ScorexEncoding import org.ergoplatform.validation.RecoverableModifierError import scorex.util.{ModifierId, ScorexLogging} import spire.syntax.all.cfor @@ -40,7 +39,7 @@ import scala.util.{Failure, Success, Try} * */ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSettings) - extends Actor with ScorexLogging with ScorexEncoding with FileUtils { + extends Actor with ScorexLogging with FileUtils { private implicit lazy val actorSystem: ActorSystem = context.system @@ -240,7 +239,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti } case Failure(e) => log.warn(s"Invalid modifier! Typeid: ${modToApply.modifierTypeId} id: ${modToApply.id} ", e) - history.reportModifierIsInvalid(modToApply, progressInfo).map { case (newHis, newProgressInfo) => + history.reportModifierIsInvalid(modToApply).map { case (newHis, newProgressInfo) => context.system.eventStream.publish(SemanticallyFailedModification(modToApply.modifierTypeId, modToApply.id, e)) UpdateInformation(newHis, updateInfo.state, Some(modToApply), Some(newProgressInfo), updateInfo.suffix) } @@ -573,7 +572,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti log.info("State and history are both empty on startup") Success(stateIn) case (stateId, Some(block), _) if stateId == block.id => - log.info(s"State and history have the same version ${encoder.encode(stateId)}, no recovery needed.") + log.info(s"State and history have the same version ${Algos.encode(stateId)}, no recovery needed.") Success(stateIn) case (_, None, _) => log.info("State and history are inconsistent. History is empty on startup, rollback state to genesis.") diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala index c001dd8e64..38c8113bf7 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala @@ -119,9 +119,7 @@ trait ErgoHistory * @return ProgressInfo with next modifier to try to apply */ @SuppressWarnings(Array("OptionGet", "TraversableHead")) - def reportModifierIsInvalid(modifier: BlockSection, - progressInfo: ProgressInfo[BlockSection] - ): Try[(ErgoHistory, ProgressInfo[BlockSection])] = synchronized { + def reportModifierIsInvalid(modifier: BlockSection): Try[(ErgoHistory, ProgressInfo[BlockSection])] = synchronized { log.warn(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as invalid") correspondingHeader(modifier) match { case Some(invalidatedHeader) => @@ -136,7 +134,7 @@ trait ErgoHistory case (false, false) => // Modifiers from best header and best full chain are not involved, no rollback and links change required historyStorage.insert(validityRow, BlockSection.emptyArray).map { _ => - this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) + this -> ProgressInfo.empty } case _ => // Modifiers from best header and best full chain are involved, links change required @@ -148,7 +146,7 @@ trait ErgoHistory newBestHeaderOpt.map(h => BestHeaderKey -> idToBytes(h.id)).toArray, BlockSection.emptyArray ).map { _ => - this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) + this -> ProgressInfo.empty } } else { val invalidatedChain: Seq[ErgoFullBlock] = bestFullBlockOpt.toSeq @@ -184,7 +182,7 @@ trait ErgoHistory //No headers become invalid. Just mark this modifier as invalid log.warn(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is missing corresponding header") historyStorage.insert(Array(validityKey(modifier.id) -> Array(0.toByte)), BlockSection.emptyArray).map { _ => - this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) + this -> ProgressInfo.empty } } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala index 3fde451134..044a967590 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala @@ -11,7 +11,6 @@ import org.ergoplatform.nodeView.history.extra.ExtraIndex import org.ergoplatform.nodeView.history.storage._ import org.ergoplatform.nodeView.history.storage.modifierprocessors.{BlockSectionProcessor, HeadersProcessor} import org.ergoplatform.settings.{ErgoSettings, NipopowSettings} -import org.ergoplatform.utils.ScorexEncoding import org.ergoplatform.validation.MalformedModifierError import scorex.util.{ModifierId, ScorexLogging} @@ -27,8 +26,7 @@ trait ErgoHistoryReader with ContainsModifiers[BlockSection] with HeadersProcessor with BlockSectionProcessor - with ScorexLogging - with ScorexEncoding { + with ScorexLogging { type ModifierIds = Seq[(NetworkObjectTypeId.Value, ModifierId)] diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 88c4a1cd30..1a885eb1fe 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -6,7 +6,6 @@ import org.ergoplatform.modifiers.history.HistoryModifierSerializer import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.nodeView.history.extra.{ExtraIndex, ExtraIndexSerializer, Segment} import org.ergoplatform.settings.{Algos, CacheSettings, ErgoSettings} -import org.ergoplatform.utils.ScorexEncoding import scorex.db.{ByteArrayWrapper, LDBFactory, LDBKVStore} import scorex.util.{ModifierId, ScorexLogging, idToBytes} @@ -28,8 +27,7 @@ import scala.jdk.CollectionConverters.asScalaIteratorConverter */ class HistoryStorage(indexStore: LDBKVStore, objectsStore: LDBKVStore, extraStore: LDBKVStore, config: CacheSettings) extends ScorexLogging - with AutoCloseable - with ScorexEncoding { + with AutoCloseable { private lazy val headersCache = Caffeine.newBuilder() @@ -84,7 +82,7 @@ class HistoryStorage(indexStore: LDBKVStore, objectsStore: LDBKVStore, extraStor cacheModifier(pm) Some(pm) case Failure(_) => - log.warn(s"Failed to parse modifier ${encoder.encode(id)} from db (bytes are: ${Algos.encode(bytes)})") + log.warn(s"Failed to parse modifier ${Algos.encode(id)} from db (bytes are: ${Algos.encode(bytes)})") None } } @@ -99,7 +97,7 @@ class HistoryStorage(indexStore: LDBKVStore, objectsStore: LDBKVStore, extraStor } Some(pm) case Failure(_) => - log.warn(s"Failed to parse index ${encoder.encode(id)} from db (bytes are: ${Algos.encode(bytes)})") + log.warn(s"Failed to parse index ${Algos.encode(id)} from db (bytes are: ${Algos.encode(bytes)})") None } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/BlockSectionProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/BlockSectionProcessor.scala index 653e592452..4f63bbdf8c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/BlockSectionProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/BlockSectionProcessor.scala @@ -2,7 +2,6 @@ package org.ergoplatform.nodeView.history.storage.modifierprocessors import org.ergoplatform.consensus.ProgressInfo import org.ergoplatform.modifiers.{BlockSection, NonHeaderBlockSection} -import org.ergoplatform.utils.ScorexEncoding import scala.util.Try @@ -10,7 +9,7 @@ import scala.util.Try * Trait that declares interfaces for validation and processing of various * block sections: BlockTransactions, ADProofs, etc. */ -trait BlockSectionProcessor extends ScorexEncoding { +trait BlockSectionProcessor { /** * Whether state requires to download adProofs before full block application diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/EmptyBlockSectionProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/EmptyBlockSectionProcessor.scala index f7d35e3ea8..6cc442d7aa 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/EmptyBlockSectionProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/EmptyBlockSectionProcessor.scala @@ -12,7 +12,7 @@ import scala.util.{Failure, Success, Try} trait EmptyBlockSectionProcessor extends BlockSectionProcessor { override protected def process(m: NonHeaderBlockSection): Try[ProgressInfo[BlockSection]] = - Success(ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty)) + Success(ProgressInfo.empty) override protected def validate(m: NonHeaderBlockSection): Try[Unit] = Failure(new Error("Regime that does not support block sections processing")) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala index 8c84852f09..cb97f9412e 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala @@ -136,7 +136,7 @@ trait FullBlockProcessor extends HeadersProcessor { //Orphaned block or full chain is not initialized yet logStatus(Seq(), Seq(), params.fullBlock, None) historyStorage.insert(Array.empty[(ByteArrayWrapper, Array[Byte])], Array(params.newModRow)).map { _ => - ProgressInfo(None, Seq.empty, Seq.empty, Seq.empty) + ProgressInfo.empty } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala index b9c36f987d..0f48baf3ab 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala @@ -84,7 +84,7 @@ trait FullBlockSectionProcessor extends BlockSectionProcessor with FullBlockProc private def justPutToHistory(m: NonHeaderBlockSection): Try[ProgressInfo[BlockSection]] = { historyStorage.insert(Array.empty[(ByteArrayWrapper, Array[Byte])], Array[BlockSection](m)).map { _ => - ProgressInfo(None, Seq.empty, Seq.empty, Seq.empty) + ProgressInfo.empty } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala index 0a801ccd5e..2273d7d478 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala @@ -14,7 +14,6 @@ import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.settings.Constants.HashLength import org.ergoplatform.settings.ValidationRules._ import org.ergoplatform.settings._ -import org.ergoplatform.utils.ScorexEncoding import org.ergoplatform.validation.{InvalidModifier, ModifierValidator, ValidationResult, ValidationState} import scorex.db.ByteArrayWrapper import scorex.util._ @@ -27,7 +26,7 @@ import scala.util.{Failure, Success, Try} /** * Contains all functions required by History to process Headers. */ -trait HeadersProcessor extends ToDownloadProcessor with PopowProcessor with ScorexLogging with ScorexEncoding { +trait HeadersProcessor extends ToDownloadProcessor with PopowProcessor with ScorexLogging { /** * Key for database record storing ID of best block header diff --git a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala index 3c5a3a4721..18d6bed53d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala @@ -29,8 +29,7 @@ class DigestState protected(override val version: VersionTag, override val store: LDBVersionedStore, override val ergoSettings: ErgoSettings) extends ErgoState[DigestState] - with ScorexLogging - with ScorexEncoding { + with ScorexLogging { store.lastVersionID .foreach(id => require(version == bytesToVersion(id), "version should always be equal to store.lastVersionID")) @@ -87,12 +86,12 @@ class DigestState protected(override val version: VersionTag, @SuppressWarnings(Array("OptionGet")) override def rollbackTo(version: VersionTag): Try[DigestState] = { - log.info(s"Rollback Digest State to version ${Algos.encoder.encode(version)}") + log.info(s"Rollback Digest State to version ${Algos.encode(version)}") val versionBytes = org.ergoplatform.core.versionToBytes(version) Try(store.rollbackTo(versionBytes)).map { _ => store.clean(nodeSettings.keepVersions) val rootHash = ADDigest @@ store.get(versionBytes).get - log.info(s"Rollback to version ${Algos.encoder.encode(version)} with roothash ${Algos.encoder.encode(rootHash)}") + log.info(s"Rollback to version ${Algos.encode(version)} with roothash ${Algos.encoder.encode(rootHash)}") new DigestState(version, rootHash, store, ergoSettings) } } @@ -200,7 +199,7 @@ object DigestState extends ScorexLogging with ScorexEncoding { case Success(state) => state case Failure(e) => store.close() - log.warn(s"Failed to create state with ${versionOpt.map(encoder.encode)} and ${rootHashOpt.map(encoder.encode)}", e) + log.warn(s"Failed to create state with ${versionOpt.map(Algos.encode)} and ${rootHashOpt.map(encoder.encode)}", e) ErgoState.generateGenesisDigestState(dir, settings) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala index 01b98a59ef..ef27e16fb0 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala @@ -12,7 +12,6 @@ import org.ergoplatform.settings.Algos.HF import org.ergoplatform.settings.ValidationRules.{fbDigestIncorrect, fbOperationFailed} import org.ergoplatform.settings.{Algos, ErgoSettings, Parameters} import org.ergoplatform.utils.LoggingUtil -import org.ergoplatform.utils.ScorexEncoding import org.ergoplatform.core._ import org.ergoplatform.nodeView.LocallyGeneratedModifier import org.ergoplatform.validation.ModifierValidator @@ -38,8 +37,7 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 override val store: LDBVersionedStore, override protected val ergoSettings: ErgoSettings) extends ErgoState[UtxoState] - with UtxoStateReader - with ScorexEncoding { + with UtxoStateReader { import UtxoState.metadata @@ -49,7 +47,7 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 override def rollbackTo(version: VersionTag): Try[UtxoState] = persistentProver.synchronized { val p = persistentProver - log.info(s"Rollback UtxoState to version ${Algos.encoder.encode(version)}") + log.info(s"Rollback UtxoState to version ${Algos.encode(version)}") store.get(versionToBytes(version)) match { case Some(hash) => val rootHash: ADDigest = ADDigest @@ hash @@ -58,7 +56,7 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 } rollbackResult case None => - Failure(new Error(s"Unable to get root hash at version ${Algos.encoder.encode(version)}")) + Failure(new Error(s"Unable to get root hash at version ${Algos.encode(version)}")) } } @@ -109,111 +107,114 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 } } - override def applyModifier(mod: BlockSection, estimatedTip: Option[Height]) - (generate: LocallyGeneratedModifier => Unit): Try[UtxoState] = mod match { - case fb: ErgoFullBlock => - - val keepVersions = ergoSettings.nodeSettings.keepVersions + private def applyFullBlock(fb: ErgoFullBlock, estimatedTip: Option[Height]) + (generate: LocallyGeneratedModifier => Unit): Try[UtxoState] = { + val keepVersions = ergoSettings.nodeSettings.keepVersions - // avoid storing versioned information in the database when block being processed is behind - // blockchain tip by `keepVersions` blocks at least - // we store `keepVersions` diffs in the database if chain tip is not known yet - if (fb.height >= estimatedTip.getOrElse(0) - keepVersions) { - if (store.getKeepVersions < keepVersions) { - store.setKeepVersions(keepVersions) - } - } else { - if (store.getKeepVersions > 0) { - store.setKeepVersions(0) - } + // avoid storing versioned information in the database when block being processed is behind + // blockchain tip by `keepVersions` blocks at least + // we store `keepVersions` diffs in the database if chain tip is not known yet + if (fb.height >= estimatedTip.getOrElse(0) - keepVersions) { + if (store.getKeepVersions < keepVersions) { + store.setKeepVersions(keepVersions) } + } else { + if (store.getKeepVersions > 0) { + store.setKeepVersions(0) + } + } - persistentProver.synchronized { - val height = fb.header.height + persistentProver.synchronized { + val height = fb.header.height - log.debug(s"Trying to apply full block with header ${fb.header.encodedId} at height $height") + log.debug(s"Trying to apply full block with header ${fb.header.encodedId} at height $height") - val inRoot = rootDigest + val inRoot = rootDigest - val stateTry = stateContext.appendFullBlock(fb).flatMap { newStateContext => - val txsTry = applyTransactions(fb.blockTransactions.txs, fb.header.id, fb.header.stateRoot, newStateContext) + val stateTry = stateContext.appendFullBlock(fb).flatMap { newStateContext => + val txsTry = applyTransactions(fb.blockTransactions.txs, fb.header.id, fb.header.stateRoot, newStateContext) - txsTry.map { _: Unit => - val emissionBox = extractEmissionBox(fb) - val meta = metadata(idToVersion(fb.id), fb.header.stateRoot, emissionBox, newStateContext) + txsTry.map { _: Unit => + val emissionBox = extractEmissionBox(fb) + val meta = metadata(idToVersion(fb.id), fb.header.stateRoot, emissionBox, newStateContext) - var proofBytes = persistentProver.generateProofAndUpdateStorage(meta) + var proofBytes = persistentProver.generateProofAndUpdateStorage(meta) - if (!store.get(org.ergoplatform.core.idToBytes(fb.id)) - .exists(w => java.util.Arrays.equals(w, fb.header.stateRoot))) { - throw new Exception("Storage kept roothash is not equal to the declared one") - } + if (!store.get(org.ergoplatform.core.idToBytes(fb.id)) + .exists(w => java.util.Arrays.equals(w, fb.header.stateRoot))) { + throw new Exception("Storage kept roothash is not equal to the declared one") + } - if (!java.util.Arrays.equals(fb.header.stateRoot, persistentProver.digest)) { - throw new Exception("Calculated stateRoot is not equal to the declared one") - } + if (!java.util.Arrays.equals(fb.header.stateRoot, persistentProver.digest)) { + throw new Exception("Calculated stateRoot is not equal to the declared one") + } - var proofHash = ADProofs.proofDigest(proofBytes) - - if (!java.util.Arrays.equals(fb.header.ADProofsRoot, proofHash)) { - - log.error("Calculated proofHash is not equal to the declared one, doing another attempt") - - /** - * Proof generated was different from one announced. - * - * In most cases, announced proof is okay, and as proof is already checked, problem in some - * extra bytes added to the proof. - * - * Could be related to https://github.com/ergoplatform/ergo/issues/1614 - * - * So the problem could appear on mining nodes only, and caused by - * proofsForTransactions() wasting the tree unexpectedly. - * - * We are trying to generate proof again now. - */ - - persistentProver.rollback(inRoot) - .ensuring(java.util.Arrays.equals(persistentProver.digest, inRoot)) - - ErgoState.stateChanges(fb.blockTransactions.txs) match { - case Success(stateChanges) => - val mods = stateChanges.operations - mods.foreach( modOp => persistentProver.performOneOperation(modOp)) - - // meta is the same as it is block-specific - proofBytes = persistentProver.generateProofAndUpdateStorage(meta) - proofHash = ADProofs.proofDigest(proofBytes) - - if(!java.util.Arrays.equals(fb.header.ADProofsRoot, proofHash)) { - throw new Exception("Regenerated proofHash is not equal to the declared one") - } - case Failure(e) => - throw new Exception("Can't generate state changes on proof regeneration ", e) - } + var proofHash = ADProofs.proofDigest(proofBytes) + + if (!java.util.Arrays.equals(fb.header.ADProofsRoot, proofHash)) { + + log.error("Calculated proofHash is not equal to the declared one, doing another attempt") + + /** + * Proof generated was different from one announced. + * + * In most cases, announced proof is okay, and as proof is already checked, problem in some + * extra bytes added to the proof. + * + * Could be related to https://github.com/ergoplatform/ergo/issues/1614 + * + * So the problem could appear on mining nodes only, and caused by + * proofsForTransactions() wasting the tree unexpectedly. + * + * We are trying to generate proof again now. + */ + + persistentProver.rollback(inRoot) + .ensuring(java.util.Arrays.equals(persistentProver.digest, inRoot)) + + ErgoState.stateChanges(fb.blockTransactions.txs) match { + case Success(stateChanges) => + val mods = stateChanges.operations + mods.foreach( modOp => persistentProver.performOneOperation(modOp)) + + // meta is the same as it is block-specific + proofBytes = persistentProver.generateProofAndUpdateStorage(meta) + proofHash = ADProofs.proofDigest(proofBytes) + + if(!java.util.Arrays.equals(fb.header.ADProofsRoot, proofHash)) { + throw new Exception("Regenerated proofHash is not equal to the declared one") + } + case Failure(e) => + throw new Exception("Can't generate state changes on proof regeneration ", e) } + } - if (fb.adProofs.isEmpty) { - if (fb.height >= estimatedTip.getOrElse(Int.MaxValue) - ergoSettings.nodeSettings.adProofsSuffixLength) { - val adProofs = ADProofs(fb.header.id, proofBytes) - generate(LocallyGeneratedModifier(adProofs)) - } + if (fb.adProofs.isEmpty) { + if (fb.height >= estimatedTip.getOrElse(Int.MaxValue) - ergoSettings.nodeSettings.adProofsSuffixLength) { + val adProofs = ADProofs(fb.header.id, proofBytes) + generate(LocallyGeneratedModifier(adProofs)) } - - log.info(s"Valid modifier with header ${fb.header.encodedId} and emission box " + - s"${emissionBox.map(e => Algos.encode(e.id))} applied to UtxoState at height ${fb.header.height}") - saveSnapshotIfNeeded(fb.height, estimatedTip) - new UtxoState(persistentProver, idToVersion(fb.id), store, ergoSettings) } + + log.info(s"Valid modifier with header ${fb.header.encodedId} and emission box " + + s"${emissionBox.map(e => Algos.encode(e.id))} applied to UtxoState at height ${fb.header.height}") + saveSnapshotIfNeeded(fb.height, estimatedTip) + new UtxoState(persistentProver, idToVersion(fb.id), store, ergoSettings) } - stateTry.recoverWith[UtxoState] { case e => - log.warn(s"Error while applying full block with header ${fb.header.encodedId} to UTXOState with root" + - s" ${Algos.encode(inRoot)}, reason: ${LoggingUtil.getReasonMsg(e)} ", e) - persistentProver.rollback(inRoot) - .ensuring(java.util.Arrays.equals(persistentProver.digest, inRoot)) - Failure(e) - } } + stateTry.recoverWith[UtxoState] { case e => + log.warn(s"Error while applying full block with header ${fb.header.encodedId} to UTXOState with root" + + s" ${Algos.encode(inRoot)}, reason: ${LoggingUtil.getReasonMsg(e)} ", e) + persistentProver.rollback(inRoot) + .ensuring(java.util.Arrays.equals(persistentProver.digest, inRoot)) + Failure(e) + } + } + } + + override def applyModifier(mod: BlockSection, estimatedTip: Option[Height]) + (generate: LocallyGeneratedModifier => Unit): Try[UtxoState] = mod match { + case fb: ErgoFullBlock => applyFullBlock(fb, estimatedTip)(generate) case bs: BlockSection => log.warn(s"Only full-blocks are expected, found $bs") diff --git a/src/test/scala/org/ergoplatform/nodeView/history/VerifyADHistorySpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/VerifyADHistorySpecification.scala index e1551f01d1..da551cf939 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/VerifyADHistorySpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/VerifyADHistorySpecification.scala @@ -1,6 +1,5 @@ package org.ergoplatform.nodeView.history -import org.ergoplatform.consensus.ProgressInfo import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.HeaderChain import org.ergoplatform.modifiers.history.header.Header @@ -268,9 +267,7 @@ class VerifyADHistorySpecification extends ErgoCorePropertyTest with NoShrink { history.isSemanticallyValid(fullBlock.blockTransactions.id) shouldBe Unknown - val progressInfo = ProgressInfo[PM](Option(fullBlock.header.parentId), Seq(fullBlock), Seq.empty, Seq.empty) - history.reportModifierIsInvalid(fullBlock.header, progressInfo) - + history.reportModifierIsInvalid(fullBlock.header) history.isSemanticallyValid(fullBlock.header.id) shouldBe Invalid history.isSemanticallyValid(fullBlock.adProofs.value.id) shouldBe Invalid history.isSemanticallyValid(fullBlock.blockTransactions.id) shouldBe Invalid @@ -287,8 +284,7 @@ class VerifyADHistorySpecification extends ErgoCorePropertyTest with NoShrink { history = applyChain(history, fork1) history = applyChain(history, fork2) - val progressInfo = ProgressInfo[PM](Some(inChain.last.parentId), fork2, Seq.empty, Seq.empty) - history.reportModifierIsInvalid(inChain.last.header, progressInfo) + history.reportModifierIsInvalid(inChain.last.header) fork1.foreach { fullBlock => history.isSemanticallyValid(fullBlock.header.id) shouldBe Invalid @@ -315,8 +311,7 @@ class VerifyADHistorySpecification extends ErgoCorePropertyTest with NoShrink { history.bestHeaderOpt.value shouldBe fork1.last.header - val progressInfo = ProgressInfo[PM](Some(common.parentId), fork1, Seq.empty, Seq.empty) - history.reportModifierIsInvalid(fork1.head.header, progressInfo) + history.reportModifierIsInvalid(fork1.head.header) history.bestHeaderOpt.value shouldBe fork2.last.header history.bestFullBlockOpt.value shouldBe fork2.last @@ -330,8 +325,7 @@ class VerifyADHistorySpecification extends ErgoCorePropertyTest with NoShrink { val invalidChain = chain.takeRight(2) - val progressInfo = ProgressInfo[PM](Some(invalidChain.head.parentId), invalidChain, Seq.empty, Seq.empty) - val report = history.reportModifierIsInvalid(invalidChain.head.header, progressInfo).get + val report = history.reportModifierIsInvalid(invalidChain.head.header).get history = report._1 val processInfo = report._2 processInfo.toApply.isEmpty shouldBe true @@ -353,8 +347,7 @@ class VerifyADHistorySpecification extends ErgoCorePropertyTest with NoShrink { history.contains(parentHeader.transactionsId) shouldBe true history.contains(parentHeader.ADProofsId) shouldBe true - val progressInfo = ProgressInfo[PM](Some(parentHeader.id), Seq(fullBlock), Seq.empty, Seq.empty) - val (repHistory, _) = history.reportModifierIsInvalid(fullBlock.blockTransactions, progressInfo).get + val (repHistory, _) = history.reportModifierIsInvalid(fullBlock.blockTransactions).get repHistory.bestFullBlockOpt.value.header shouldBe history.bestHeaderOpt.value repHistory.bestHeaderOpt.value shouldBe parentHeader } diff --git a/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala index 9cff6acd5e..b6e116b285 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala @@ -1,6 +1,5 @@ package org.ergoplatform.nodeView.history -import org.ergoplatform.consensus.ProgressInfo import org.ergoplatform.modifiers.{ErgoFullBlock, NetworkObjectTypeId} import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.Extension @@ -78,8 +77,7 @@ class VerifyNonADHistorySpecification extends ErgoCorePropertyTest { val invalidChainHead = altChain.head // invalidate modifier from fork - history.reportModifierIsInvalid(invalidChainHead.blockTransactions, - ProgressInfo(None, Seq.empty, Seq.empty, Seq.empty)) + history.reportModifierIsInvalid(invalidChainHead.blockTransactions) history.bestFullBlockIdOpt.get shouldEqual initChain.last.id