diff --git a/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/AVLTreeBatchPerformance.scala b/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/AVLTreeBatchPerformance.scala index 9742eaa13b..39c7fa2434 100644 --- a/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/AVLTreeBatchPerformance.scala +++ b/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/AVLTreeBatchPerformance.scala @@ -4,9 +4,9 @@ import java.util.concurrent.TimeUnit import org.openjdk.jmh.annotations._ import org.slf4j.LoggerFactory -import scorex.crypto.authds.avltree.batch.{Operation, PersistentBatchAVLProver, VersionedLDBAVLStorage} +import scorex.crypto.authds.avltree.batch.{Operation, PersistentBatchAVLProver, VersionedRocksDBAVLStorage} import scorex.crypto.hash.{Blake2b256, Digest32} -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore object AVLTreeBatchPerformance extends { @@ -20,8 +20,8 @@ object AVLTreeBatchPerformance extends { val logger = LoggerFactory.getLogger("TEST") var prover: Prover = _ - var store: LDBVersionedStore = _ - var storage: VersionedLDBAVLStorage = _ + var store: RocksDBVersionedStore = _ + var storage: VersionedRocksDBAVLStorage = _ var operations: Array[Operation] = _ @Setup(Level.Iteration) diff --git a/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala b/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala index ab87dc73a9..577dd2b890 100644 --- a/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala +++ b/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala @@ -4,7 +4,7 @@ import com.google.common.primitives.Longs import scorex.crypto.authds.avltree.batch._ import scorex.crypto.authds.{ADKey, ADValue} import scorex.crypto.hash.{Blake2b256, Digest32} -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore import scorex.utils.Random object Helper { @@ -34,11 +34,11 @@ object Helper { } def persistentProverWithVersionedStore(initialKeepVersions: Int, - baseOperationsCount: Int = 0): (Prover, LDBVersionedStore, VersionedLDBAVLStorage) = { + baseOperationsCount: Int = 0): (Prover, RocksDBVersionedStore, VersionedRocksDBAVLStorage) = { val dir = java.nio.file.Files.createTempDirectory("bench_testing_" + scala.util.Random.alphanumeric.take(15)).toFile dir.deleteOnExit() - val store = new LDBVersionedStore(dir, initialKeepVersions = initialKeepVersions) - val storage = new VersionedLDBAVLStorage(store) + val store = new RocksDBVersionedStore(dir, initialKeepVersions = initialKeepVersions) + val storage = new VersionedRocksDBAVLStorage(store) require(storage.isEmpty) val prover = new BatchAVLProver[Digest32, HF](kl, Some(vl)) diff --git a/avldb/build.sbt b/avldb/build.sbt index c37c3f36c4..bb48e0bba6 100644 --- a/avldb/build.sbt +++ b/avldb/build.sbt @@ -26,7 +26,7 @@ libraryDependencies ++= Seq( "org.scalacheck" %% "scalacheck" % "1.14.3" % "test", "org.scalatestplus" %% "scalatestplus-scalacheck" % "3.1.0.0-RC2" % Test, "com.storm-enroute" %% "scalameter" % Versions.scalameter(scalaVersion.value) % "test", - "org.ethereum" % "leveldbjni-all" % "1.18.3", + "org.rocksdb" % "rocksdbjni" % "9.7.3", "org.typelevel" %% "spire" % Versions.spire(scalaVersion.value) ) diff --git a/avldb/src/main/scala/org/ergoplatform/serialization/ManifestSerializer.scala b/avldb/src/main/scala/org/ergoplatform/serialization/ManifestSerializer.scala index 985eb421fe..4e3246b2e0 100644 --- a/avldb/src/main/scala/org/ergoplatform/serialization/ManifestSerializer.scala +++ b/avldb/src/main/scala/org/ergoplatform/serialization/ManifestSerializer.scala @@ -2,14 +2,14 @@ package org.ergoplatform.serialization import scorex.crypto.authds.avltree.batch.Constants.DigestType import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, ProxyInternalNode} -import scorex.crypto.authds.avltree.batch.{InternalProverNode, ProverLeaf, ProverNodes, VersionedLDBAVLStorage} +import scorex.crypto.authds.avltree.batch.{InternalProverNode, ProverLeaf, ProverNodes, VersionedRocksDBAVLStorage} import scorex.util.serialization.{Reader, Writer} /** * Serializer of manifest, a tree which is cut at some `manifestDepth` from root */ class ManifestSerializer(manifestDepth: Byte) extends ErgoSerializer[BatchAVLProverManifest[DigestType]] { - private val nodeSerializer = VersionedLDBAVLStorage.noStoreSerializer + private val nodeSerializer = VersionedRocksDBAVLStorage.noStoreSerializer /** * Serialize manifest provided as top subtree and height separately. Used in tests. diff --git a/avldb/src/main/scala/org/ergoplatform/serialization/SubtreeSerializer.scala b/avldb/src/main/scala/org/ergoplatform/serialization/SubtreeSerializer.scala index f4822bbe3e..558a7b5eb6 100644 --- a/avldb/src/main/scala/org/ergoplatform/serialization/SubtreeSerializer.scala +++ b/avldb/src/main/scala/org/ergoplatform/serialization/SubtreeSerializer.scala @@ -1,7 +1,7 @@ package org.ergoplatform.serialization import scorex.crypto.authds.avltree.batch.Constants.DigestType -import scorex.crypto.authds.avltree.batch.{InternalProverNode, ProverLeaf, ProverNodes, VersionedLDBAVLStorage} +import scorex.crypto.authds.avltree.batch.{InternalProverNode, ProverLeaf, ProverNodes, VersionedRocksDBAVLStorage} import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverSubtree, ProxyInternalNode} import scorex.util.ScorexLogging import scorex.util.serialization.{Reader, Writer} @@ -10,7 +10,7 @@ import scorex.util.serialization.{Reader, Writer} * Serializer for subtree */ object SubtreeSerializer extends ErgoSerializer[BatchAVLProverSubtree[DigestType]] with ScorexLogging { - private val nodeSerializer = VersionedLDBAVLStorage.noStoreSerializer + private val nodeSerializer = VersionedRocksDBAVLStorage.noStoreSerializer override def serialize(subtree: BatchAVLProverSubtree[DigestType], w: Writer): Unit = { def loop(node: ProverNodes[DigestType]): Unit = { diff --git a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProverNodeSerializer.scala b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProverNodeSerializer.scala index 9199acdd8f..9c5a12fe2a 100644 --- a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProverNodeSerializer.scala +++ b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProverNodeSerializer.scala @@ -6,7 +6,7 @@ import scorex.crypto.authds.{ADKey, ADValue, Balance} import scorex.crypto.authds.avltree.batch.Constants.{DigestType, hashFn} import scorex.crypto.authds.avltree.batch.serialization.ProxyInternalNode import scorex.crypto.hash.Digest32 -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore import scorex.util.serialization.{Reader, Writer} /** @@ -15,9 +15,9 @@ import scorex.util.serialization.{Reader, Writer} * * @param store - store which could be provided to fetch children of a node when a child is requested */ -class ProverNodeSerializer(store: LDBVersionedStore) extends ErgoSerializer[ProverNodes[DigestType]] { +class ProverNodeSerializer(store: RocksDBVersionedStore) extends ErgoSerializer[ProverNodes[DigestType]] { - import VersionedLDBAVLStorage.{InternalNodePrefix,LeafPrefix} + import VersionedRocksDBAVLStorage.{InternalNodePrefix,LeafPrefix} override def serialize(node: ProverNodes[DigestType], w: Writer): Unit = { node match { diff --git a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProxyInternalProverNode.scala b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProxyInternalProverNode.scala index c82b52cf3e..ebc1551bbf 100644 --- a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProxyInternalProverNode.scala +++ b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProxyInternalProverNode.scala @@ -1,7 +1,7 @@ package scorex.crypto.authds.avltree.batch import scorex.crypto.authds.{ADKey, Balance} -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore import InternalNode.InternalNodePrefix import scorex.crypto.authds.avltree.batch.Constants.{DigestType, hashFn} @@ -15,7 +15,7 @@ class ProxyInternalProverNode(protected var pk: ADKey, val leftLabel: ADKey, val rightLabel: ADKey, protected var pb: Balance = Balance @@ 0.toByte) - (store: LDBVersionedStore) + (store: RocksDBVersionedStore) extends InternalProverNode(k = pk, l = null, r = null, b = pb)(hashFn) { override protected def computeLabel: DigestType = { @@ -24,14 +24,14 @@ class ProxyInternalProverNode(protected var pk: ADKey, override def left: ProverNodes[DigestType] = { if (l == null) { - l = VersionedLDBAVLStorage.fetch(leftLabel)(store) + l = VersionedRocksDBAVLStorage.fetch(leftLabel)(store) } l } override def right: ProverNodes[DigestType] = { if (r == null) { - r = VersionedLDBAVLStorage.fetch(rightLabel)(store) + r = VersionedRocksDBAVLStorage.fetch(rightLabel)(store) } r } diff --git a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedRocksDBAVLStorage.scala similarity index 90% rename from avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala rename to avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedRocksDBAVLStorage.scala index 95d8e536dc..ec98a933a3 100644 --- a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala +++ b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedRocksDBAVLStorage.scala @@ -2,13 +2,13 @@ package scorex.crypto.authds.avltree.batch import com.google.common.primitives.Ints import scorex.crypto.authds.avltree.batch.Constants.{DigestType, HashFnType, hashFn} -import scorex.crypto.authds.avltree.batch.VersionedLDBAVLStorage.{topNodeHashKey, topNodeHeightKey} +import scorex.crypto.authds.avltree.batch.VersionedRocksDBAVLStorage.{topNodeHashKey, topNodeHeightKey} import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, BatchAVLProverSubtree, ProxyInternalNode} import scorex.crypto.authds.{ADDigest, ADKey} import scorex.util.encode.Base16 import scorex.crypto.hash import scorex.crypto.hash.Digest32 -import scorex.db.{LDBKVStore, LDBVersionedStore} +import scorex.db.{RocksDBKVStore, RocksDBVersionedStore} import scorex.util.ScorexLogging import scala.collection.mutable @@ -19,13 +19,13 @@ import scala.util.{Failure, Try} * * @param store - level db storage to save the tree in */ -class VersionedLDBAVLStorage(store: LDBVersionedStore) +class VersionedRocksDBAVLStorage(store: RocksDBVersionedStore) extends VersionedAVLStorage[DigestType] with ScorexLogging { - import VersionedLDBAVLStorage.nodeLabel + import VersionedRocksDBAVLStorage.nodeLabel private def restorePrunedRootNode(): Try[(ProverNodes[DigestType], Int)] = Try { - val rootNode = VersionedLDBAVLStorage.fetch(ADKey @@ store.get(topNodeHashKey).get)(store) + val rootNode = VersionedRocksDBAVLStorage.fetch(ADKey @@ store.get(topNodeHashKey).get)(store) val rootHeight = Ints.fromByteArray(store.get(topNodeHeightKey).get) rootNode -> rootHeight @@ -73,7 +73,7 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore) isTop: Boolean): Array[(Array[Byte], Array[Byte])] = { // Should always serialize top node. It may not be new if it is the creation of the tree if (node.isNew || isTop) { - val pair: (Array[Byte], Array[Byte]) = (nodeLabel(node), VersionedLDBAVLStorage.noStoreSerializer.toBytes(node)) + val pair: (Array[Byte], Array[Byte]) = (nodeLabel(node), VersionedRocksDBAVLStorage.noStoreSerializer.toBytes(node)) node match { case n: InternalProverNode[DigestType] => val leftSubtree = serializedVisitedNodes(n.left, isTop = false) @@ -98,13 +98,13 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore) * @param expectedRootHash - expected UTXO set authenticating tree root hash * @return - hash of root node of tree, or failure if an error (e.g. in database) happened */ - def dumpSnapshot(dumpStorage: LDBKVStore, manifestDepth: Byte, expectedRootHash: Array[Byte]): Try[Array[Byte]] = { + def dumpSnapshot(dumpStorage: RocksDBKVStore, manifestDepth: Byte, expectedRootHash: Array[Byte]): Try[Array[Byte]] = { store.processSnapshot { dbReader => def subtreeLoop(label: DigestType, builder: mutable.ArrayBuilder[Byte]): Unit = { val nodeBytes = dbReader.get(label) builder ++= nodeBytes - val node = VersionedLDBAVLStorage.noStoreSerializer.parseBytes(nodeBytes) + val node = VersionedRocksDBAVLStorage.noStoreSerializer.parseBytes(nodeBytes) node match { case in: ProxyInternalNode[DigestType] => subtreeLoop(Digest32 @@@ in.leftLabel, builder) @@ -123,7 +123,7 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore) def manifestLoop(nodeDbKey: Array[Byte], level: Int, manifestBuilder: mutable.ArrayBuilder[Byte]): Unit = { val nodeBytes = dbReader.get(nodeDbKey) manifestBuilder ++= nodeBytes - val node = VersionedLDBAVLStorage.noStoreSerializer.parseBytes(nodeBytes) + val node = VersionedRocksDBAVLStorage.noStoreSerializer.parseBytes(nodeBytes) node match { case in: ProxyInternalNode[DigestType] if level == manifestDepth => dumpSubtree(Digest32 @@@ in.leftLabel) @@ -156,7 +156,7 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore) } -object VersionedLDBAVLStorage { +object VersionedRocksDBAVLStorage { private[batch] val topNodeHashKey: Array[Byte] = Array.fill(StateTreeParameters.labelSize)(123: Byte) private[batch] val topNodeHeightKey: Array[Byte] = Array.fill(StateTreeParameters.labelSize)(124: Byte) @@ -179,7 +179,7 @@ object VersionedLDBAVLStorage { * @return node read from the database (or throws exception if there is no such node), in case of internal node it * returns pruned internal node, so a node where left and right children not stored, only their hashes */ - def fetch(dbKey: ADKey)(store: LDBVersionedStore): ProverNodes[DigestType] = { + def fetch(dbKey: ADKey)(store: RocksDBVersionedStore): ProverNodes[DigestType] = { val bytes = store(dbKey) val node = new ProverNodeSerializer(store).parseBytes(bytes) node.isNew = false @@ -202,7 +202,7 @@ object VersionedLDBAVLStorage { def recreate(manifest: BatchAVLProverManifest[DigestType], chunks: Iterator[BatchAVLProverSubtree[DigestType]], additionalData: Iterator[(Array[Byte], Array[Byte])], - store: LDBVersionedStore): Try[VersionedLDBAVLStorage] = { + store: RocksDBVersionedStore): Try[VersionedRocksDBAVLStorage] = { //todo: the function below copy-pasted from BatchAVLProver, eliminate boilerplate? def idCollector(node: ProverNodes[DigestType]): Iterator[(Array[Byte], Array[Byte])] = { @@ -219,12 +219,12 @@ object VersionedLDBAVLStorage { val rootNode = manifest.root val rootNodeHeight = manifest.rootHeight - val digestWrapper = VersionedLDBAVLStorage.digest(rootNode.label, rootNodeHeight) + val digestWrapper = VersionedRocksDBAVLStorage.digest(rootNode.label, rootNodeHeight) val indices = Iterator(topNodeHashKey -> nodeLabel(rootNode), topNodeHeightKey -> Ints.toByteArray(rootNodeHeight)) val nodesIterator = idCollector(manifest.root) ++ chunks.flatMap(subtree => idCollector(subtree.subtreeTop)) store.update(digestWrapper, toRemove = Nil, toUpdate = indices ++ nodesIterator ++ additionalData).map { _ => - new VersionedLDBAVLStorage(store) + new VersionedRocksDBAVLStorage(store) } } diff --git a/avldb/src/main/scala/scorex/db/KVStoreReader.scala b/avldb/src/main/scala/scorex/db/KVStoreReader.scala index 9147aeb358..c46a6dc4cb 100644 --- a/avldb/src/main/scala/scorex/db/KVStoreReader.scala +++ b/avldb/src/main/scala/scorex/db/KVStoreReader.scala @@ -1,9 +1,9 @@ package scorex.db -import java.util.concurrent.locks.ReentrantReadWriteLock - -import org.iq80.leveldb.{DB, ReadOptions} +import org.rocksdb.ReadOptions +import scorex.db.RocksDBFactory.RegisteredDB +import java.util.concurrent.locks.ReentrantReadWriteLock import scala.collection.mutable /** @@ -15,7 +15,7 @@ trait KVStoreReader extends AutoCloseable { type K = Array[Byte] type V = Array[Byte] - protected val db: DB + protected val db: RegisteredDB protected val lock = new ReentrantReadWriteLock() @@ -33,38 +33,43 @@ trait KVStoreReader extends AutoCloseable { } } + /** + * Query if database contains key + * @param key - key + * @return true if key exists, false otherwise + */ + def contains(key: K): Boolean = { + lock.readLock().lock() + try { + db.contains(key) + } finally { + lock.readLock().unlock() + } + } /** - * Iterate through the database to read elements according to a filter function. - * @param cond - the filter function - * @return iterator over elements satisfying the filter function + * Read all the database elements. + * @return iterator over database contents */ - def getWithFilter(cond: (K, V) => Boolean): Iterator[(K, V)] = { + def getAll: Iterator[(K, V)] = { val ro = new ReadOptions() - ro.snapshot(db.getSnapshot) + ro.setSnapshot(db.getSnapshot) val iter = db.iterator(ro) try { iter.seekToFirst() val bf = mutable.ArrayBuffer.empty[(K, V)] - while (iter.hasNext) { - val next = iter.next() - val key = next.getKey - val value = next.getValue - if (cond(key, value)) bf += (key -> value) + while (iter.isValid) { + bf += (iter.key() -> iter.value()) + iter.next() } bf.toIterator } finally { iter.close() - ro.snapshot().close() + db.releaseSnapshot(ro.snapshot()) + ro.close() } } - /** - * Read all the database elements. - * @return iterator over database contents - */ - def getAll: Iterator[(K, V)] = getWithFilter((_, _) => true) - /** Returns value associated with the key, or default value from user */ def getOrElse(key: K, default: => V): V = @@ -97,23 +102,24 @@ trait KVStoreReader extends AutoCloseable { */ def getRange(start: K, end: K, limit: Int = Int.MaxValue): Array[(K, V)] = { val ro = new ReadOptions() - ro.snapshot(db.getSnapshot) + ro.setSnapshot(db.getSnapshot) val iter = db.iterator(ro) try { iter.seek(start) val bf = mutable.ArrayBuffer.empty[(K, V)] var elemCounter = 0 - while (iter.hasNext && elemCounter < limit) { - val next = iter.next() - if(ByteArrayUtils.compare(next.getKey, end) <= 0) { + while (iter.isValid && elemCounter < limit) { + if(ByteArrayUtils.compare(iter.key(), end) <= 0) { elemCounter += 1 - bf += (next.getKey -> next.getValue) + bf += (iter.key() -> iter.value()) } else elemCounter = limit // break + iter.next() } bf.toArray[(K,V)] } finally { iter.close() - ro.snapshot().close() + db.releaseSnapshot(ro.snapshot()) + ro.close() } } diff --git a/avldb/src/main/scala/scorex/db/LDBFactory.scala b/avldb/src/main/scala/scorex/db/LDBFactory.scala deleted file mode 100644 index 65fca5a397..0000000000 --- a/avldb/src/main/scala/scorex/db/LDBFactory.scala +++ /dev/null @@ -1,176 +0,0 @@ -package scorex.db - -import java.io.File -import java.util.concurrent.locks.ReentrantReadWriteLock -import org.iq80.leveldb.{DB, DBFactory, DBIterator, Options, Range, ReadOptions, Snapshot, WriteBatch, WriteOptions} -import scorex.util.ScorexLogging - -import scala.collection.mutable - -/** - * Registry of opened LevelDB instances. - * LevelDB prohibit access to the same storage file from more than one DB instance. - * And ergo application (mostly tests) quite frequently doesn't not explicitly close - * database and tries to reopen it. - */ -case class StoreRegistry(factory: DBFactory) extends DBFactory with ScorexLogging { - - val lock = new ReentrantReadWriteLock() - val map = new mutable.HashMap[File, RegisteredDB] - - /** - * Decorator of LevelDB DB class which overrides close() methods and unlinks database from registry on close. - * So if database was not explicitly closed, then next attempt to open database with the same path will - * return existed instance instead of creating new one. - */ - case class RegisteredDB(impl: DB, path: File) extends DB { - - def get(key: Array[Byte]): Array[Byte] = impl.get(key) - - def get(key: Array[Byte], options: ReadOptions): Array[Byte] = impl.get(key, options) - - def iterator: DBIterator = impl.iterator - - def iterator(options: ReadOptions): DBIterator = impl.iterator(options) - - def put(key: Array[Byte], value: Array[Byte]): Unit = impl.put(key, value) - - def delete(key: Array[Byte]): Unit = impl.delete(key) - - def write(batch: WriteBatch): Unit = impl.write(batch) - - def write(batch: WriteBatch, options: WriteOptions): Snapshot = impl.write(batch, options) - - def createWriteBatch: WriteBatch = impl.createWriteBatch() - - def put(key: Array[Byte], value: Array[Byte], options: WriteOptions): Snapshot = impl.put(key, value, options) - - def delete(key: Array[Byte], options: WriteOptions): Snapshot = impl.delete(key, options) - - def getSnapshot: Snapshot = impl.getSnapshot - - def getApproximateSizes(ranges: Range*): Array[Long] = impl.getApproximateSizes(ranges: _*) - - def getProperty(name: String): String = impl.getProperty(name) - - def suspendCompactions(): Unit = impl.suspendCompactions() - - def resumeCompactions(): Unit = impl.resumeCompactions() - - def compactRange(begin: Array[Byte], end: Array[Byte]): Unit = impl.compactRange(begin, end) - - override def close(): Unit = { - remove(path) - impl.close() - } - } - - private def add(file: File, create: => DB): DB = { - lock.writeLock().lock() - try { - map.getOrElseUpdate(file, RegisteredDB(create, file)) - } finally { - lock.writeLock().unlock() - } - } - - private def remove(path: File): Option[RegisteredDB] = { - lock.writeLock().lock() - try { - map.remove(path) - } finally { - lock.writeLock().unlock() - } - } - - def open(path: File, options: Options): DB = { - lock.writeLock().lock() - try { - add(path, factory.open(path, options)) - } catch { - case x: Throwable => - log.error(s"Failed to initialize storage: $x. Please check that directory $path exists and is not used by some other active node") - java.lang.System.exit(2) - null - } finally { - lock.writeLock().unlock() - } - } - - def destroy(path: File, options: Options): Unit = { - factory.destroy(path, options) - } - - def repair(path: File, options: Options): Unit = { - factory.repair(path, options) - } -} - -object LDBFactory extends ScorexLogging { - - private val nativeFactory = "org.fusesource.leveldbjni.JniDBFactory" - private val javaFactory = "org.iq80.leveldb.impl.Iq80DBFactory" - private val memoryPoolSize = 512 * 1024 - - def setLevelDBParams(factory: DBFactory): AnyRef = { - val pushMemoryPool = factory.getClass.getDeclaredMethod("pushMemoryPool", classOf[Int]) - pushMemoryPool.invoke(null, Integer.valueOf(memoryPoolSize)) - } - - def createKvDb(path: String): LDBKVStore = { - val dir = new File(path) - dir.mkdirs() - val options = new Options() - options.createIfMissing(true) - try { - val db = factory.open(dir, options) - new LDBKVStore(db) - } catch { - case x: Throwable => - log.error(s"Failed to initialize storage: $x. Please check that directory $path could be accessed " + - s"and is not used by some other active node") - java.lang.System.exit(2) - null - } - } - - - lazy val factory: DBFactory = { - val loaders = List(ClassLoader.getSystemClassLoader, this.getClass.getClassLoader) - - // As LevelDB-JNI has problems on Mac (see https://github.com/ergoplatform/ergo/issues/1067), - // we are using only pure-Java LevelDB on Mac - val isMac = System.getProperty("os.name").toLowerCase().indexOf("mac") >= 0 - val factories = if(isMac) { - List(javaFactory) - } else { - List(nativeFactory, javaFactory) - } - - val pairs = loaders.view - .zip(factories) - .flatMap { case (loader, factoryName) => - loadFactory(loader, factoryName).map(factoryName -> _) - } - - val (name, factory) = pairs.headOption.getOrElse { - throw new RuntimeException(s"Could not load any of the factory classes: $factories") - } - - if (name == javaFactory) { - log.warn("Using the pure java LevelDB implementation which is still experimental") - } else { - log.info(s"Loaded $name with $factory") - setLevelDBParams(factory) - } - StoreRegistry(factory) - } - - private def loadFactory(loader: ClassLoader, factoryName: String): Option[DBFactory] = - try Some(loader.loadClass(factoryName).getConstructor().newInstance().asInstanceOf[DBFactory]) - catch { - case e: Throwable => - log.warn(s"Failed to load database factory $factoryName due to: $e") - None - } -} diff --git a/avldb/src/main/scala/scorex/db/RocksDBFactory.scala b/avldb/src/main/scala/scorex/db/RocksDBFactory.scala new file mode 100644 index 0000000000..571ab14ce8 --- /dev/null +++ b/avldb/src/main/scala/scorex/db/RocksDBFactory.scala @@ -0,0 +1,90 @@ +package scorex.db + +import org.rocksdb._ +import org.rocksdb.util.SizeUnit +import scorex.util.ScorexLogging + +import java.io.File +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.locks.ReentrantReadWriteLock +import scala.collection.mutable + +/** + * Registry of opened RocksDB instances. + * RocksDB prohibit access to the same storage file from more than one DB instance. + * And ergo application (mostly tests) quite frequently doesn't not explicitly close + * database and tries to reopen it. + */ +object RocksDBFactory extends ScorexLogging { + + RocksDB.loadLibrary() + + private val lock = new ReentrantReadWriteLock() + private val map = new mutable.HashMap[File, RegisteredDB] + + /** + * Decorator of RocksDB class which overrides close() methods and unlinks database from registry on close. + * So if database was not explicitly closed, then next attempt to open database with the same path will + * return existed instance instead of creating new one. + */ + case class RegisteredDB(impl: RocksDB, path: File) { + val open: AtomicBoolean = new AtomicBoolean(true) + def get(key: Array[Byte]): Array[Byte] = { + if(open.get()) + impl.get(key) + else + null + } + def get(options: ReadOptions, key: Array[Byte]): Array[Byte] = impl.get(options, key) + def contains(key: Array[Byte]): Boolean = impl.keyExists(key) + def iterator: RocksIterator = impl.newIterator() + def iterator(options: ReadOptions): RocksIterator = impl.newIterator(options) + def put(key: Array[Byte], value: Array[Byte]): Unit = impl.put(key, value) + def write(options: WriteOptions, batch: WriteBatch): Unit = impl.write(options, batch) + def getSnapshot: Snapshot = impl.getSnapshot + def releaseSnapshot(snapshot: Snapshot): Unit = impl.releaseSnapshot(snapshot) + def close(): Unit = { + lock.writeLock().lock() + try { + map.remove(path) + if(System.getProperty("env") != "test") impl.syncWal() + impl.close() + open.set(false) + } finally { + lock.writeLock().unlock() + } + } + } + + private val normalOptions: Options = new Options() + .setCreateIfMissing(true) + .setAllowMmapReads(true) + .setAtomicFlush(true) + .setIncreaseParallelism(4) + .setCompressionType(CompressionType.LZ4_COMPRESSION) + .setCompactionStyle(CompactionStyle.LEVEL) + + private val testOptions: Options = new Options() + .setCreateIfMissing(true) + .setWriteBufferSize(64 * SizeUnit.KB) + .setManifestPreallocationSize(32 * SizeUnit.KB) + .setCompressionType(CompressionType.LZ4_COMPRESSION) + .setCompactionStyle(CompactionStyle.LEVEL) + + def open(path: File): RegisteredDB = { + lock.writeLock().lock() + try { + path.mkdirs() + val options = if(System.getProperty("env") == "test") testOptions else normalOptions + map.getOrElseUpdate(path, RegisteredDB(RocksDB.open(options, path.toString), path)) + } catch { + case x: Throwable => + log.error(s"Failed to initialize storage: $x. Please check that directory $path exists and is not used by some other active node") + java.lang.System.exit(2) + null + } finally { + lock.writeLock().unlock() + } + } + +} diff --git a/avldb/src/main/scala/scorex/db/LDBKVStore.scala b/avldb/src/main/scala/scorex/db/RocksDBKVStore.scala similarity index 72% rename from avldb/src/main/scala/scorex/db/LDBKVStore.scala rename to avldb/src/main/scala/scorex/db/RocksDBKVStore.scala index 1c8215315f..b24519ceca 100644 --- a/avldb/src/main/scala/scorex/db/LDBKVStore.scala +++ b/avldb/src/main/scala/scorex/db/RocksDBKVStore.scala @@ -1,10 +1,11 @@ package scorex.db -import org.iq80.leveldb.DB +import org.rocksdb.{WriteBatch, WriteOptions} +import scorex.db.RocksDBFactory.RegisteredDB import scorex.util.ScorexLogging +import spire.syntax.all.cfor import scala.util.{Failure, Success, Try} -import spire.syntax.all.cfor /** @@ -12,10 +13,13 @@ import spire.syntax.all.cfor * * Both keys and values are var-sized byte arrays. */ -class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging { +class RocksDBKVStore(protected val db: RegisteredDB) extends KVStoreReader with ScorexLogging { /** Immutable empty array can be shared to avoid allocations. */ private val emptyArrayOfByteArray = Array.empty[Array[Byte]] + /** default write options, snyc enabled */ + private val wo: WriteOptions = new WriteOptions().setSync(true) + /** * Update this database atomically with a batch of insertion and removal operations * @@ -25,12 +29,12 @@ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging * @return - error if it happens, or success status */ def update(toInsertKeys: Array[K], toInsertValues: Array[V], toRemove: Array[K]): Try[Unit] = { - val batch = db.createWriteBatch() + val batch = new WriteBatch() try { require(toInsertKeys.length == toInsertValues.length) cfor(0)(_ < toInsertKeys.length, _ + 1) { i => batch.put(toInsertKeys(i), toInsertValues(i))} cfor(0)(_ < toRemove.length, _ + 1) { i => batch.delete(toRemove(i))} - db.write(batch) + db.write(wo, batch) Success(()) } catch { case t: Throwable => Failure(t) @@ -68,24 +72,4 @@ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging update(emptyArrayOfByteArray, emptyArrayOfByteArray, keys) } - /** - * Get last key within some range (inclusive) by used comparator. - * Could be useful for applications with sequential ids. - * The method iterates over all the keys so could be slow if there are many keys in the range. - */ - def lastKeyInRange(first: Array[Byte], last: Array[Byte]): Option[K] = { - import util.control.Breaks._ - - val i = db.iterator() - var res: Option[K] = None - i.seek(first) - breakable { - while (i.hasNext) { - val key = i.next().getKey - if (ByteArrayUtils.compare(key, last) <= 0) res = Some(key) else break() - } - } - res - } - } diff --git a/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala b/avldb/src/main/scala/scorex/db/RocksDBVersionedStore.scala similarity index 86% rename from avldb/src/main/scala/scorex/db/LDBVersionedStore.scala rename to avldb/src/main/scala/scorex/db/RocksDBVersionedStore.scala index 51340723f9..3c46cf327e 100644 --- a/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala +++ b/avldb/src/main/scala/scorex/db/RocksDBVersionedStore.scala @@ -1,16 +1,15 @@ package scorex.db -import java.io.File -import scorex.db.LDBFactory.factory -import org.iq80.leveldb._ - -import java.nio.ByteBuffer -import scala.collection.mutable.ArrayBuffer -import java.util.concurrent.locks.ReentrantReadWriteLock +import org.rocksdb.{ReadOptions, WriteBatch, WriteOptions} import scorex.crypto.hash.Blake2b256 -import scorex.db.LDBVersionedStore.SnapshotReadInterface +import scorex.db.RocksDBFactory.RegisteredDB +import scorex.db.RocksDBVersionedStore.SnapshotReadInterface import scorex.util.ScorexLogging +import java.io.File +import java.nio.ByteBuffer +import java.util.concurrent.locks.ReentrantReadWriteLock +import scala.collection.mutable.ArrayBuffer import scala.util.{Failure, Success, Try} @@ -26,7 +25,7 @@ import scala.util.{Failure, Success, Try} * @param initialKeepVersions - number of versions to keep when the store is created. Can be changed after. * */ -class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) +class RocksDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) extends KVStoreReader with ScorexLogging { type VersionID = Array[Byte] @@ -37,10 +36,10 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) private var keepVersions: Int = initialKeepVersions - override val db: DB = createDB(dir, "ldb_main") // storage for main data + override val db: RegisteredDB = createDB(dir, "ldb_main") // storage for main data override val lock = new ReentrantReadWriteLock() - private val undo: DB = createDB(dir, "ldb_undo") // storage for undo data + private val undo: RegisteredDB = createDB(dir, "ldb_undo") // storage for undo data private var lsn: LSN = getLastLSN // last assigned logical serial number private var versionLsn = ArrayBuffer.empty[LSN] // LSNs of versions (var because we need to invert this array) @@ -52,12 +51,8 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) //default write options, no sync! private val writeOptions = new WriteOptions() - private def createDB(dir: File, storeName: String): DB = { - val op = new Options() - op.createIfMissing(true) - op.paranoidChecks(true) - factory.open(new File(dir, storeName), op) - } + private def createDB(dir: File, storeName: String): RegisteredDB = + RocksDBFactory.open(new File(dir, storeName)) /** Set new keep versions threshold, remove not needed versions and return old value of keep versions */ def setKeepVersions(newKeepVersions: Int): Int = { @@ -102,12 +97,12 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) def processAll(consumer: (K, V) => Unit): Unit = { lock.readLock().lock() - val iterator = db.iterator() + val iterator = db.iterator try { iterator.seekToFirst() - while (iterator.hasNext) { - val n = iterator.next() - consumer(n.getKey, n.getValue) + while (iterator.isValid) { + consumer(iterator.key(), iterator.value()) + iterator.next() } } finally { iterator.close() @@ -138,8 +133,8 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) val iterator = undo.iterator try { iterator.seekToFirst() - if (iterator.hasNext) { - decodeLSN(iterator.peekNext().getKey) + if (iterator.isValid) { + decodeLSN(iterator.key()) } else { 0 } @@ -166,17 +161,17 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) var lastVersion: Option[VersionID] = None var lastLsn: LSN = 0 // We iterate in LSN descending order - val iterator = undo.iterator() + val iterator = undo.iterator iterator.seekToFirst() - while (iterator.hasNext) { - val entry = iterator.next - val currVersion = deserializeUndo(entry.getValue).versionID - lastLsn = decodeLSN(entry.getKey) + while (iterator.isValid) { + val currVersion = deserializeUndo(iterator.value()).versionID + lastLsn = decodeLSN(iterator.key()) if (!lastVersion.exists(_.sameElements(currVersion))) { versionLsn += lastLsn + 1 // this is first LSN of successor version versions += currVersion lastVersion = Some(currVersion) } + iterator.next() } iterator.close() // As far as org.iq80.leveldb doesn't support iteration in reverse order, we have to iterate in the order @@ -241,8 +236,8 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) toUpdate: TraversableOnce[(Array[Byte], Array[Byte])]): Try[Unit] = Try { lock.writeLock().lock() val lastLsn = lsn // remember current LSN value - val batch = db.createWriteBatch() - val undoBatch = undo.createWriteBatch() + val batch = new WriteBatch() + val undoBatch = new WriteBatch() try { toRemove.foreach(key => { batch.delete(key) @@ -266,7 +261,7 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) if (lsn == lastLsn) { // no records were written for this version: generate dummy record undoBatch.put(newLSN(), serializeUndo(versionID, new Array[Byte](0), null)) } - undo.write(undoBatch, writeOptions) + undo.write(writeOptions, undoBatch) if (lastVersion.isEmpty || !versionID.sameElements(lastVersion.get)) { versions += versionID versionLsn += lastLsn + 1 // first LSN for this version @@ -284,7 +279,7 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) } } - db.write(batch, writeOptions) + db.write(writeOptions, batch) lastVersion = Some(versionID) } finally { // Make sure you close the batch to avoid resource leaks. @@ -305,12 +300,12 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) if (deteriorated >= 0) { val fromLsn = versionLsn(0) val tillLsn = if (deteriorated+1 < versions.size) versionLsn(deteriorated+1) else lsn+1 - val batch = undo.createWriteBatch() + val batch = new WriteBatch() try { for (lsn <- fromLsn until tillLsn) { batch.delete(encodeLSN(lsn)) } - undo.write(batch, writeOptions) + undo.write(writeOptions, batch) } finally { batch.close() } @@ -330,13 +325,6 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) } finally { lock.writeLock().unlock() } - undo.resumeCompactions() - db.resumeCompactions() - } - - def cleanStop(): Unit = { - undo.suspendCompactions() - db.suspendCompactions() } override def close(): Unit = { @@ -356,22 +344,21 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) val versionIndex = versions.indexWhere(_.sameElements(versionID)) if (versionIndex >= 0) { if (versionIndex != versions.size-1) { - val batch = db.createWriteBatch() - val undoBatch = undo.createWriteBatch() + val batch = new WriteBatch() + val undoBatch = new WriteBatch() var nUndoRecords: Long = 0 - val iterator = undo.iterator() + val iterator = undo.iterator var lastLsn: LSN = 0 try { var undoing = true iterator.seekToFirst() - while (undoing && iterator.hasNext) { - val entry = iterator.next() - val undo = deserializeUndo(entry.getValue) + while (undoing && iterator.isValid) { + val undo = deserializeUndo(iterator.value()) if (undo.versionID.sameElements(versionID)) { undoing = false - lastLsn = decodeLSN(entry.getKey) + lastLsn = decodeLSN(iterator.key()) } else { - undoBatch.delete(entry.getKey) + undoBatch.delete(iterator.key()) nUndoRecords += 1 if (undo.value == null) { if (undo.key.length != 0) { // dummy record @@ -381,9 +368,10 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) batch.put(undo.key, undo.value) } } + iterator.next() } - db.write(batch, writeOptions) - undo.write(undoBatch, writeOptions) + db.write(writeOptions, batch) + undo.write(writeOptions, undoBatch) } finally { // Make sure you close the batch to avoid resource leaks. iterator.close() @@ -424,11 +412,11 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) val ro = new ReadOptions() try { lock.writeLock().lock() - ro.snapshot(db.getSnapshot) + ro.setSnapshot(db.getSnapshot) lock.writeLock().unlock() object readInterface extends SnapshotReadInterface { - def get(key: Array[Byte]): Array[Byte] = db.get(key, ro) + def get(key: Array[Byte]): Array[Byte] = db.get(ro, key) } Success(logic(readInterface)) } catch { @@ -437,13 +425,14 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) Failure(t) } finally { // Close the snapshot to avoid resource leaks - ro.snapshot().close() + db.releaseSnapshot(ro.snapshot()) + ro.close() } } } -object LDBVersionedStore { +object RocksDBVersionedStore { /** * Interface to read from versioned database snapshot which can be provided to clients in order to serve them with diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/AVLStorageWithPersistentProverSpec.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/AVLStorageWithPersistentProverSpec.scala index 47588f0af5..6c979225e3 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/AVLStorageWithPersistentProverSpec.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/AVLStorageWithPersistentProverSpec.scala @@ -2,11 +2,11 @@ package scorex.crypto.authds.avltree.batch import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec -import scorex.crypto.authds.avltree.batch.benchmark.LDBVersionedStoreBenchmark.getRandomTempDir +import scorex.crypto.authds.avltree.batch.benchmark.RocksDBVersionedStoreBenchmark.getRandomTempDir import scorex.crypto.authds.{ADDigest, ADKey, SerializedAdProof, ADValue} import scorex.crypto.hash.{Digest32, Blake2b256} import scorex.utils.Random -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore import scala.util.{Success, Failure, Try} @@ -15,9 +15,9 @@ class AVLStorageWithPersistentProverSpec extends AnyPropSpec with Matchers { type HF = Blake2b256.type implicit val hf: HF = Blake2b256 - val stateStore = new LDBVersionedStore(getRandomTempDir, 10) + val stateStore = new RocksDBVersionedStore(getRandomTempDir, 10) - protected lazy val storage = new VersionedLDBAVLStorage(stateStore) + protected lazy val storage = new VersionedRocksDBAVLStorage(stateStore) protected lazy val persistentProver: PersistentBatchAVLProver[Digest32, HF] = PersistentBatchAVLProver.create( diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/LDBVersionedStoreSpecification.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/RocksDBVersionedStoreSpecification.scala similarity index 83% rename from avldb/src/test/scala/scorex/crypto/authds/avltree/batch/LDBVersionedStoreSpecification.scala rename to avldb/src/test/scala/scorex/crypto/authds/avltree/batch/RocksDBVersionedStoreSpecification.scala index c1fe98d5df..001bf01cd7 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/LDBVersionedStoreSpecification.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/RocksDBVersionedStoreSpecification.scala @@ -6,11 +6,11 @@ import org.scalatest.propspec.AnyPropSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks import scorex.crypto.authds.avltree.batch.helpers.TestHelper import scorex.crypto.hash.Blake2b256 -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore import scala.collection.mutable.ArrayBuffer -class LDBVersionedStoreSpecification extends AnyPropSpec +class RocksDBVersionedStoreSpecification extends AnyPropSpec with ScalaCheckPropertyChecks with Matchers with TestHelper { @@ -18,7 +18,7 @@ class LDBVersionedStoreSpecification extends AnyPropSpec override protected val KL = 32 override protected val VL = 8 - val storeTest: LDBVersionedStore => Unit = { store => + val storeTest: RocksDBVersionedStore => Unit = { store => var version = store.lastVersionID val keys: ArrayBuffer[(Array[Byte], Array[Byte])] = ArrayBuffer() forAll { b: Array[Byte] => @@ -37,6 +37,6 @@ class LDBVersionedStoreSpecification extends AnyPropSpec } } - property("LDBVersionedStore") { storeTest(createVersionedStore()) } + property("RocksDBVersionedStore") { storeTest(createVersionedStore()) } } diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedRocksDBAVLStorageSpecification.scala similarity index 97% rename from avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala rename to avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedRocksDBAVLStorageSpecification.scala index 810e986b00..aea3535589 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedRocksDBAVLStorageSpecification.scala @@ -11,7 +11,7 @@ import scorex.crypto.authds.avltree.batch.helpers.TestHelper import scorex.crypto.authds.{ADDigest, ADKey, ADValue, SerializedAdProof} import scorex.util.encode.Base16 import scorex.crypto.hash.{Blake2b256, Digest32} -import scorex.db.{LDBFactory, LDBVersionedStore} +import scorex.db.{RocksDBFactory, RocksDBKVStore, RocksDBVersionedStore} import scorex.util.ByteArrayBuilder import scorex.util.serialization.VLQByteBufferWriter import scorex.utils.{Random => RandomBytes} @@ -21,7 +21,7 @@ import scala.concurrent.duration._ import scala.concurrent.{Await, Future} import scala.util.{Success, Try} -class VersionedLDBAVLStorageSpecification +class VersionedRocksDBAVLStorageSpecification extends AnyPropSpec with ScalaCheckPropertyChecks with Matchers @@ -204,7 +204,7 @@ class VersionedLDBAVLStorageSpecification noException should be thrownBy storage.rollbackVersions.foreach(v => prover.rollback(v).get) } - def testAddInfoSaving(createStore: Int => LDBVersionedStore): Assertion = { + def testAddInfoSaving(createStore: Int => RocksDBVersionedStore): Assertion = { val store = createStore(1000) val storage = createVersionedStorage(store) val prover = createPersistentProver(storage) @@ -247,7 +247,7 @@ class VersionedLDBAVLStorageSpecification store.get(addInfo2._1) shouldBe None } - def removeFromLargerSetSingleRandomElementTest(createStore: Int => LDBVersionedStore): Unit = { + def removeFromLargerSetSingleRandomElementTest(createStore: Int => RocksDBVersionedStore): Unit = { val minSetSize = 10000 val maxSetSize = 200000 @@ -349,8 +349,8 @@ class VersionedLDBAVLStorageSpecification val prover = createPersistentProver() blockchainWorkflowTest(prover) - val storage = prover.storage.asInstanceOf[VersionedLDBAVLStorage] - val store = LDBFactory.createKvDb(getRandomTempDir.getAbsolutePath) + val storage = prover.storage.asInstanceOf[VersionedRocksDBAVLStorage] + val store = new RocksDBKVStore(RocksDBFactory.open(getRandomTempDir)) val rootNodeLabel = storage.dumpSnapshot(store, manifestDepth, prover.digest.dropRight(1)).get rootNodeLabel.sameElements(prover.digest.dropRight(1)) shouldBe true diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageStatefulSpecification.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedRocksDBAVLStorageStatefulSpecification.scala similarity index 97% rename from avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageStatefulSpecification.scala rename to avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedRocksDBAVLStorageStatefulSpecification.scala index 3d393582c6..7a0a3c62ca 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageStatefulSpecification.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedRocksDBAVLStorageStatefulSpecification.scala @@ -12,18 +12,18 @@ import scorex.utils.{Random => RandomBytes} import scala.util.{Random, Success, Failure, Try} -class VersionedLDBAVLStorageStatefulSpecification extends AnyPropSpec { +class VersionedRocksDBAVLStorageStatefulSpecification extends AnyPropSpec { val params = Parameters.default .withMinSize(10) .withMaxSize(50) .withMinSuccessfulTests(15) - property("LDBAVLStorage: rollback in stateful environment") { - WithLDB.property().check(params) + property("RocksDBAVLStorage: rollback in stateful environment") { + WithRocksDB.property().check(params) } } -object WithLDB extends VersionedLDBAVLStorageStatefulCommands with TestHelper { +object WithRocksDB extends VersionedRocksDBAVLStorageStatefulCommands with TestHelper { override protected val KL = 32 override protected val VL = 8 @@ -33,7 +33,7 @@ object WithLDB extends VersionedLDBAVLStorageStatefulCommands with TestHelper { } } -trait VersionedLDBAVLStorageStatefulCommands extends Commands { this: TestHelper => +trait VersionedRocksDBAVLStorageStatefulCommands extends Commands { this: TestHelper => override type State = Operations override type Sut = PersistentBatchAVLProver[Digest32, HF] diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala index 6179c933cc..5e9a81aad4 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/BatchingBenchmark.scala @@ -2,10 +2,10 @@ package scorex.crypto.authds.avltree.batch.benchmark import scorex.crypto.authds._ import scorex.crypto.authds.avltree.batch.helpers.FileHelper -import scorex.crypto.authds.avltree.batch.{VersionedLDBAVLStorage, _} +import scorex.crypto.authds.avltree.batch._ import scorex.crypto.hash.{Blake2b256, Digest32} import scorex.utils.Random -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore object BatchingBenchmark extends App with FileHelper { val KeyLength = 26 @@ -18,8 +18,8 @@ object BatchingBenchmark extends App with FileHelper { implicit val hf: HF = Blake2b256 type HF = Blake2b256.type - val store = new LDBVersionedStore(getRandomTempDir, initialKeepVersions = 10) - val storage = new VersionedLDBAVLStorage(store) + val store = new RocksDBVersionedStore(getRandomTempDir, initialKeepVersions = 10) + val storage = new VersionedRocksDBAVLStorage(store) require(storage.isEmpty) val mods = generateModifications() var digest: ADDigest = ADDigest @@ Array[Byte]() diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/OOMTest.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/OOMTest.scala index 3547549ee2..f7e70a32ad 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/OOMTest.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/OOMTest.scala @@ -8,7 +8,7 @@ import scorex.crypto.authds.{ADDigest, ADKey, ADValue} import scorex.crypto.authds.avltree.batch._ import scorex.util.encode.Base16 import scorex.crypto.hash.{Blake2b256, Digest32} -import scorex.db.{ByteArrayWrapper, LDBVersionedStore} +import scorex.db.{ByteArrayWrapper, RocksDBVersionedStore} import scala.collection.immutable.SortedMap import scala.math.Ordering.Implicits._ @@ -27,10 +27,10 @@ object OOMTest extends App { protected implicit val hf = Blake2b256 val dir: File = Files.createTempDirectory("oom-test").toFile - val store = new LDBVersionedStore(dir, initialKeepVersions = 200) + val store = new RocksDBVersionedStore(dir, initialKeepVersions = 200) val bestVersionKey = Blake2b256("best state version") - protected lazy val storage = new VersionedLDBAVLStorage(store) + protected lazy val storage = new VersionedRocksDBAVLStorage(store) val afterGenesisStateDigestHex: String = "78b130095239561ecf5449a7794c0615326d1fd007cc79dcc286e46e4beb1d3f01" val afterGenesisStateDigest: ADDigest = ADDigest @@ Base16.decode(afterGenesisStateDigestHex).get diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/LDBVersionedStoreBenchmark.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/RocksDBVersionedStoreBenchmark.scala similarity index 86% rename from avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/LDBVersionedStoreBenchmark.scala rename to avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/RocksDBVersionedStoreBenchmark.scala index 62c16fcc10..2c5b10e1a0 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/LDBVersionedStoreBenchmark.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/RocksDBVersionedStoreBenchmark.scala @@ -3,16 +3,16 @@ package scorex.crypto.authds.avltree.batch.benchmark import com.google.common.primitives.Longs import scorex.crypto.authds.avltree.batch.helpers.FileHelper import scorex.utils.Random -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore -object LDBVersionedStoreBenchmark extends App with FileHelper { +object RocksDBVersionedStoreBenchmark extends App with FileHelper { val KL = 32 val VL = 8 val LL = 32 val NumMods = 2000000 val Step = 1000 - val store = new LDBVersionedStore(getRandomTempDir, 10) + val store = new RocksDBVersionedStore(getRandomTempDir, 10) val mods = generateModifications() var currentVersion: Option[Long] = None diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala index 0b2c4d79a9..dde6f8f99c 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala @@ -4,7 +4,7 @@ import scorex.crypto.authds.avltree.batch._ import scorex.crypto.authds.{ADDigest, SerializedAdProof} import scorex.util.encode.Base58 import scorex.crypto.hash.{Blake2b256, Digest32} -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore trait TestHelper extends FileHelper { @@ -15,20 +15,20 @@ trait TestHelper extends FileHelper { type PROVER = BatchAVLProver[D, HF] type VERIFIER = BatchAVLVerifier[D, HF] type PERSISTENT_PROVER = PersistentBatchAVLProver[D, HF] - type STORAGE = VersionedLDBAVLStorage + type STORAGE = VersionedRocksDBAVLStorage protected val KL: Int protected val VL: Int implicit val hf: HF = Blake2b256 - def createVersionedStore(initialKeepVersions: Int = 10): LDBVersionedStore = { + def createVersionedStore(initialKeepVersions: Int = 10): RocksDBVersionedStore = { val dir = getRandomTempDir - new LDBVersionedStore(dir, initialKeepVersions = initialKeepVersions) + new RocksDBVersionedStore(dir, initialKeepVersions = initialKeepVersions) } - def createVersionedStorage(store: LDBVersionedStore): STORAGE = - new VersionedLDBAVLStorage(store) + def createVersionedStorage(store: RocksDBVersionedStore): STORAGE = + new VersionedRocksDBAVLStorage(store) def createPersistentProver(storage: STORAGE): PERSISTENT_PROVER = { val prover = new BatchAVLProver[D, HF](KL, Some(VL)) diff --git a/avldb/src/test/scala/scorex/db/LDBVersionedStoreSpec.scala b/avldb/src/test/scala/scorex/db/RocksDBVersionedStoreSpec.scala similarity index 89% rename from avldb/src/test/scala/scorex/db/LDBVersionedStoreSpec.scala rename to avldb/src/test/scala/scorex/db/RocksDBVersionedStoreSpec.scala index 59863452ab..f52b54d900 100644 --- a/avldb/src/test/scala/scorex/db/LDBVersionedStoreSpec.scala +++ b/avldb/src/test/scala/scorex/db/RocksDBVersionedStoreSpec.scala @@ -3,16 +3,16 @@ package scorex.db import com.google.common.primitives.Longs import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec -import scorex.crypto.authds.avltree.batch.benchmark.LDBVersionedStoreBenchmark.getRandomTempDir +import scorex.crypto.authds.avltree.batch.benchmark.RocksDBVersionedStoreBenchmark.getRandomTempDir import scala.collection.mutable import scala.util.Random //todo: rollbacks and pruning are checked in VersionedStoreSpec, merge both tests? -class LDBVersionedStoreSpec extends AnyPropSpec with Matchers { +class RocksDBVersionedStoreSpec extends AnyPropSpec with Matchers { private val dir = getRandomTempDir - private val store = new LDBVersionedStore(dir, 100) + private val store = new RocksDBVersionedStore(dir, 100) property("last version correct && versionIdExists && rollbackVersions") { val versionNum = Random.nextInt().toLong @@ -44,7 +44,7 @@ class LDBVersionedStoreSpec extends AnyPropSpec with Matchers { store.rollbackVersions().toSeq.map(_.toSeq) shouldBe Seq(versionId3.toSeq, versionId2.toSeq, versionId.toSeq) } - property("processAll && getWithFilter") { + property("processAll && getAll") { val version = Longs.toByteArray(Long.MaxValue) val k1 = Longs.toByteArray(Int.MaxValue + 1) val k2 = Longs.toByteArray(Int.MaxValue + 2) @@ -53,7 +53,7 @@ class LDBVersionedStoreSpec extends AnyPropSpec with Matchers { store.update(version, Seq.empty, Seq(k1 -> v1, k2 -> v2)).get //read all keys - val keys = store.getWithFilter((_, _) => true).toSeq.map(_._1) + val keys = store.getAll.toSeq.map(_._1) val ks = keys.map(_.toSeq) ks.contains(k1.toSeq) shouldBe true @@ -74,7 +74,7 @@ class LDBVersionedStoreSpec extends AnyPropSpec with Matchers { store.update(Longs.toByteArray(Long.MinValue), keys, Seq.empty).get - store.getWithFilter((_, _) => true).toSeq.length shouldBe 0 + store.getAll.toSeq.length shouldBe 0 var cnt = 0 store.processAll({ case (_, _) => diff --git a/build.sbt b/build.sbt index 38fcb0fd47..1de5fa49f4 100644 --- a/build.sbt +++ b/build.sbt @@ -124,7 +124,7 @@ assemblyMergeStrategy in assembly := { case x if x.endsWith("module-info.class") => MergeStrategy.discard case "reference.conf" => CustomMergeStrategy.concatReversed case PathList("org", "bouncycastle", xs @ _*) => MergeStrategy.first - case PathList("org", "iq80", "leveldb", xs @ _*) => MergeStrategy.first + case PathList("org", "rocksdb", xs @ _*) => MergeStrategy.first case PathList("org", "bouncycastle", xs @ _*) => MergeStrategy.first case PathList("javax", "activation", xs @ _*) => MergeStrategy.last case PathList("javax", "annotation", xs @ _*) => MergeStrategy.last @@ -203,6 +203,7 @@ scapegoatVersion in ThisBuild := "1.3.3" scapegoatDisabledInspections := Seq("FinalModifierOnCaseClass") Test / testOptions := Seq(Tests.Filter(s => !s.endsWith("Bench"))) +Test / javaOptions := Seq("-Denv=test") lazy val avldb = (project in file("avldb")) .disablePlugins(ScapegoatSbtPlugin) // not compatible with crossScalaVersions @@ -219,11 +220,7 @@ lazy val avldb = (project in file("avldb")) javacOptions in(Compile, compile) ++= javacReleaseOption, libraryDependencies ++= Seq( // database dependencies - "org.ethereum" % "leveldbjni-all" % "1.18.3", - //the following pure-java leveldb implementation is needed only on specific platforms, such as 32-bit Raspberry Pi - //in future, it could be reasonable to have special builds with this Java db only, and for most of platforms use - //jni wrapper over native library included in leveldbjni-all - "org.iq80.leveldb" % "leveldb" % "0.12" + "org.rocksdb" % "rocksdbjni" % "8.11.3" ) ) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/utils/FileUtils.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/utils/FileUtils.scala index 898116dcd2..c3658263a6 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/utils/FileUtils.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/utils/FileUtils.scala @@ -1,7 +1,7 @@ package org.ergoplatform.wallet.utils import java.io.File -import java.nio.file.{Files, Path} +import java.nio.file.Files import scala.collection.JavaConverters._ import scala.util.Try @@ -18,22 +18,14 @@ trait FileUtils { } } - def createTempFile: java.io.File = { - val dir = createTempDir - val prefix = scala.util.Random.alphanumeric.take(randomPrefixLength).mkString - val suffix = scala.util.Random.alphanumeric.take(randomPrefixLength).mkString - val file = java.nio.file.Files.createTempFile(dir.toPath, prefix, suffix).toFile - file.deleteOnExit() - file - } - - def createTempDir: java.io.File = { + implicit def createTempDir: java.io.File = { val rndString = scala.util.Random.alphanumeric.take(randomPrefixLength).mkString createTempDirForPrefix(rndString) } private def createTempDirForPrefix(prefix: String): java.io.File = { val file = java.nio.file.Files.createTempDirectory(prefix).toFile + file.mkdirs() file.deleteOnExit() file } diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/TestFileUtils.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/TestFileUtils.scala deleted file mode 100644 index 3d29330cd6..0000000000 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/TestFileUtils.scala +++ /dev/null @@ -1,12 +0,0 @@ -package org.ergoplatform.wallet.utils - -import java.nio.file.Path - -trait TestFileUtils extends FileUtils { - - val basePath: Path = java.nio.file.Files.createTempDirectory(s"scorex-${System.nanoTime()}") - - sys.addShutdownHook { - deleteRecursive(basePath.toFile) - } -} diff --git a/src/main/resources/api/openapi-ai.yaml b/src/main/resources/api/openapi-ai.yaml index abdd65e7f7..cca4485bcd 100644 --- a/src/main/resources/api/openapi-ai.yaml +++ b/src/main/resources/api/openapi-ai.yaml @@ -1,7 +1,7 @@ openapi: "3.0.2" info: - version: "5.0.24" + version: "5.1.0" title: Ergo Node API description: Specification of Ergo Node API for ChatGPT plugin. The following endpoints supported diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 69bd3d991a..33b400a54b 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.2" info: - version: "5.0.24" + version: "5.1.0" title: Ergo Node API description: API docs for Ergo Node. Models are shared between all Ergo products contact: diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 2639e8b7f3..108b3087c1 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -436,7 +436,7 @@ scorex { nodeName = "ergo-node" # Network protocol version to be sent in handshakes - appVersion = 5.0.24 + appVersion = 5.1.0 # Network agent name. May contain information about client code # stack, starting from core code-base up to the end graphical interface. diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index e2a550282a..015f872939 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -35,7 +35,7 @@ import org.ergoplatform.modifiers.history.{ADProofs, ADProofsSerializer, BlockTr import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionSerializer} import org.ergoplatform.modifiers.transaction.TooHighCostError import org.ergoplatform.serialization.{ErgoSerializer, ManifestSerializer, SubtreeSerializer} -import scorex.crypto.authds.avltree.batch.VersionedLDBAVLStorage.splitDigest +import scorex.crypto.authds.avltree.batch.VersionedRocksDBAVLStorage.splitDigest import scala.annotation.tailrec import scala.collection.mutable diff --git a/src/main/scala/org/ergoplatform/network/peer/PeerDatabase.scala b/src/main/scala/org/ergoplatform/network/peer/PeerDatabase.scala index 5040fbe561..e68f54d8b3 100644 --- a/src/main/scala/org/ergoplatform/network/peer/PeerDatabase.scala +++ b/src/main/scala/org/ergoplatform/network/peer/PeerDatabase.scala @@ -1,10 +1,11 @@ package org.ergoplatform.network.peer import org.ergoplatform.nodeView.history.ErgoHistoryUtils._ -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream} + +import java.io.{ByteArrayInputStream, ByteArrayOutputStream, File, ObjectInputStream, ObjectOutputStream} import java.net.{InetAddress, InetSocketAddress} import org.ergoplatform.settings.ErgoSettings -import scorex.db.LDBFactory +import scorex.db.{RocksDBFactory, RocksDBKVStore} import scorex.util.ScorexLogging import scala.concurrent.duration._ @@ -15,7 +16,7 @@ import scala.util.{Failure, Success, Try} */ final class PeerDatabase(settings: ErgoSettings) extends ScorexLogging { - private val persistentStore = LDBFactory.createKvDb(s"${settings.directory}/peers") + private val persistentStore = new RocksDBKVStore(RocksDBFactory.open(new File(s"${settings.directory}/peers"))) private var peers = loadPeers match { diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 88c4a1cd30..7c4413b10a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -7,7 +7,7 @@ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.nodeView.history.extra.{ExtraIndex, ExtraIndexSerializer, Segment} import org.ergoplatform.settings.{Algos, CacheSettings, ErgoSettings} import org.ergoplatform.utils.ScorexEncoding -import scorex.db.{ByteArrayWrapper, LDBFactory, LDBKVStore} +import scorex.db.{ByteArrayWrapper, RocksDBFactory, RocksDBKVStore} import scorex.util.{ModifierId, ScorexLogging, idToBytes} import scala.util.{Failure, Success, Try} @@ -26,7 +26,7 @@ import scala.jdk.CollectionConverters.asScalaIteratorConverter * @param extraStore - key-value store, where key is id of Index and value is it's bytes * @param config - cache configs */ -class HistoryStorage(indexStore: LDBKVStore, objectsStore: LDBKVStore, extraStore: LDBKVStore, config: CacheSettings) +class HistoryStorage private(indexStore: RocksDBKVStore, objectsStore: RocksDBKVStore, extraStore: RocksDBKVStore, config: CacheSettings) extends ScorexLogging with AutoCloseable with ScorexEncoding { @@ -125,8 +125,11 @@ class HistoryStorage(indexStore: LDBKVStore, objectsStore: LDBKVStore, extraStor /** * @return if object with `id` is in the objects database */ - def contains(id: Array[Byte]): Boolean = get(id).isDefined - def contains(id: ModifierId): Boolean = get(id).isDefined + def contains(id: Array[Byte]): Boolean = objectsStore.contains(id) || extraStore.contains(id) + def contains(id: ModifierId): Boolean = { + val idBytes = idToBytes(id) + objectsStore.contains(idBytes) || extraStore.contains(idBytes) + } def insert(indexesToInsert: Array[(ByteArrayWrapper, Array[Byte])], objectsToInsert: Array[BlockSection]): Try[Unit] = { @@ -226,9 +229,9 @@ class HistoryStorage(indexStore: LDBKVStore, objectsStore: LDBKVStore, extraStor object HistoryStorage { def apply(ergoSettings: ErgoSettings): HistoryStorage = { - val indexStore = LDBFactory.createKvDb(s"${ergoSettings.directory}/history/index") - val objectsStore = LDBFactory.createKvDb(s"${ergoSettings.directory}/history/objects") - val extraStore = LDBFactory.createKvDb(s"${ergoSettings.directory}/history/extra") + val indexStore = new RocksDBKVStore(RocksDBFactory.open(new File(s"${ergoSettings.directory}/history/index"))) + val objectsStore = new RocksDBKVStore(RocksDBFactory.open(new File(s"${ergoSettings.directory}/history/objects"))) + val extraStore = new RocksDBKVStore(RocksDBFactory.open(new File(s"${ergoSettings.directory}/history/extra"))) new HistoryStorage(indexStore, objectsStore, extraStore, ergoSettings.cacheSettings) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/UtxoSetSnapshotProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/UtxoSetSnapshotProcessor.scala index 1d2972cfe9..c886cdf523 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/UtxoSetSnapshotProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/UtxoSetSnapshotProcessor.scala @@ -13,12 +13,12 @@ import scorex.core.network.ConnectedPeer import org.ergoplatform.serialization.SubtreeSerializer import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, BatchAVLProverSubtree} import scorex.crypto.hash.{Blake2b256, Digest32} -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore import scorex.util.{ModifierId, ScorexLogging} import spire.syntax.all.cfor import scala.util.{Failure, Random, Success, Try} -import scorex.crypto.authds.avltree.batch.{BatchAVLProver, PersistentBatchAVLProver, VersionedLDBAVLStorage} +import scorex.crypto.authds.avltree.batch.{BatchAVLProver, PersistentBatchAVLProver, VersionedRocksDBAVLStorage} /** * Parts of history processing and storage corresponding to UTXO set snapshot processing and storage @@ -204,7 +204,7 @@ trait UtxoSetSnapshotProcessor extends MinimalFullBlockHeightFunctions with Scor * @param blockId - id of a block corresponding to the tree (tree is on top of a state after the block) * @return prover with initialized tree database */ - def createPersistentProver(stateStore: LDBVersionedStore, + def createPersistentProver(stateStore: RocksDBVersionedStore, historyReader: ErgoHistoryReader, height: Height, blockId: ModifierId): Try[PersistentBatchAVLProver[Digest32, HF]] = { @@ -213,15 +213,15 @@ trait UtxoSetSnapshotProcessor extends MinimalFullBlockHeightFunctions with Scor log.info("Starting UTXO set snapshot transfer into state database") ErgoStateReader.reconstructStateContextBeforeEpoch(historyReader, height, settings) match { case Success(esc) => - val metadata = UtxoState.metadata(VersionTag @@@ blockId, VersionedLDBAVLStorage.digest(manifest.id, manifest.rootHeight), None, esc) - VersionedLDBAVLStorage.recreate(manifest, downloadedChunksIterator(), additionalData = metadata.toIterator, stateStore).flatMap { + val metadata = UtxoState.metadata(VersionTag @@@ blockId, VersionedRocksDBAVLStorage.digest(manifest.id, manifest.rootHeight), None, esc) + VersionedRocksDBAVLStorage.recreate(manifest, downloadedChunksIterator(), additionalData = metadata.toIterator, stateStore).flatMap { ldbStorage => log.info("Finished UTXO set snapshot transfer into state database") ldbStorage.restorePrunedProver().map { prunedAvlProver => new PersistentBatchAVLProver[Digest32, HF] { override var avlProver: BatchAVLProver[Digest32, ErgoAlgos.HF] = prunedAvlProver - override val storage: VersionedLDBAVLStorage = ldbStorage + override val storage: VersionedRocksDBAVLStorage = ldbStorage } } } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala index 94a51540b7..0a029a1fd8 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala @@ -11,7 +11,7 @@ import org.ergoplatform.nodeView.state.ErgoState.ModifierProcessing import org.ergoplatform.settings._ import org.ergoplatform.utils.LoggingUtil import org.ergoplatform.wallet.boxes.ErgoBoxSerializer -import scorex.db.{ByteArrayWrapper, LDBVersionedStore} +import scorex.db.{ByteArrayWrapper, RocksDBVersionedStore} import org.ergoplatform.core._ import org.ergoplatform.nodeView.LocallyGeneratedModifier import org.ergoplatform.utils.ScorexEncoding @@ -26,7 +26,7 @@ import scala.util.{Failure, Success, Try} */ class DigestState protected(override val version: VersionTag, override val rootDigest: ADDigest, - override val store: LDBVersionedStore, + override val store: RocksDBVersionedStore, override val ergoSettings: ErgoSettings) extends ErgoState[DigestState] with ScorexLogging @@ -161,7 +161,7 @@ object DigestState extends ScorexLogging with ScorexEncoding { stateContext: ErgoStateContext, dir: File, settings: ErgoSettings): Try[DigestState] = { - val store = new LDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) + val store = new RocksDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) val toUpdate = DigestState.metadata(version, rootHash, stateContext) store.update(org.ergoplatform.core.versionToBytes(version), Seq.empty, toUpdate).map { _ => @@ -176,7 +176,7 @@ object DigestState extends ScorexLogging with ScorexEncoding { rootHashOpt: Option[ADDigest], dir: File, settings: ErgoSettings): DigestState = { - val store = new LDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) + val store = new RocksDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) Try { val context = ErgoStateReader.storageStateContext(store, settings) (versionOpt, rootHashOpt) match { diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala index 9b57c853cf..157561e296 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala @@ -7,7 +7,7 @@ import org.ergoplatform.settings.{Algos, Constants, ErgoSettings, LaunchParamete import org.ergoplatform.core.VersionTag import scorex.crypto.authds.ADDigest import scorex.crypto.hash.Digest32 -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore import scorex.util.ScorexLogging import scala.util.{Failure, Success, Try} @@ -29,7 +29,7 @@ trait ErgoStateReader extends NodeViewComponent with ScorexLogging { */ def version: VersionTag - val store: LDBVersionedStore + val store: RocksDBVersionedStore protected def ergoSettings: ErgoSettings @@ -64,7 +64,7 @@ object ErgoStateReader extends ScorexLogging { /** * Read blockchain-related state context from `store` database */ - def storageStateContext(store: LDBVersionedStore, settings: ErgoSettings): ErgoStateContext = { + def storageStateContext(store: RocksDBVersionedStore, settings: ErgoSettings): ErgoStateContext = { store.get(ErgoStateReader.ContextKey) .flatMap(b => ErgoStateContextSerializer(settings.chainSettings).parseBytesTry(b).toOption) .getOrElse { diff --git a/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsDb.scala b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsDb.scala index 5889c9458f..b7623cbea9 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsDb.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsDb.scala @@ -4,12 +4,13 @@ import org.ergoplatform.ErgoLikeContext.Height import org.ergoplatform.nodeView.state.UtxoState.{ManifestId, SubtreeId} import org.ergoplatform.settings.{Algos, ErgoSettings} import org.ergoplatform.serialization.ManifestSerializer -import scorex.crypto.authds.avltree.batch.VersionedLDBAVLStorage +import scorex.crypto.authds.avltree.batch.VersionedRocksDBAVLStorage import scorex.crypto.hash.Digest32 -import scorex.db.{LDBFactory, LDBKVStore} +import scorex.db.{RocksDBFactory, RocksDBKVStore} import scorex.util.ScorexLogging import scorex.util.encode.Base16 +import java.io.File import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.util.{Failure, Success, Try} @@ -17,7 +18,7 @@ import scala.util.{Failure, Success, Try} /** * Interface for a (non-versioned) database storing UTXO set snapshots and metadata about them */ -class SnapshotsDb(store: LDBKVStore) extends ScorexLogging { +class SnapshotsDb(store: RocksDBKVStore) extends ScorexLogging { private val snapshotInfoKey: Array[Byte] = Array.fill(32)(0: Byte) @@ -105,7 +106,7 @@ class SnapshotsDb(store: LDBKVStore) extends ScorexLogging { * @return - id of the snapshot (root hash of its authenticating AVL+ tree), * or error happened during read-write process */ - def writeSnapshot(pullFrom: VersionedLDBAVLStorage, + def writeSnapshot(pullFrom: VersionedRocksDBAVLStorage, height: Height, expectedRootHash: Array[Byte], manifestDepth: Byte = ManifestSerializer.MainnetManifestDepth): Try[Array[Byte]] = { @@ -141,7 +142,7 @@ object SnapshotsDb { // internal method to open or init snapshots database in given folder // private[nodeView] to use it in tests also private[nodeView] def create(dir: String): SnapshotsDb = { - val store = LDBFactory.createKvDb(dir) + val store = new RocksDBKVStore(RocksDBFactory.open(new File(dir))) new SnapshotsDb(store) } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoSetSnapshotPersistence.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoSetSnapshotPersistence.scala index 1416bc11d7..80e999635c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoSetSnapshotPersistence.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoSetSnapshotPersistence.scala @@ -3,7 +3,7 @@ package org.ergoplatform.nodeView.state import org.ergoplatform.ErgoLikeContext.Height import org.ergoplatform.nodeView.state.UtxoState.{ManifestId, SubtreeId} import org.ergoplatform.settings.Algos.HF -import scorex.crypto.authds.avltree.batch.{PersistentBatchAVLProver, VersionedLDBAVLStorage} +import scorex.crypto.authds.avltree.batch.{PersistentBatchAVLProver, VersionedRocksDBAVLStorage} import scorex.crypto.hash.Digest32 import scorex.util.ScorexLogging import org.ergoplatform.settings.ErgoSettings @@ -27,7 +27,7 @@ trait UtxoSetSnapshotPersistence extends ScorexLogging { private[nodeView] def dumpSnapshot(height: Height, expectedRootHash: Array[Byte], manifestDepth: Byte = ManifestSerializer.MainnetManifestDepth): Try[Array[Byte]] = { - val storage = persistentProver.storage.asInstanceOf[VersionedLDBAVLStorage] + val storage = persistentProver.storage.asInstanceOf[VersionedRocksDBAVLStorage] snapshotsDb.writeSnapshot(storage, height, expectedRootHash, manifestDepth) } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala index a16f29265b..bc5372a3b2 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala @@ -20,7 +20,7 @@ import scorex.crypto.authds.avltree.batch._ import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, BatchAVLProverSubtree} import scorex.crypto.authds.{ADDigest, ADValue} import scorex.crypto.hash.Digest32 -import scorex.db.{ByteArrayWrapper, LDBVersionedStore} +import scorex.db.{ByteArrayWrapper, RocksDBVersionedStore} import scorex.util.ModifierId import scala.util.{Failure, Success, Try} @@ -35,7 +35,7 @@ import scala.util.{Failure, Success, Try} */ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32, HF], override val version: VersionTag, - override val store: LDBVersionedStore, + override val store: RocksDBVersionedStore, override protected val ergoSettings: ErgoSettings) extends ErgoState[UtxoState] with UtxoStateReader @@ -284,12 +284,12 @@ object UtxoState { * @return UTXO set based state on top of existing database, or genesis state if the database is empty */ def create(dir: File, settings: ErgoSettings): UtxoState = { - val store = new LDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) + val store = new RocksDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) val version = store.get(bestVersionKey).map(w => bytesToVersion(w)) .getOrElse(ErgoState.genesisStateVersion) val persistentProver: PersistentBatchAVLProver[Digest32, HF] = { val bp = new BatchAVLProver[Digest32, HF](keyLength = 32, valueLengthOpt = None) - val storage = new VersionedLDBAVLStorage(store) + val storage = new VersionedRocksDBAVLStorage(store) PersistentBatchAVLProver.create(bp, storage).get } new UtxoState(persistentProver, version, store, settings) @@ -309,10 +309,10 @@ object UtxoState { p.performOneOperation(Insert(b.id, ADValue @@ b.bytes)).ensuring(_.isSuccess) } - val store = new LDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) + val store = new RocksDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) val defaultStateContext = ErgoStateContext.empty(settings.chainSettings, parameters) - val storage = new VersionedLDBAVLStorage(store) + val storage = new VersionedRocksDBAVLStorage(store) val persistentProver = PersistentBatchAVLProver.create( p, storage, diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala index d891a6e30e..97ab31261d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala @@ -11,7 +11,7 @@ import org.ergoplatform.settings.Algos.HF import org.ergoplatform.wallet.boxes.ErgoBoxSerializer import org.ergoplatform.wallet.interpreter.ErgoInterpreter import org.ergoplatform.validation.MalformedModifierError -import scorex.crypto.authds.avltree.batch.{Lookup, PersistentBatchAVLProver, VersionedLDBAVLStorage} +import scorex.crypto.authds.avltree.batch.{Lookup, PersistentBatchAVLProver, VersionedRocksDBAVLStorage} import scorex.crypto.authds.{ADDigest, ADKey, SerializedAdProof} import scorex.crypto.hash.Digest32 @@ -31,7 +31,7 @@ trait UtxoStateReader extends ErgoStateReader with UtxoSetSnapshotPersistence { /** * Versioned database where UTXO set and its authenticating AVL+ tree are stored */ - protected lazy val storage = new VersionedLDBAVLStorage(store) + protected lazy val storage = new VersionedRocksDBAVLStorage(store) protected val persistentProver: PersistentBatchAVLProver[Digest32, HF] diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistry.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistry.scala index f3e7deba48..8389b9dbce 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistry.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistry.scala @@ -15,7 +15,7 @@ import org.ergoplatform.wallet.boxes.{TrackedBox, TrackedBoxSerializer} import org.ergoplatform.wallet.transactions.TransactionBuilder import org.ergoplatform.core.VersionTag import scorex.crypto.authds.ADKey -import scorex.db.LDBVersionedStore +import scorex.db.RocksDBVersionedStore import scorex.util.encode.Base16 import scorex.util.{ModifierId, ScorexLogging, bytesToId, idToBytes} @@ -31,7 +31,7 @@ import scala.util.{Failure, Success, Try} * * boxes, spent or not * */ -class WalletRegistry(private val store: LDBVersionedStore)(ws: WalletSettings) extends ScorexLogging { +class WalletRegistry(private val store: RocksDBVersionedStore)(ws: WalletSettings) extends ScorexLogging { import WalletRegistry._ @@ -454,7 +454,7 @@ object WalletRegistry { def apply(settings: ErgoSettings): Try[WalletRegistry] = Try { val dir = registryFolder(settings) dir.mkdirs() - new LDBVersionedStore(dir, settings.nodeSettings.keepVersions) + new RocksDBVersionedStore(dir, settings.nodeSettings.keepVersions) }.flatMap { case store if !store.versionIdExists(PreGenesisStateVersion) => // Create pre-genesis state checkpoint @@ -679,14 +679,14 @@ case class KeyValuePairsBag(toInsert: Seq[(Array[Byte], Array[Byte])], * Applies non-versioned transaction to a given `store`. * */ - def transact(store: LDBVersionedStore): Try[Unit] = transact(store, None) + def transact(store: RocksDBVersionedStore): Try[Unit] = transact(store, None) /** * Applies versioned transaction to a given `store`. */ - def transact(store: LDBVersionedStore, version: Array[Byte]): Try[Unit] = transact(store, Some(version)) + def transact(store: RocksDBVersionedStore, version: Array[Byte]): Try[Unit] = transact(store, Some(version)) - private def transact(store: LDBVersionedStore, versionOpt: Option[Array[Byte]]): Try[Unit] = + private def transact(store: RocksDBVersionedStore, versionOpt: Option[Array[Byte]]): Try[Unit] = if (toInsert.nonEmpty || toRemove.nonEmpty) { store.update(versionOpt.getOrElse(scorex.utils.Random.randomBytes()), toRemove, toInsert) } else { diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala index 09eb04bc2f..aa3bee9596 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala @@ -8,7 +8,7 @@ import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, DerivationPathSerial import org.ergoplatform.settings.{Constants, ErgoSettings, Parameters} import org.ergoplatform.wallet.Constants.{PaymentsScanId, ScanId} import scorex.crypto.hash.Blake2b256 -import scorex.db.{LDBFactory, LDBKVStore} +import scorex.db.{RocksDBFactory, RocksDBKVStore} import scorex.util.ScorexLogging import sigma.serialization.SigmaSerializer @@ -25,7 +25,7 @@ import scala.util.{Failure, Success, Try} * * ErgoStateContext (not version-agnostic, but state changes including rollbacks it is updated externally) * * external scans */ -final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends ScorexLogging { +final class WalletStorage(store: RocksDBKVStore, settings: ErgoSettings) extends ScorexLogging { import WalletStorage._ @@ -183,14 +183,8 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco * @return identifier of last inserted scan */ def lastUsedScanId: Short = { - // pre-3.3.7 method to get last used scan id, now useful to read pre-3.3.7 databases - def oldScanId: Option[Short] = - store.lastKeyInRange(SmallestPossibleScanId, BiggestPossibleScanId) - .map(bs => Shorts.fromByteArray(bs.takeRight(2))) - store.get(lastUsedScanIdKey) .map(bs => Shorts.fromByteArray(bs)) - .orElse(oldScanId) .getOrElse(PaymentsScanId) } @@ -251,7 +245,7 @@ object WalletStorage { def storageFolder(settings: ErgoSettings): File = new File(s"${settings.directory}/wallet/storage") def readOrCreate(settings: ErgoSettings): WalletStorage = { - val db = LDBFactory.createKvDb(storageFolder(settings).getPath) + val db = new RocksDBKVStore(RocksDBFactory.open(storageFolder(settings))) new WalletStorage(db, settings) } diff --git a/src/test/scala/org/ergoplatform/db/DBSpec.scala b/src/test/scala/org/ergoplatform/db/DBSpec.scala index 58f908be48..663057b8d5 100644 --- a/src/test/scala/org/ergoplatform/db/DBSpec.scala +++ b/src/test/scala/org/ergoplatform/db/DBSpec.scala @@ -2,12 +2,11 @@ package org.ergoplatform.db import akka.util.ByteString import org.ergoplatform.settings.Algos -import org.ergoplatform.wallet.utils.TestFileUtils -import org.iq80.leveldb.{DB, Options} -import scorex.db.LDBFactory.factory -import scorex.db.{LDBKVStore, LDBVersionedStore} +import org.ergoplatform.wallet.utils.FileUtils +import scorex.db.RocksDBFactory.RegisteredDB +import scorex.db.{RocksDBFactory, RocksDBKVStore, RocksDBVersionedStore} -trait DBSpec extends TestFileUtils { +trait DBSpec extends FileUtils { implicit class ValueOps(x: Option[Array[Byte]]) { def toBs: Option[ByteString] = x.map(ByteString.apply) @@ -21,22 +20,18 @@ trait DBSpec extends TestFileUtils { protected def byteString32(s: String): Array[Byte] = Algos.hash(byteString(s)) - protected def withDb[T](body: DB => T): T = { - val options = new Options() - options.createIfMissing(true) - options.verifyChecksums(true) - options.maxOpenFiles(2000) - val db = factory.open(createTempDir, options) + protected def withDb[T](body: RegisteredDB => T): T = { + val db = RocksDBFactory.open(createTempDir) try body(db) finally db.close() } protected def versionId(s: String): Array[Byte] = byteString32(s) - protected def withStore[T](body: LDBKVStore => T): T = - withDb { db: DB => body(new LDBKVStore(db)) } + protected def withStore[T](body: RocksDBKVStore => T): T = + withDb { db: RegisteredDB => body(new RocksDBKVStore(db)) } - protected def withVersionedStore[T](keepVersions: Int)(body: LDBVersionedStore => T): T = { - val db = new LDBVersionedStore(createTempDir, keepVersions) + protected def withVersionedStore[T](keepVersions: Int)(body: RocksDBVersionedStore => T): T = { + val db = new RocksDBVersionedStore(createTempDir, keepVersions) try body(db) finally db.close() } diff --git a/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala b/src/test/scala/org/ergoplatform/db/RocksDBKVStoreSpec.scala similarity index 51% rename from src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala rename to src/test/scala/org/ergoplatform/db/RocksDBKVStoreSpec.scala index cd32b21c29..eb2527f4aa 100644 --- a/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala +++ b/src/test/scala/org/ergoplatform/db/RocksDBKVStoreSpec.scala @@ -3,7 +3,7 @@ package org.ergoplatform.db import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec -class LDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { +class RocksDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { property("put/get/getAll/delete") { withStore { store => @@ -40,25 +40,4 @@ class LDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { } } - property("last key in range") { - withStore { store => - val valueA = (byteString("A"), byteString("1")) - val valueB = (byteString("B"), byteString("2")) - val valueC = (byteString("C"), byteString("1")) - val valueD = (byteString("D"), byteString("2")) - val valueE = (byteString("E"), byteString("3")) - val valueF = (byteString("F"), byteString("4")) - - val values = Array(valueA, valueB, valueC, valueD, valueE, valueF) - store.insert(values.map(_._1), values.map(_._2)).get - - store.lastKeyInRange(valueA._1, valueC._1).get.toSeq shouldBe valueC._1.toSeq - store.lastKeyInRange(valueD._1, valueF._1).get.toSeq shouldBe valueF._1.toSeq - store.lastKeyInRange(valueF._1, byteString32("Z")).get.toSeq shouldBe valueF._1.toSeq - store.lastKeyInRange(Array(10: Byte), valueA._1).get.toSeq shouldBe valueA._1.toSeq - - store.lastKeyInRange(Array(10: Byte), Array(11: Byte)).isDefined shouldBe false - } - } - } diff --git a/src/test/scala/org/ergoplatform/http/routes/EmissionApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/EmissionApiRouteSpec.scala index c48d4a6dbe..135d6cf8dd 100644 --- a/src/test/scala/org/ergoplatform/http/routes/EmissionApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/EmissionApiRouteSpec.scala @@ -10,12 +10,14 @@ import io.circe.syntax._ import org.ergoplatform.http.api.EmissionApiRoute import org.ergoplatform.mining.emission.EmissionRules import org.ergoplatform.settings.{ErgoSettings, ErgoSettingsReader, ReemissionSettings} +import org.ergoplatform.wallet.utils.FileUtils import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import scala.concurrent.duration._ class EmissionApiRouteSpec extends AnyFlatSpec + with FileUtils with Matchers with ScalatestRouteTest with FailFastCirceSupport { diff --git a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala index e1543727fc..ef10b02b36 100644 --- a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala +++ b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala @@ -20,6 +20,7 @@ import org.ergoplatform.nodeView.{ErgoNodeViewRef, ErgoReadersHolderRef} import org.ergoplatform.settings.{ErgoSettings, ErgoSettingsReader} import org.ergoplatform.utils.ErgoTestHelpers import org.ergoplatform.wallet.interpreter.ErgoInterpreter +import org.ergoplatform.wallet.utils.FileUtils import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoTreePredef, Input} import org.scalatest.concurrent.Eventually import org.scalatest.flatspec.AnyFlatSpec @@ -32,7 +33,7 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ import scala.language.postfixOps -class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with Eventually { +class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with FileUtils with Eventually { import org.ergoplatform.utils.ErgoNodeTestConstants._ import org.ergoplatform.utils.ErgoCoreTestConstants._ import org.ergoplatform.utils.generators.ValidBlocksGenerators._ diff --git a/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala b/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala index 1265b5631a..7a2cbe0511 100644 --- a/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala +++ b/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala @@ -14,7 +14,6 @@ import org.ergoplatform.nodeView.mempool.ErgoMemPool import org.ergoplatform.nodeView.state.UtxoState.ManifestId import org.ergoplatform.nodeView.state._ import org.ergoplatform.settings.Algos -import org.ergoplatform.wallet.utils.TestFileUtils import org.scalacheck.Gen import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec @@ -23,6 +22,7 @@ import scorex.core.network.NetworkController.ReceivableMessages.{PenalizePeer, S import org.ergoplatform.network.message._ import org.ergoplatform.network.peer.PenaltyType import org.ergoplatform.serialization.{ErgoSerializer, ManifestSerializer} +import org.ergoplatform.wallet.utils.FileUtils import scorex.testkit.generators.{SyntacticallyTargetedModifierProducer, TotallyValidModifierProducer} import scorex.testkit.utils.AkkaFixture import scorex.crypto.hash.Digest32 @@ -40,7 +40,7 @@ trait NodeViewSynchronizerTests[ST <: ErgoState[ST]] extends AnyPropSpec with ScorexLogging with SyntacticallyTargetedModifierProducer with TotallyValidModifierProducer[ST] - with TestFileUtils { + with FileUtils { import org.ergoplatform.utils.ErgoNodeTestConstants._ import org.ergoplatform.utils.generators.ChainGenerator._ diff --git a/src/test/scala/org/ergoplatform/nodeView/history/UtxoSetSnapshotProcessorSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/UtxoSetSnapshotProcessorSpecification.scala index 0904548824..d5b0b7a35f 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/UtxoSetSnapshotProcessorSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/UtxoSetSnapshotProcessorSpecification.scala @@ -8,12 +8,13 @@ import org.ergoplatform.settings.{Algos, ErgoSettings} import org.ergoplatform.utils.ErgoCorePropertyTest import org.ergoplatform.core.VersionTag import org.ergoplatform.serialization.{ManifestSerializer, SubtreeSerializer} -import scorex.db.LDBVersionedStore +import org.ergoplatform.wallet.utils.FileUtils +import scorex.db.RocksDBVersionedStore import scorex.util.ModifierId import scala.util.Random -class UtxoSetSnapshotProcessorSpecification extends ErgoCorePropertyTest { +class UtxoSetSnapshotProcessorSpecification extends ErgoCorePropertyTest with FileUtils { import org.ergoplatform.utils.HistoryTestHelpers.generateHistory import org.ergoplatform.utils.ErgoCoreTestConstants._ import org.ergoplatform.utils.ErgoNodeTestConstants._ @@ -85,7 +86,7 @@ class UtxoSetSnapshotProcessorSpecification extends ErgoCorePropertyTest { s shouldBe subtreeIdsEncoded val dir = createTempDir - val store = new LDBVersionedStore(dir, initialKeepVersions = 100) + val store = new RocksDBVersionedStore(dir, initialKeepVersions = 100) val restoredProver = utxoSetSnapshotProcessor.createPersistentProver(store, history, snapshotHeight, blockId).get bh.sortedBoxes.foreach { box => restoredProver.unauthenticatedLookup(box.id).isDefined shouldBe true diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerTestActor.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerTestActor.scala index e7fd56f122..fcf2360565 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerTestActor.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerTestActor.scala @@ -32,7 +32,6 @@ class ExtraIndexerTestActor(test: ExtraIndexerSpecification) extends ExtraIndexe def createDB(): Unit = { val dir: File = createTempDir - dir.mkdirs() val fullHistorySettings: ErgoSettings = ErgoSettings(dir.getAbsolutePath, NetworkType.TestNet, test.initSettings.chainSettings, nodeSettings, test.initSettings.scorexSettings, test.initSettings.walletSettings, test.initSettings.cacheSettings) diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/SegmentSpec.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/SegmentSpec.scala index 4ab6dd4e82..069b8a5a6d 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/extra/SegmentSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/SegmentSpec.scala @@ -12,6 +12,7 @@ import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.settings.ErgoSettings +import scala.reflect.ClassTag import scala.util.Try class SegmentSpec extends ErgoCorePropertyTest { @@ -31,40 +32,23 @@ class SegmentSpec extends ErgoCorePropertyTest { val ia = IndexedErgoAddress(hash, new ArrayBuffer[Long](), boxes.slice(segmentTreshold * 3, boxes.size).reverse) val hr = new ErgoHistoryReader { - override protected[history] val historyStorage: HistoryStorage = new HistoryStorage(null, null, null ,null) { - override def getExtraIndex(id: ModifierId): Option[ExtraIndex] = { - val b = Base16.decode(id).get.head - if(b == 0) { - Some(segment0) - } else if(b == 1) { - Some(segment1) - } else if(b == 2) { - Some(segment2) - } else { - None - } + override def typedExtraIndexById[T <: ExtraIndex : ClassTag](id: ModifierId): Option[T] = { + val b = Base16.decode(id).get.head + if(b == 0) { + Some(segment0.asInstanceOf[T]) + } else if(b == 1) { + Some(segment1.asInstanceOf[T]) + } else if(b == 2) { + Some(segment2.asInstanceOf[T]) + } else { + None } } - + override protected[history] val historyStorage: HistoryStorage = null override protected val settings: ErgoSettings = null - - /** - * Whether state requires to download adProofs before full block application - */ override protected def requireProofs: Boolean = ??? - - /** - * @param m - modifier to process - * @return ProgressInfo - info required for State to be consistent with History - */ override protected def process(m: NonHeaderBlockSection): Try[ProgressInfo[BlockSection]] = ??? - - /** - * @param m - modifier to validate - * @return Success() if modifier is valid from History point of view, Failure(error) otherwise - */ override protected def validate(m: NonHeaderBlockSection): Try[Unit] = ??? - override val powScheme: AutolykosPowScheme = null } diff --git a/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala index f136241f42..5a6f45d76c 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala @@ -6,10 +6,11 @@ import org.ergoplatform.modifiers.history.ADProofs import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.utils.{ErgoCorePropertyTest, RandomWrapper} import org.ergoplatform.core._ +import org.ergoplatform.wallet.utils.FileUtils import scorex.crypto.authds.ADDigest import sigma.interpreter.ProverResult -class DigestStateSpecification extends ErgoCorePropertyTest { +class DigestStateSpecification extends ErgoCorePropertyTest with FileUtils { import org.ergoplatform.utils.ErgoNodeTestConstants._ import org.ergoplatform.utils.ErgoCoreTestConstants._ import org.ergoplatform.utils.generators.ErgoNodeTransactionGenerators._ diff --git a/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala index d61fd82fbb..bc887da747 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala @@ -11,12 +11,13 @@ import org.scalacheck.Gen import org.ergoplatform.core.bytesToVersion import org.ergoplatform.utils.generators.ErgoNodeTransactionGenerators.boxesHolderGen import org.ergoplatform.validation.ValidationResult.Valid +import org.ergoplatform.wallet.utils.FileUtils import scorex.db.ByteArrayWrapper import scala.collection.mutable import scala.util.{Failure, Try} -class ErgoStateSpecification extends ErgoCorePropertyTest { +class ErgoStateSpecification extends ErgoCorePropertyTest with FileUtils { import org.ergoplatform.utils.ErgoNodeTestConstants._ import org.ergoplatform.utils.ErgoCoreTestConstants._ import org.ergoplatform.utils.generators.ErgoCoreTransactionGenerators._ diff --git a/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsDbSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsDbSpecification.scala index 7100b15b69..91b1878f5c 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsDbSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsDbSpecification.scala @@ -1,15 +1,15 @@ package org.ergoplatform.nodeView.state import org.ergoplatform.utils.ErgoCorePropertyTest +import org.ergoplatform.wallet.utils.FileUtils import org.scalacheck.Gen import scorex.crypto.hash.Digest32 import scorex.util.{ModifierId, bytesToId, idToBytes} import scala.util.Random -class SnapshotsDbSpecification extends ErgoCorePropertyTest { +class SnapshotsDbSpecification extends ErgoCorePropertyTest with FileUtils { import org.ergoplatform.utils.generators.CoreObjectGenerators._ - import org.ergoplatform.utils.generators.ValidBlocksGenerators._ def seededDatabase(manifestIds: Seq[ModifierId]): (SnapshotsInfo, SnapshotsDb) = { val m = manifestIds.map { mid => diff --git a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala index 563387eea6..dbb1b08d57 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala @@ -13,13 +13,13 @@ import org.ergoplatform.core.{VersionTag, idToVersion} import org.ergoplatform.nodeView.LocallyGeneratedModifier import scorex.crypto.authds.avltree.batch._ import scorex.crypto.hash.Digest32 -import scorex.db.{ByteArrayWrapper, LDBVersionedStore} +import scorex.db.{ByteArrayWrapper, RocksDBVersionedStore} import scala.util.{Failure, Success, Try} class WrappedUtxoState(prover: PersistentBatchAVLProver[Digest32, HF], override val version: VersionTag, - store: LDBVersionedStore, + store: RocksDBVersionedStore, val versionedBoxHolder: VersionedInMemoryBoxHolder, settings: ErgoSettings) extends UtxoState(prover, version, store, settings) { diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala index e3c4c00638..c1973b9808 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala @@ -22,7 +22,7 @@ import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.mnemonic.Mnemonic import org.scalacheck.Gen import org.scalatest.BeforeAndAfterAll -import scorex.db.{LDBKVStore, LDBVersionedStore} +import scorex.db.{RocksDBKVStore, RocksDBVersionedStore} import scorex.util.encode.Base16 import sigma.ast.{ByteArrayConstant, ErgoTree, EvaluatedValue, FalseLeaf, SType, TrueLeaf} import sigma.Extensions.ArrayOps @@ -54,7 +54,7 @@ class ErgoWalletServiceSpec override def afterAll(): Unit = try super.afterAll() finally x.stop() - private def initialState(store: LDBKVStore, versionedStore: LDBVersionedStore, mempool: Option[ErgoMemPoolReader] = None) = { + private def initialState(store: RocksDBKVStore, versionedStore: RocksDBVersionedStore, mempool: Option[ErgoMemPoolReader] = None) = { ErgoWalletState( new WalletStorage(store, settings), secretStorageOpt = Option.empty, diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala index c9909854ca..60367c5198 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala @@ -9,7 +9,7 @@ import org.scalacheck.Gen import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks -import scorex.db.LDBKVStore +import scorex.db.RocksDBKVStore class WalletStorageSpec extends AnyFlatSpec @@ -21,7 +21,7 @@ class WalletStorageSpec import org.ergoplatform.wallet.utils.WalletGenerators._ it should "add and read derivation paths" in { - def addPath(store: LDBKVStore, storedPaths: Seq[DerivationPath], derivationPath: DerivationPath): Unit = { + def addPath(store: RocksDBKVStore, storedPaths: Seq[DerivationPath], derivationPath: DerivationPath): Unit = { val updatedPaths = (storedPaths :+ derivationPath).toSet val toInsert = Ints.toByteArray(updatedPaths.size) ++ updatedPaths .foldLeft(Array.empty[Byte]) { case (acc, path) => diff --git a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala index fc6adce7fd..563ae84b21 100644 --- a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala +++ b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala @@ -14,11 +14,6 @@ class ErgoSettingsSpecification extends ErgoCorePropertyTest { private val txCostLimit = initSettings.nodeSettings.maxTransactionCost private val txSizeLimit = initSettings.nodeSettings.maxTransactionSize - property("should keep data user home by default") { - val settings = ErgoSettingsReader.read() - settings.directory shouldBe System.getProperty("user.dir") + "/.ergo_test/data" - } - property("should read default settings") { val settings = ErgoSettingsReader.read() settings.nodeSettings shouldBe NodeConfigurationSettings( diff --git a/src/test/scala/org/ergoplatform/utils/NodeViewTestConfig.scala b/src/test/scala/org/ergoplatform/utils/NodeViewTestConfig.scala index 3b708d9d06..6f7e3b6f05 100644 --- a/src/test/scala/org/ergoplatform/utils/NodeViewTestConfig.scala +++ b/src/test/scala/org/ergoplatform/utils/NodeViewTestConfig.scala @@ -3,11 +3,12 @@ package org.ergoplatform.utils import org.ergoplatform.mining.DefaultFakePowScheme import org.ergoplatform.nodeView.state.StateType import org.ergoplatform.settings.{ErgoSettings, ErgoSettingsReader, NipopowSettings} +import org.ergoplatform.wallet.utils.FileUtils case class NodeViewTestConfig(stateType: StateType, verifyTransactions: Boolean, - popowBootstrap: Boolean) { + popowBootstrap: Boolean) extends FileUtils { def toSettings: ErgoSettings = { val defaultSettings = ErgoSettingsReader.read() diff --git a/src/test/scala/org/ergoplatform/utils/Stubs.scala b/src/test/scala/org/ergoplatform/utils/Stubs.scala index 1414789dae..fdef6d4eca 100644 --- a/src/test/scala/org/ergoplatform/utils/Stubs.scala +++ b/src/test/scala/org/ergoplatform/utils/Stubs.scala @@ -34,7 +34,7 @@ import org.ergoplatform.wallet.boxes.{ChainStatus, TrackedBox} import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.interpreter.ErgoProvingInterpreter import org.ergoplatform.wallet.mnemonic.Mnemonic -import org.ergoplatform.wallet.utils.TestFileUtils +import org.ergoplatform.wallet.utils.FileUtils import org.scalacheck.Gen import scorex.core.network.NetworkController.ReceivableMessages.GetConnectedPeers import scorex.crypto.authds.ADDigest @@ -48,7 +48,7 @@ import scala.collection.mutable import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} -trait Stubs extends ErgoTestHelpers with TestFileUtils { +trait Stubs extends ErgoTestHelpers with FileUtils { import org.ergoplatform.utils.ErgoNodeTestConstants._ import org.ergoplatform.utils.ErgoCoreTestConstants._ import org.ergoplatform.utils.generators.ChainGenerator._ diff --git a/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala b/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala index 5127620226..3ee6fe3d9b 100644 --- a/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala +++ b/src/test/scala/org/ergoplatform/utils/fixtures/NodeViewFixture.scala @@ -6,14 +6,14 @@ import org.ergoplatform.mining.emission.EmissionRules import org.ergoplatform.nodeView.ErgoNodeViewRef import org.ergoplatform.settings.{ErgoSettings, Parameters} import org.ergoplatform.utils.NodeViewTestContext -import org.ergoplatform.wallet.utils.TestFileUtils +import org.ergoplatform.wallet.utils.FileUtils import scala.concurrent.ExecutionContext /** This uses TestProbe to receive messages from actor. * To make TestProbe work `defaultSender` implicit should be imported */ -class NodeViewFixture(protoSettings: ErgoSettings, parameters: Parameters) extends NodeViewTestContext with TestFileUtils { self => +class NodeViewFixture(protoSettings: ErgoSettings, parameters: Parameters) extends NodeViewTestContext with FileUtils { self => implicit val actorSystem: ActorSystem = ActorSystem() implicit val executionContext: ExecutionContext = actorSystem.dispatchers.lookup("scorex.executionContext") diff --git a/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala index e09ef9c9cc..45f9cb1b5c 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala @@ -11,7 +11,7 @@ import org.ergoplatform.nodeView.state._ import org.ergoplatform.nodeView.state.wrapped.WrappedUtxoState import org.ergoplatform.settings.{Algos, Constants, ErgoSettings, Parameters} import org.ergoplatform.utils.{LoggingUtil, RandomLike, RandomWrapper} -import org.ergoplatform.wallet.utils.TestFileUtils +import org.ergoplatform.wallet.utils.FileUtils import org.scalatest.matchers.should.Matchers import org.ergoplatform.core.VersionTag import scorex.crypto.authds.avltree.batch.Remove @@ -25,7 +25,7 @@ import scala.collection.mutable import scala.util.{Failure, Random, Success} object ValidBlocksGenerators - extends TestkitHelpers with TestFileUtils with Matchers with ScorexLogging { + extends TestkitHelpers with FileUtils with Matchers with ScorexLogging { import org.ergoplatform.utils.ErgoNodeTestConstants._ import org.ergoplatform.utils.ErgoCoreTestConstants._ import org.ergoplatform.utils.generators.ErgoNodeTransactionGenerators._