feat: Key Prefix Encoding + Decoding

Author: Leibnizhu

src/main/scala/io/github/leibnizhu/tinylsm/Block.scala

@@ -34,6 +34,14 @@ class Block(val data: Array[Byte], val offsets: Array[Int]) {
     buffer.appendAll(intLow2Bytes(offsets.length))
     buffer.toArray
   }
+
+  def getFirstKey(): MemTableKey = {
+    val buffer = ByteArrayReader(data)
+    val overlapLen = buffer.readUint16()
+    assert(overlapLen == 0)
+    val keyLen = buffer.readUint16()
+    buffer.readBytes(keyLen)
+  }
 }
 
 object Block {
@@ -80,10 +88,16 @@ class BlockBuilder(val blockSize: Int) {
     // 显然，新数据的offset就是当前data长度
     offsets += data.length
 
-    // key的长度
-    data.appendAll(intLow2Bytes(key.length))
+    // overlap 格式
+    // key_overlap_len (u16) | rest_key_len (u16) | key (rest_key_len)
+    // 当前key与firstKey的共同前缀byte数量
+    val overlap = commonPrefix(key)
+    // key_overlap_len
+    data.appendAll(intLow2Bytes(overlap))
+    //  rest_key_len (u16)
+    data.appendAll(intLow2Bytes(key.length - overlap))
     // key内容
-    data.appendAll(key)
+    data.appendAll(key.slice(overlap, key.length))
     // value的长度
     data.appendAll(intLow2Bytes(value.length))
     // value内容
@@ -95,6 +109,20 @@ class BlockBuilder(val blockSize: Int) {
     true
   }
 
+  /**
+   * @param key 指定key
+   * @return 指定key与 firstKey 有多少个相同的前缀byte
+   */
+  private def commonPrefix(key: MemTableKey): Int = {
+    if (firstKey.isEmpty) {
+      return 0
+    }
+    var index = 0
+    while (index < firstKey.get.length && index < key.length && firstKey.get(index) == key(index)) {
+      index += 1
+    }
+    index
+  }
 
   /**
    * @return 按data和offsets估算的体积
@@ -138,6 +166,7 @@ class BlockIterator(block: Block) extends MemTableStorageIterator {
    * 当前value在Block中data的下标
    */
   private var curValuePos: (Int, Int) = (0, 0)
+  private val firstKey = block.getFirstKey()
 
   def seekToFirst(): Unit = {
     seekToIndex(0)
@@ -211,10 +240,12 @@ class BlockIterator(block: Block) extends MemTableStorageIterator {
 
     // 根据 offset 段获取entry位置
     val entryOffset = block.offsets(index)
-    // 先后读取key长度、key、value长度
-    val keyLength = low2BytesToInt(block.data(entryOffset), block.data(entryOffset + 1))
-    curKey = Some(block.data.slice(entryOffset + 2, entryOffset + 2 + keyLength))
-    val valueOffset = entryOffset + 2 + keyLength
+    // 先后读取overlap长度、剩余key长度、剩余key、value长度
+    val overlapLength = low2BytesToInt(block.data(entryOffset), block.data(entryOffset + 1))
+    val restKeyLength = low2BytesToInt(block.data(entryOffset + 2), block.data(entryOffset + 3))
+    curKey = Some(firstKey.slice(0, overlapLength) ++
+      block.data.slice(entryOffset + 4, entryOffset + 4 + restKeyLength))
+    val valueOffset = entryOffset + 4 + restKeyLength
     val valueLength = low2BytesToInt(block.data(valueOffset), block.data(valueOffset + 1))
     curValuePos = (valueOffset + 2, valueOffset + 2 + valueLength)
     this.index = index

src/test/scala/io/github/leibnizhu/tinylsm/LsmStorageTest.scala

@@ -291,7 +291,7 @@ class LsmStorageTest extends AnyFunSuite {
     for (i <- 0 until 6000) {
       storage.put(i.toString, value)
     }
-    Thread.sleep(500)
+    Thread.sleep(1000)
     storage.inner.state.read(st => assert(st.l0SsTables.nonEmpty))
     storage.close()
   }

src/test/scala/io/github/leibnizhu/tinylsm/SsTableTest.scala

@@ -78,8 +78,8 @@ class SsTableTest extends AnyFunSuite {
   test("week1_day4_task2_sst_iterator") {
     val sst = generateSst()
     val iterator = SsTableIterator.createAndSeekToFirst(sst)
-    for(_ <- 0 until 5) {
-      for(i <- 0 until keyNum) {
+    for (_ <- 0 until 5) {
+      for (i <- 0 until keyNum) {
         val key = iterator.key()
         val value = iterator.value()
         assertResult(keyOf(i))(new String(key))
@@ -93,8 +93,8 @@ class SsTableTest extends AnyFunSuite {
   test("week1_day4_task1_sst_seek_key") {
     val sst = generateSst()
     val iterator = SsTableIterator.createAndSeekToKey(sst, keyOf(0).getBytes)
-    for(offset <- 1 to 5) {
-      for(i <- 0 until keyNum) {
+    for (offset <- 1 to 5) {
+      for (i <- 0 until keyNum) {
         val key = iterator.key()
         val value = iterator.value()
         assertResult(keyOf(i))(new String(key))
@@ -104,7 +104,7 @@ class SsTableTest extends AnyFunSuite {
       iterator.seekToKey("k".getBytes)
     }
   }
-  
+
   test("week1_day7_task2_sst_decode") {
     val sst1 = generateSst()
     val sst2 = SsTable.open(0, None, FileObject.open(sst1.file.file.get))
@@ -113,4 +113,9 @@ class SsTableTest extends AnyFunSuite {
     assertResult(bloom1.hashFuncNum)(bloom2.hashFuncNum)
     assertResult(bloom1.filter)(bloom2.filter)
   }
+
+  test("week1_day7_task3_block_key_compression") {
+    val sst = generateSst()
+    assert(sst.blockMeta.length <= 25)
+  }
 }