Backport "HBASE-29158 Unknown checksum type code exception occurred while reading HFileBlock (#6740)" to branch-2.6

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java

@@ -1596,6 +1596,24 @@ public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL, boolean
       }
     }
 
+    /**
+     * Check that checksumType on {@code headerBuf} read from a block header seems reasonable,
+     * within the known value range.
+     * @return {@code true} if the headerBuf is safe to proceed, {@code false} otherwise.
+     */
+    private boolean checkCheckSumTypeOnHeaderBuf(ByteBuff headerBuf) {
+      if (headerBuf == null) {
+        return true;
+      }
+      byte b = headerBuf.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX);
+      for (ChecksumType t : ChecksumType.values()) {
+        if (t.getCode() == b) {
+          return true;
+        }
+      }
+      return false;
+    }
+
     /**
      * Check that {@code value} read from a block header seems reasonable, within a large margin of
      * error.
@@ -1751,6 +1769,21 @@ protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset,
         onDiskSizeWithHeader = getOnDiskSizeWithHeader(headerBuf, checksumSupport);
       }
 
+      // Inspect the header's checksumType for known valid values. If we don't find such a value,
+      // assume that the bytes read are corrupted.We will clear the cached value and roll back to
+      // HDFS checksum
+      if (!checkCheckSumTypeOnHeaderBuf(headerBuf)) {
+        if (verifyChecksum) {
+          invalidateNextBlockHeader();
+          span.addEvent("Falling back to HDFS checksumming.", attributesBuilder.build());
+          return null;
+        } else {
+          throw new IOException(
+            "Unknown checksum type code " + headerBuf.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX)
+              + "for file " + pathName + ", the headerBuf of HFileBlock may corrupted.");
+        }
+      }
+
       // The common case is that onDiskSizeWithHeader was produced by a read without checksum
       // validation, so give it a sanity check before trying to use it.
       if (!checkOnDiskSizeWithHeader(onDiskSizeWithHeader)) {
@@ -1894,6 +1927,17 @@ private boolean validateChecksum(long offset, ByteBuff data, int hdrSize) {
       if (!fileContext.isUseHBaseChecksum()) {
         return false;
       }
+
+      // If the checksumType of the read block header is incorrect, it indicates that the block is
+      // corrupted and can be directly rolled back to HDFS checksum verification
+      if (!checkCheckSumTypeOnHeaderBuf(data)) {
+        HFile.LOG.warn(
+          "HBase checksumType verification failed for file {} at offset {} filesize {}"
+            + " checksumType {}. Retrying read with HDFS checksums turned on...",
+          pathName, offset, fileSize, data.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX));
+        return false;
+      }
+
       return ChecksumUtil.validateChecksum(data, pathName, offset, hdrSize);
     }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockHeaderCorruption.java

@@ -23,6 +23,7 @@
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.startsWith;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -54,6 +55,7 @@
 import org.apache.hadoop.hbase.testclassification.IOTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
 import org.hamcrest.Description;
 import org.hamcrest.Matcher;
 import org.hamcrest.TypeSafeMatcher;
@@ -92,6 +94,141 @@ public TestHFileBlockHeaderCorruption() throws IOException {
     ruleChain = RuleChain.outerRule(testName).around(hFileTestRule);
   }
 
+  @Test
+  public void testChecksumTypeCorruptionFirstBlock() throws Exception {
+    HFileBlockChannelPosition firstBlock = null;
+    try {
+      try (HFileBlockChannelPositionIterator it =
+        new HFileBlockChannelPositionIterator(hFileTestRule)) {
+        assertTrue(it.hasNext());
+        firstBlock = it.next();
+      }
+
+      Corrupter c = new Corrupter(firstBlock);
+
+      logHeader(firstBlock);
+
+      // test corrupted HFileBlock with unknown checksumType code -1
+      c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new byte[] { -1 }));
+      logHeader(firstBlock);
+      try (HFileBlockChannelPositionIterator it =
+        new HFileBlockChannelPositionIterator(hFileTestRule)) {
+        CountingConsumer consumer = new CountingConsumer(it);
+        try {
+          consumer.readFully();
+          fail();
+        } catch (Exception e) {
+          assertThat(e, new IsThrowableMatching().withInstanceOf(IOException.class)
+            .withMessage(startsWith("Unknown checksum type code")));
+        }
+        assertEquals(0, consumer.getItemsRead());
+      }
+
+      // valid checksumType code test
+      for (ChecksumType t : ChecksumType.values()) {
+        testValidChecksumTypeReadBlock(t.getCode(), c, firstBlock);
+      }
+
+      c.restore();
+      // test corrupted HFileBlock with unknown checksumType code 3
+      c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new byte[] { 3 }));
+      logHeader(firstBlock);
+      try (HFileBlockChannelPositionIterator it =
+        new HFileBlockChannelPositionIterator(hFileTestRule)) {
+        CountingConsumer consumer = new CountingConsumer(it);
+        try {
+          consumer.readFully();
+          fail();
+        } catch (Exception e) {
+          assertThat(e, new IsThrowableMatching().withInstanceOf(IOException.class)
+            .withMessage(startsWith("Unknown checksum type code")));
+        }
+        assertEquals(0, consumer.getItemsRead());
+      }
+    } finally {
+      if (firstBlock != null) {
+        firstBlock.close();
+      }
+    }
+  }
+
+  @Test
+  public void testChecksumTypeCorruptionSecondBlock() throws Exception {
+    HFileBlockChannelPosition secondBlock = null;
+    try {
+      try (HFileBlockChannelPositionIterator it =
+        new HFileBlockChannelPositionIterator(hFileTestRule)) {
+        assertTrue(it.hasNext());
+        it.next();
+        assertTrue(it.hasNext());
+        secondBlock = it.next();
+      }
+
+      Corrupter c = new Corrupter(secondBlock);
+
+      logHeader(secondBlock);
+      // test corrupted HFileBlock with unknown checksumType code -1
+      c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new byte[] { -1 }));
+      logHeader(secondBlock);
+      try (HFileBlockChannelPositionIterator it =
+        new HFileBlockChannelPositionIterator(hFileTestRule)) {
+        CountingConsumer consumer = new CountingConsumer(it);
+        try {
+          consumer.readFully();
+          fail();
+        } catch (Exception e) {
+          assertThat(e, new IsThrowableMatching().withInstanceOf(RuntimeException.class)
+            .withMessage(startsWith("Unknown checksum type code")));
+        }
+        assertEquals(1, consumer.getItemsRead());
+      }
+
+      // valid checksumType code test
+      for (ChecksumType t : ChecksumType.values()) {
+        testValidChecksumTypeReadBlock(t.getCode(), c, secondBlock);
+      }
+
+      c.restore();
+      // test corrupted HFileBlock with unknown checksumType code 3
+      c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new byte[] { 3 }));
+      logHeader(secondBlock);
+      try (HFileBlockChannelPositionIterator it =
+        new HFileBlockChannelPositionIterator(hFileTestRule)) {
+        CountingConsumer consumer = new CountingConsumer(it);
+        try {
+          consumer.readFully();
+          fail();
+        } catch (Exception e) {
+          assertThat(e, new IsThrowableMatching().withInstanceOf(RuntimeException.class)
+            .withMessage(startsWith("Unknown checksum type code")));
+        }
+        assertEquals(1, consumer.getItemsRead());
+      }
+    } finally {
+      if (secondBlock != null) {
+        secondBlock.close();
+      }
+    }
+  }
+
+  public void testValidChecksumTypeReadBlock(byte checksumTypeCode, Corrupter c,
+    HFileBlockChannelPosition testBlock) throws IOException {
+    c.restore();
+    c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX,
+      ByteBuffer.wrap(new byte[] { checksumTypeCode }));
+    logHeader(testBlock);
+    try (
+      HFileBlockChannelPositionIterator it = new HFileBlockChannelPositionIterator(hFileTestRule)) {
+      CountingConsumer consumer = new CountingConsumer(it);
+      try {
+        consumer.readFully();
+      } catch (Exception e) {
+        fail("test fail: valid checksumType are not executing properly");
+      }
+      assertNotEquals(0, consumer.getItemsRead());
+    }
+  }
+
   @Test
   public void testOnDiskSizeWithoutHeaderCorruptionFirstBlock() throws Exception {
     HFileBlockChannelPosition firstBlock = null;
@@ -330,7 +467,8 @@ public HFileBlockChannelPositionIterator(HFileTestRule hFileTestRule) throws IOE
       try {
         reader = HFile.createReader(hfs, hfsPath, CacheConfig.DISABLED, true, conf);
         HFileBlock.FSReader fsreader = reader.getUncachedBlockReader();
-        iter = fsreader.blockRange(0, hfs.getFileStatus(hfsPath).getLen());
+        // The read block offset cannot out of the range:0,loadOnOpenDataOffset
+        iter = fsreader.blockRange(0, reader.getTrailer().getLoadOnOpenDataOffset());
       } catch (IOException e) {
         if (reader != null) {
           closeQuietly(reader::close);