From f4441966592636253fd5ab0bb9ed44fc2697fc53 Mon Sep 17 00:00:00 2001 From: anand76 Date: Mon, 11 Mar 2024 11:26:24 -0700 Subject: [PATCH] Add a FS flag to detect and correct corruption (#12408) Summary: Add a flag in `IOOptions` to request the file system to make best efforts to detect data corruption and reconstruct the data if possible. This will be used by RocksDB to retry a read if the previous read returns corrupt data (checksum mismatch). Add a new op to `FSSupportedOps` that, if supported, will trigger this behavior in RocksDB. Pull Request resolved: https://github.com/facebook/rocksdb/pull/12408 Reviewed By: akankshamahajan15 Differential Revision: D54564218 Pulled By: anand1976 fbshipit-source-id: bc401dcd22a7d320bf63b5183c41714acdce39f5 --- include/rocksdb/file_system.h | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/include/rocksdb/file_system.h b/include/rocksdb/file_system.h index 6c4fee57482..c1d9b87add7 100644 --- a/include/rocksdb/file_system.h +++ b/include/rocksdb/file_system.h @@ -81,7 +81,14 @@ enum class IOType : uint8_t { // enum representing various operations supported by underlying FileSystem. // These need to be set in SupportedOps API for RocksDB to use them. -enum FSSupportedOps { kAsyncIO, kFSBuffer }; +enum FSSupportedOps { + kAsyncIO, // Supports async reads + kFSBuffer, // Supports handing off the file system allocated read buffer + // to the caller of Read/MultiRead + kVerifyAndReconstructRead, // Supports a higher level of data integrity. See + // the verify_and_reconstruct_read flag in + // IOOptions. +}; // Per-request options that can be passed down to the FileSystem // implementation. These are hints and are not necessarily guaranteed to be @@ -120,6 +127,18 @@ struct IOOptions { // directories and list only files in GetChildren API. bool do_not_recurse; + // Setting this flag indicates a corruption was detected by a previous read, + // so the caller wants to re-read the data with much stronger data integrity + // checking and correction, i.e requests the file system to reconstruct the + // data from redundant copies and verify checksums, if available, in order + // to have a better chance of success. It is expected that this will have a + // much higher overhead than a normal read. + // This is a hint. At a minimum, the file system should implement this flag in + // FSRandomAccessFile::Read and FSSequentialFile::Read + // NOTE: The file system must support kVerifyAndReconstructRead in + // FSSupportedOps, otherwise this feature will not be used. + bool verify_and_reconstruct_read; + // EXPERIMENTAL Env::IOActivity io_activity = Env::IOActivity::kUnknown;