Skip to content

Commit db00942

Browse files
kewang1024kewang1024
committed
Add fault injection for write operation
1 parent d796cfc commit db00942

File tree

4 files changed

+151
-34
lines changed

4 files changed

+151
-34
lines changed

‎velox/common/file/tests/FaultyFile.cpp

+11-1
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,24 @@ uint64_t FaultyReadFile::preadv(
5454
}
5555

5656
FaultyWriteFile::FaultyWriteFile(
57+
const std::string& path,
5758
std::shared_ptr<WriteFile> delegatedFile,
5859
FileFaultInjectionHook injectionHook)
59-
: delegatedFile_(std::move(delegatedFile)),
60+
: path_(path),
61+
delegatedFile_(std::move(delegatedFile)),
6062
injectionHook_(std::move(injectionHook)) {
6163
VELOX_CHECK_NOT_NULL(delegatedFile_);
6264
}
6365

6466
void FaultyWriteFile::append(std::string_view data) {
67+
if (injectionHook_ != nullptr) {
68+
FaultFileWriteOperation op(path_, data);
69+
injectionHook_(&op);
70+
if (op.delegate) {
71+
delegatedFile_->append(op.data);
72+
}
73+
return;
74+
}
6575
delegatedFile_->append(data);
6676
}
6777

‎velox/common/file/tests/FaultyFile.h

+14
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ struct FaultFileOperation {
2626
/// Injects faults for file read operations.
2727
kRead,
2828
kReadv,
29+
kWrite,
2930
/// TODO: add to support fault injections for the other file operation
3031
/// types.
3132
};
@@ -77,6 +78,17 @@ struct FaultFileReadvOperation : FaultFileOperation {
7778
buffers(_buffers) {}
7879
};
7980

81+
/// Fault injection parameters for file write API.
82+
struct FaultFileWriteOperation : FaultFileOperation {
83+
std::string_view data;
84+
85+
FaultFileWriteOperation(
86+
const std::string& _path,
87+
const std::string_view& _data)
88+
: FaultFileOperation(FaultFileOperation::Type::kWrite, _path),
89+
data(_data) {}
90+
};
91+
8092
/// The fault injection hook on the file operation path.
8193
using FileFaultInjectionHook = std::function<void(FaultFileOperation*)>;
8294

@@ -125,6 +137,7 @@ class FaultyReadFile : public ReadFile {
125137
class FaultyWriteFile : public WriteFile {
126138
public:
127139
FaultyWriteFile(
140+
const std::string& path,
128141
std::shared_ptr<WriteFile> delegatedFile,
129142
FileFaultInjectionHook injectionHook);
130143

@@ -143,6 +156,7 @@ class FaultyWriteFile : public WriteFile {
143156
}
144157

145158
private:
159+
const std::string path_;
146160
const std::shared_ptr<WriteFile> delegatedFile_;
147161
const FileFaultInjectionHook injectionHook_;
148162
};

‎velox/common/file/tests/FaultyFileSystem.cpp

+3-2
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,9 @@ std::unique_ptr<WriteFile> FaultyFileSystem::openFileForWrite(
7979
auto delegatedFile = getFileSystem(delegatedPath, config_)
8080
->openFileForWrite(delegatedPath, options);
8181
return std::make_unique<FaultyWriteFile>(
82-
std::move(delegatedFile),
83-
[&](FaultFileOperation* op) { maybeInjectFileFault(op); });
82+
std::string(path), std::move(delegatedFile), [&](FaultFileOperation* op) {
83+
maybeInjectFileFault(op);
84+
});
8485
}
8586

8687
void FaultyFileSystem::remove(std::string_view path) {

‎velox/common/file/tests/FileTest.cpp

+123-31
Original file line numberDiff line numberDiff line change
@@ -366,17 +366,18 @@ class FaultyFsTest : public ::testing::Test {
366366
fs_ = std::dynamic_pointer_cast<tests::utils::FaultyFileSystem>(
367367
filesystems::getFileSystem(dir_->getPath(), {}));
368368
VELOX_CHECK_NOT_NULL(fs_);
369-
filePath_ = fmt::format("{}/faultyTestFile", dir_->getPath());
369+
readFilePath_ = fmt::format("{}/faultyTestReadFile", dir_->getPath());
370+
writeFilePath_ = fmt::format("{}/faultyTestWriteFile", dir_->getPath());
370371
const int bufSize = 1024;
371372
buffer_.resize(bufSize);
372373
for (int i = 0; i < bufSize; ++i) {
373374
buffer_[i] = i % 256;
374375
}
375376
{
376-
auto writeFile = fs_->openFileForWrite(filePath_, {});
377+
auto writeFile = fs_->openFileForWrite(readFilePath_, {});
377378
writeData(writeFile.get());
378379
}
379-
auto readFile = fs_->openFileForRead(filePath_, {});
380+
auto readFile = fs_->openFileForRead(readFilePath_, {});
380381
readData(readFile.get(), true);
381382
try {
382383
VELOX_FAIL("InjectedFaultFileError");
@@ -411,7 +412,8 @@ class FaultyFsTest : public ::testing::Test {
411412
}
412413

413414
std::shared_ptr<exec::test::TempDirectoryPath> dir_;
414-
std::string filePath_;
415+
std::string readFilePath_;
416+
std::string writeFilePath_;
415417
std::shared_ptr<tests::utils::FaultyFileSystem> fs_;
416418
std::string buffer_;
417419
std::exception_ptr fileError_;
@@ -421,46 +423,43 @@ TEST_F(FaultyFsTest, fileReadErrorInjection) {
421423
// Set read error.
422424
fs_->setFileInjectionError(fileError_, {FaultFileOperation::Type::kRead});
423425
{
424-
auto readFile = fs_->openFileForRead(filePath_, {});
426+
auto readFile = fs_->openFileForRead(readFilePath_, {});
425427
VELOX_ASSERT_THROW(
426428
readData(readFile.get(), false), "InjectedFaultFileError");
427429
}
428430
{
429-
auto readFile = fs_->openFileForRead(filePath_, {});
431+
auto readFile = fs_->openFileForRead(readFilePath_, {});
430432
// We only inject error for pread API so preadv should be fine.
431433
readData(readFile.get(), true);
432434
}
433435

434436
// Set readv error
435437
fs_->setFileInjectionError(fileError_, {FaultFileOperation::Type::kReadv});
436438
{
437-
auto readFile = fs_->openFileForRead(filePath_, {});
439+
auto readFile = fs_->openFileForRead(readFilePath_, {});
438440
VELOX_ASSERT_THROW(
439441
readData(readFile.get(), true), "InjectedFaultFileError");
440442
}
441443
{
442-
auto readFile = fs_->openFileForRead(filePath_, {});
443-
// We only inject error for pread API so preadv should be fine.
444+
auto readFile = fs_->openFileForRead(readFilePath_, {});
445+
// We only inject error for preadv API so pread should be fine.
444446
readData(readFile.get(), false);
445447
}
446448

447449
// Set error for all kinds of operations.
448450
fs_->setFileInjectionError(fileError_);
449-
auto readFile = fs_->openFileForRead(filePath_, {});
451+
auto readFile = fs_->openFileForRead(readFilePath_, {});
450452
VELOX_ASSERT_THROW(readData(readFile.get(), true), "InjectedFaultFileError");
451453
VELOX_ASSERT_THROW(readData(readFile.get(), false), "InjectedFaultFileError");
452-
fs_->remove(filePath_);
453-
// Check there is no interference on write as we don't support it for now.
454-
auto writeFile = fs_->openFileForWrite(filePath_, {});
455-
writeData(writeFile.get());
454+
fs_->remove(readFilePath_);
456455
}
457456

458457
TEST_F(FaultyFsTest, fileReadDelayInjection) {
459-
// Set 3 seconds delay.
458+
// Set 2 seconds delay.
460459
const uint64_t injectDelay{2'000'000};
461460
fs_->setFileInjectionDelay(injectDelay, {FaultFileOperation::Type::kRead});
462461
{
463-
auto readFile = fs_->openFileForRead(filePath_, {});
462+
auto readFile = fs_->openFileForRead(readFilePath_, {});
464463
uint64_t readDurationUs{0};
465464
{
466465
MicrosecondTimer readTimer(&readDurationUs);
@@ -469,7 +468,7 @@ TEST_F(FaultyFsTest, fileReadDelayInjection) {
469468
ASSERT_GE(readDurationUs, injectDelay);
470469
}
471470
{
472-
auto readFile = fs_->openFileForRead(filePath_, {});
471+
auto readFile = fs_->openFileForRead(readFilePath_, {});
473472
// We only inject error for pread API so preadv should be fine.
474473
uint64_t readDurationUs{0};
475474
{
@@ -482,7 +481,7 @@ TEST_F(FaultyFsTest, fileReadDelayInjection) {
482481
// Set readv error
483482
fs_->setFileInjectionDelay(injectDelay, {FaultFileOperation::Type::kReadv});
484483
{
485-
auto readFile = fs_->openFileForRead(filePath_, {});
484+
auto readFile = fs_->openFileForRead(readFilePath_, {});
486485
uint64_t readDurationUs{0};
487486
{
488487
MicrosecondTimer readTimer(&readDurationUs);
@@ -491,7 +490,7 @@ TEST_F(FaultyFsTest, fileReadDelayInjection) {
491490
ASSERT_GE(readDurationUs, injectDelay);
492491
}
493492
{
494-
auto readFile = fs_->openFileForRead(filePath_, {});
493+
auto readFile = fs_->openFileForRead(readFilePath_, {});
495494
// We only inject error for pread API so preadv should be fine.
496495
uint64_t readDurationUs{0};
497496
{
@@ -504,7 +503,7 @@ TEST_F(FaultyFsTest, fileReadDelayInjection) {
504503
// Set error for all kinds of operations.
505504
fs_->setFileInjectionDelay(injectDelay);
506505
{
507-
auto readFile = fs_->openFileForRead(filePath_, {});
506+
auto readFile = fs_->openFileForRead(readFilePath_, {});
508507
// We only inject error for pread API so preadv should be fine.
509508
uint64_t readDurationUs{0};
510509
{
@@ -514,7 +513,7 @@ TEST_F(FaultyFsTest, fileReadDelayInjection) {
514513
ASSERT_GE(readDurationUs, injectDelay);
515514
}
516515
{
517-
auto readFile = fs_->openFileForRead(filePath_, {});
516+
auto readFile = fs_->openFileForRead(readFilePath_, {});
518517
// We only inject error for pread API so preadv should be fine.
519518
uint64_t readDurationUs{0};
520519
{
@@ -523,16 +522,6 @@ TEST_F(FaultyFsTest, fileReadDelayInjection) {
523522
}
524523
ASSERT_GE(readDurationUs, injectDelay);
525524
}
526-
527-
fs_->remove(filePath_);
528-
// Check there is no interference on write as we don't support it for now.
529-
auto writeFile = fs_->openFileForWrite(filePath_, {});
530-
uint64_t writeDurationUs{0};
531-
{
532-
MicrosecondTimer writeTimer(&writeDurationUs);
533-
writeData(writeFile.get());
534-
}
535-
ASSERT_LT(writeDurationUs, injectDelay);
536525
}
537526

538527
TEST_F(FaultyFsTest, fileReadFaultHookInjection) {
@@ -605,3 +594,106 @@ TEST_F(FaultyFsTest, fileReadFaultHookInjection) {
605594
VELOX_ASSERT_THROW(readData(readFile.get(), false), "Data Mismatch");
606595
}
607596
}
597+
598+
TEST_F(FaultyFsTest, fileWriteErrorInjection) {
599+
// Set write error.
600+
fs_->setFileInjectionError(fileError_, {FaultFileOperation::Type::kWrite});
601+
{
602+
auto writeFile = fs_->openFileForWrite(writeFilePath_, {});
603+
VELOX_ASSERT_THROW(writeFile->append("hello"), "InjectedFaultFileError");
604+
fs_->remove(writeFilePath_);
605+
}
606+
// Set error for all kinds of operations.
607+
fs_->setFileInjectionError(fileError_);
608+
{
609+
auto writeFile = fs_->openFileForWrite(writeFilePath_, {});
610+
VELOX_ASSERT_THROW(writeFile->append("hello"), "InjectedFaultFileError");
611+
fs_->remove(writeFilePath_);
612+
}
613+
}
614+
615+
TEST_F(FaultyFsTest, fileWriteDelayInjection) {
616+
// Set 2 seconds delay.
617+
const uint64_t injectDelay{2'000'000};
618+
fs_->setFileInjectionDelay(injectDelay, {FaultFileOperation::Type::kWrite});
619+
{
620+
auto writeFile = fs_->openFileForWrite(writeFilePath_, {});
621+
uint64_t readDurationUs{0};
622+
{
623+
MicrosecondTimer readTimer(&readDurationUs);
624+
writeFile->append("hello");
625+
}
626+
ASSERT_GE(readDurationUs, injectDelay);
627+
fs_->remove(writeFilePath_);
628+
}
629+
}
630+
631+
TEST_F(FaultyFsTest, fileWriteFaultHookInjection) {
632+
const std::string path1 = fmt::format("{}/hookFile1", dir_->getPath());
633+
const std::string path2 = fmt::format("{}/hookFile2", dir_->getPath());
634+
// Set to write fake data.
635+
fs_->setFileInjectionHook([&](FaultFileOperation* op) {
636+
// Only inject for write.
637+
if (op->type != FaultFileOperation::Type::kWrite) {
638+
return;
639+
}
640+
// Only inject for path2.
641+
if (op->path != path2) {
642+
return;
643+
}
644+
auto* writeOp = static_cast<FaultFileWriteOperation*>(op);
645+
writeOp->data = "Error data";
646+
});
647+
{
648+
auto writeFile = fs_->openFileForWrite(path1, {});
649+
writeFile->append("hello");
650+
writeFile->close();
651+
auto readFile = fs_->openFileForRead(path1, {});
652+
char buffer[5];
653+
ASSERT_EQ(readFile->size(), 5);
654+
ASSERT_EQ(readFile->pread(0, 5, &buffer), "hello");
655+
fs_->remove(path1);
656+
}
657+
{
658+
auto writeFile = fs_->openFileForWrite(path2, {});
659+
writeFile->append("hello");
660+
writeFile->close();
661+
auto readFile = fs_->openFileForRead(path2, {});
662+
char buffer[10];
663+
ASSERT_EQ(readFile->size(), 10);
664+
ASSERT_EQ(readFile->pread(0, 10, &buffer), "Error data");
665+
fs_->remove(path2);
666+
}
667+
668+
// Set to not delegate.
669+
fs_->setFileInjectionHook([&](FaultFileOperation* op) {
670+
// Only inject for write.
671+
if (op->type != FaultFileOperation::Type::kWrite) {
672+
return;
673+
}
674+
// Only inject for path2.
675+
if (op->path != path2) {
676+
return;
677+
}
678+
auto* writeOp = static_cast<FaultFileWriteOperation*>(op);
679+
writeOp->delegate = false;
680+
});
681+
{
682+
auto writeFile = fs_->openFileForWrite(path1, {});
683+
writeFile->append("hello");
684+
writeFile->close();
685+
auto readFile = fs_->openFileForRead(path1, {});
686+
char buffer[5];
687+
ASSERT_EQ(readFile->size(), 5);
688+
ASSERT_EQ(readFile->pread(0, 5, &buffer), "hello");
689+
fs_->remove(path1);
690+
}
691+
{
692+
auto writeFile = fs_->openFileForWrite(path2, {});
693+
writeFile->append("hello");
694+
writeFile->close();
695+
auto readFile = fs_->openFileForRead(path2, {});
696+
ASSERT_EQ(readFile->size(), 0);
697+
fs_->remove(path2);
698+
}
699+
}

0 commit comments

Comments
 (0)