From cd916c6a7cf6b7df2fd910bc6f5d123416715b36 Mon Sep 17 00:00:00 2001 From: Siyang Tang Date: Mon, 23 Dec 2024 12:25:46 +0800 Subject: [PATCH] [enhancement](tablet-meta) Avoid be coredump due to potential race condition when updating tablet cumu point (#45643) Currently, when setting tablet's cumu point, aseert fail will happend if new point is less than local value, resulting BE coredump. This could happend when race condition happend: 1. thread A try to sync rowset 2. thread A fetch cumu point from ms 3. thread B update cumu point(like sc/compaction),commit to ms after 2. and set be tablet cumu point before 4. 4. thread A try to set cumu point seen before and meet the assertion, coredump. --- be/src/cloud/cloud_tablet.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index c7d3170726b2d5..4e351f7cfa5110 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -33,6 +33,7 @@ #include "cloud/cloud_meta_mgr.h" #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet_mgr.h" +#include "common/logging.h" #include "io/cache/block_file_cache_downloader.h" #include "io/cache/block_file_cache_factory.h" #include "olap/cumulative_compaction_time_series_policy.h" @@ -657,11 +658,14 @@ void CloudTablet::get_compaction_status(std::string* json_result) { } void CloudTablet::set_cumulative_layer_point(int64_t new_point) { + if (new_point == Tablet::K_INVALID_CUMULATIVE_POINT || new_point >= _cumulative_point) { + _cumulative_point = new_point; + return; + } // cumulative point should only be reset to -1, or be increased - CHECK(new_point == Tablet::K_INVALID_CUMULATIVE_POINT || new_point >= _cumulative_point) - << "Unexpected cumulative point: " << new_point - << ", origin: " << _cumulative_point.load(); - _cumulative_point = new_point; + // FIXME: could happen in currently unresolved race conditions + LOG(WARNING) << "Unexpected cumulative point: " << new_point + << ", origin: " << _cumulative_point.load(); } std::vector CloudTablet::pick_candidate_rowsets_to_base_compaction() {