Merge branch 'master' into fix-script-on-mac

be/src/cloud/cloud_base_compaction.cpp

@@ -268,8 +268,9 @@ Status CloudBaseCompaction::execute_compact() {
                      << ", output_version=" << _output_version;
         return res;
     }
-    LOG_INFO("finish CloudBaseCompaction, tablet_id={}, cost={}ms", _tablet->tablet_id(),
-             duration_cast<milliseconds>(steady_clock::now() - start).count())
+    LOG_INFO("finish CloudBaseCompaction, tablet_id={}, cost={}ms range=[{}-{}]",
+             _tablet->tablet_id(), duration_cast<milliseconds>(steady_clock::now() - start).count(),
+             _input_rowsets.front()->start_version(), _input_rowsets.back()->end_version())
             .tag("job_id", _uuid)
             .tag("input_rowsets", _input_rowsets.size())
             .tag("input_rows", _input_row_num)
@@ -343,7 +344,7 @@ Status CloudBaseCompaction::modify_rowsets() {
                 .tag("input_rowsets", _input_rowsets.size())
                 .tag("input_rows", _input_row_num)
                 .tag("input_segments", _input_segments)
-                .tag("update_bitmap_size", output_rowset_delete_bitmap->delete_bitmap.size());
+                .tag("num_output_delete_bitmap", output_rowset_delete_bitmap->delete_bitmap.size());
         compaction_job->set_delete_bitmap_lock_initiator(initiator);
     }
 

be/src/cloud/cloud_cumulative_compaction.cpp

@@ -204,8 +204,9 @@ Status CloudCumulativeCompaction::execute_compact() {
                      << ", output_version=" << _output_version;
         return res;
     }
-    LOG_INFO("finish CloudCumulativeCompaction, tablet_id={}, cost={}ms", _tablet->tablet_id(),
-             duration_cast<milliseconds>(steady_clock::now() - start).count())
+    LOG_INFO("finish CloudCumulativeCompaction, tablet_id={}, cost={}ms, range=[{}-{}]",
+             _tablet->tablet_id(), duration_cast<milliseconds>(steady_clock::now() - start).count(),
+             _input_rowsets.front()->start_version(), _input_rowsets.back()->end_version())
             .tag("job_id", _uuid)
             .tag("input_rowsets", _input_rowsets.size())
             .tag("input_rows", _input_row_num)
@@ -299,7 +300,8 @@ Status CloudCumulativeCompaction::modify_rowsets() {
                 .tag("input_rowsets", _input_rowsets.size())
                 .tag("input_rows", _input_row_num)
                 .tag("input_segments", _input_segments)
-                .tag("update_bitmap_size", output_rowset_delete_bitmap->delete_bitmap.size());
+                .tag("number_output_delete_bitmap",
+                     output_rowset_delete_bitmap->delete_bitmap.size());
         compaction_job->set_delete_bitmap_lock_initiator(initiator);
     }
 

be/src/cloud/cloud_storage_engine.h

@@ -75,7 +75,7 @@ class CloudStorageEngine final : public BaseStorageEngine {
     void _check_file_cache_ttl_block_valid();
 
     std::optional<StorageResource> get_storage_resource(const std::string& vault_id) {
-        LOG(INFO) << "Getting storage resource for vault_id: " << vault_id;
+        VLOG_DEBUG << "Getting storage resource for vault_id: " << vault_id;
 
         bool synced = false;
         do {

be/src/cloud/cloud_tablet.cpp

@@ -33,6 +33,7 @@
 #include "cloud/cloud_meta_mgr.h"
 #include "cloud/cloud_storage_engine.h"
 #include "cloud/cloud_tablet_mgr.h"
+#include "common/logging.h"
 #include "io/cache/block_file_cache_downloader.h"
 #include "io/cache/block_file_cache_factory.h"
 #include "olap/cumulative_compaction_time_series_policy.h"
@@ -408,6 +409,9 @@ uint64_t CloudTablet::delete_expired_stale_rowsets() {
                 auto rs_it = _stale_rs_version_map.find(v_ts->version());
                 if (rs_it != _stale_rs_version_map.end()) {
                     expired_rowsets.push_back(rs_it->second);
+                    LOG(INFO) << "erase stale rowset, tablet_id=" << tablet_id()
+                              << " rowset_id=" << rs_it->second->rowset_id().to_string()
+                              << " version=" << rs_it->first.to_string();
                     _stale_rs_version_map.erase(rs_it);
                 } else {
                     LOG(WARNING) << "cannot find stale rowset " << v_ts->version() << " in tablet "
@@ -657,11 +661,14 @@ void CloudTablet::get_compaction_status(std::string* json_result) {
 }
 
 void CloudTablet::set_cumulative_layer_point(int64_t new_point) {
+    if (new_point == Tablet::K_INVALID_CUMULATIVE_POINT || new_point >= _cumulative_point) {
+        _cumulative_point = new_point;
+        return;
+    }
     // cumulative point should only be reset to -1, or be increased
-    CHECK(new_point == Tablet::K_INVALID_CUMULATIVE_POINT || new_point >= _cumulative_point)
-            << "Unexpected cumulative point: " << new_point
-            << ", origin: " << _cumulative_point.load();
-    _cumulative_point = new_point;
+    // FIXME: could happen in currently unresolved race conditions
+    LOG(WARNING) << "Unexpected cumulative point: " << new_point
+                 << ", origin: " << _cumulative_point.load();
 }
 
 std::vector<RowsetSharedPtr> CloudTablet::pick_candidate_rowsets_to_base_compaction() {

be/src/exec/table_connector.cpp

@@ -118,16 +118,17 @@ Status TableConnector::convert_column_data(const vectorized::ColumnPtr& column_p
             fmt::format_to(_insert_stmt_buffer, "\"{}\"", str);
         }
     };
-    const vectorized::IColumn* column = column_ptr;
+    const vectorized::IColumn* column = column_ptr.get();
     if (type_ptr->is_nullable()) {
-        auto nullable_column = assert_cast<const vectorized::ColumnNullable*>(column_ptr.get());
+        const auto* nullable_column =
+                assert_cast<const vectorized::ColumnNullable*>(column_ptr.get());
         if (nullable_column->is_null_at(row)) {
             fmt::format_to(_insert_stmt_buffer, "{}", "NULL");
             return Status::OK();
         }
         column = nullable_column->get_nested_column_ptr().get();
     } else {
-        column = column_ptr;
+        column = column_ptr.get();
     }
     auto [item, size] = column->get_data_at(row);
     switch (type.type) {

be/src/http/http_channel.cpp

@@ -123,7 +123,8 @@ void HttpChannel::send_files(HttpRequest* request, const std::string& root_dir,
         VLOG_DEBUG << "http channel send file " << file_path << ", size: " << file_size;
 
         evbuffer_add_printf(evb.get(), "File-Name: %s\r\n", file.c_str());
-        evbuffer_add_printf(evb.get(), "Content-Length: %ld\r\n", file_size);
+        evbuffer_add_printf(evb.get(), "Content-Length: %" PRIi64 "\r\n", file_size);
+
         evbuffer_add_printf(evb.get(), "\r\n");
         if (file_size > 0) {
             evbuffer_add_file(evb.get(), fd, 0, file_size);

be/src/olap/lru_cache.cpp

@@ -604,12 +604,12 @@ ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t capacity, LRUCa
     INT_GAUGE_METRIC_REGISTER(_entity, cache_capacity);
     INT_GAUGE_METRIC_REGISTER(_entity, cache_usage);
     INT_GAUGE_METRIC_REGISTER(_entity, cache_element_count);
-    INT_DOUBLE_METRIC_REGISTER(_entity, cache_usage_ratio);
-    INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_lookup_count);
-    INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_hit_count);
-    INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_stampede_count);
-    INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_miss_count);
-    INT_DOUBLE_METRIC_REGISTER(_entity, cache_hit_ratio);
+    DOUBLE_GAUGE_METRIC_REGISTER(_entity, cache_usage_ratio);
+    INT_COUNTER_METRIC_REGISTER(_entity, cache_lookup_count);
+    INT_COUNTER_METRIC_REGISTER(_entity, cache_hit_count);
+    INT_COUNTER_METRIC_REGISTER(_entity, cache_stampede_count);
+    INT_COUNTER_METRIC_REGISTER(_entity, cache_miss_count);
+    DOUBLE_GAUGE_METRIC_REGISTER(_entity, cache_hit_ratio);
 
     _hit_count_bvar.reset(new bvar::Adder<uint64_t>("doris_cache", _name));
     _hit_count_per_second.reset(new bvar::PerSecond<bvar::Adder<uint64_t>>(

be/src/olap/lru_cache.h

@@ -447,10 +447,10 @@ class ShardedLRUCache : public Cache {
     IntGauge* cache_usage = nullptr;
     IntGauge* cache_element_count = nullptr;
     DoubleGauge* cache_usage_ratio = nullptr;
-    IntAtomicCounter* cache_lookup_count = nullptr;
-    IntAtomicCounter* cache_hit_count = nullptr;
-    IntAtomicCounter* cache_miss_count = nullptr;
-    IntAtomicCounter* cache_stampede_count = nullptr;
+    IntCounter* cache_lookup_count = nullptr;
+    IntCounter* cache_hit_count = nullptr;
+    IntCounter* cache_miss_count = nullptr;
+    IntCounter* cache_stampede_count = nullptr;
     DoubleGauge* cache_hit_ratio = nullptr;
     // bvars
     std::unique_ptr<bvar::Adder<uint64_t>> _hit_count_bvar;

be/src/olap/push_handler.cpp

@@ -518,7 +518,7 @@ Status PushBrokerReader::_convert_to_output_block(vectorized::Block* block) {
         column_ptr = _src_block.get_by_position(result_column_id).column;
         // column_ptr maybe a ColumnConst, convert it to a normal column
         column_ptr = column_ptr->convert_to_full_column_if_const();
-        DCHECK(column_ptr != nullptr);
+        DCHECK(column_ptr);
 
         // because of src_slot_desc is always be nullable, so the column_ptr after do dest_expr
         // is likely to be nullable

be/src/olap/rowset/segment_v2/column_reader.cpp

@@ -1267,8 +1267,8 @@ Status FileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& d
                     DCHECK_EQ(this_run, num_rows);
                 } else {
                     *has_null = true;
-                    auto* null_col =
-                            vectorized::check_and_get_column<vectorized::ColumnNullable>(dst);
+                    const auto* null_col =
+                            vectorized::check_and_get_column<vectorized::ColumnNullable>(dst.get());
                     if (null_col != nullptr) {
                         const_cast<vectorized::ColumnNullable*>(null_col)->insert_null_elements(
                                 this_run);
@@ -1328,8 +1328,9 @@ Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t co
                 auto origin_index = _page.data_decoder->current_index();
                 if (this_read_count > 0) {
                     if (is_null) {
-                        auto* null_col =
-                                vectorized::check_and_get_column<vectorized::ColumnNullable>(dst);
+                        const auto* null_col =
+                                vectorized::check_and_get_column<vectorized::ColumnNullable>(
+                                        dst.get());
                         if (UNLIKELY(null_col == nullptr)) {
                             return Status::InternalError("unexpected column type in column reader");
                         }
@@ -1710,9 +1711,9 @@ Status DefaultNestedColumnIterator::next_batch(size_t* n, vectorized::MutableCol
 static void fill_nested_with_defaults(vectorized::MutableColumnPtr& dst,
                                       vectorized::MutableColumnPtr& sibling_column, size_t nrows) {
     const auto* sibling_array = vectorized::check_and_get_column<vectorized::ColumnArray>(
-            remove_nullable(sibling_column->get_ptr()));
+            remove_nullable(sibling_column->get_ptr()).get());
     const auto* dst_array = vectorized::check_and_get_column<vectorized::ColumnArray>(
-            remove_nullable(dst->get_ptr()));
+            remove_nullable(dst->get_ptr()).get());
     if (!dst_array || !sibling_array) {
         throw doris::Exception(ErrorCode::INTERNAL_ERROR,
                                "Expected array column, but met %s and %s", dst->get_name(),

be/src/olap/rowset/segment_v2/hierarchical_data_reader.h

@@ -165,8 +165,8 @@ class HierarchicalDataReader : public ColumnIterator {
         // will type the type of ColumnObject::NESTED_TYPE, whih is Nullable<ColumnArray<NULLABLE(ColumnObject)>>.
         for (auto& entry : nested_subcolumns) {
             MutableColumnPtr nested_object = ColumnObject::create(true, false);
-            const auto* base_array =
-                    check_and_get_column<ColumnArray>(remove_nullable(entry.second[0].column));
+            const auto* base_array = check_and_get_column<ColumnArray>(
+                    remove_nullable(entry.second[0].column).get());
             MutableColumnPtr offset = base_array->get_offsets_ptr()->assume_mutable();
             auto* nested_object_ptr = assert_cast<ColumnObject*>(nested_object.get());
             // flatten nested arrays