[Enhancement] Skip duplicate key large base rowset compaction (#23369)

Signed-off-by: meegoo <[email protected]>
(cherry picked from commit 840265d)

Author: meegoo

be/src/storage/size_tiered_compaction_policy.cpp

@@ -177,7 +177,17 @@ Status SizeTieredCompactionPolicy::_pick_rowsets_to_size_tiered_compact(bool for
     int64_t level_size = -1;
     int64_t total_size = 0;
     int64_t prev_end_version = -1;
+    bool skip_dup_large_base_rowset = true;
     for (auto rowset : candidate_rowsets) {
+        // when duplicate key's base rowset larger than 0.8 * max_segment_file_size, we don't need compact it
+        if (keys_type == KeysType::DUP_KEYS && skip_dup_large_base_rowset &&
+            !rowset->rowset_meta()->is_segments_overlapping() &&
+            rowset->data_disk_size() > config::max_segment_file_size * 0.8) {
+            continue;
+        } else {
+            skip_dup_large_base_rowset = false;
+        }
+
         int64_t rowset_size = rowset->data_disk_size() > 0 ? rowset->data_disk_size() : 1;
         if (level_size == -1) {
             level_size = rowset_size < _max_level_size ? rowset_size : _max_level_size;

be/test/storage/size_tiered_compaction_policy_test.cpp

@@ -1379,4 +1379,59 @@ TEST_F(SizeTieredCompactionPolicyTest, test_delete_limit) {
     }
 }
 
+TEST_F(SizeTieredCompactionPolicyTest, test_large_dup_base_rowset) {
+    LOG(INFO) << "test_large_dup_base_rowset";
+    create_tablet_schema(DUP_KEYS);
+
+    TabletMetaSharedPtr tablet_meta = std::make_shared<TabletMeta>();
+    create_tablet_meta(tablet_meta.get());
+
+    write_new_version(tablet_meta, 2);
+    write_new_version(tablet_meta, 1);
+    write_new_version(tablet_meta, 1);
+    write_new_version(tablet_meta, 1);
+
+    TabletSharedPtr tablet =
+            Tablet::create_tablet_from_meta(tablet_meta, starrocks::StorageEngine::instance()->get_stores()[0]);
+    tablet->init();
+    init_compaction_context(tablet);
+
+    ASSERT_EQ(4, tablet->version_count());
+
+    config::max_segment_file_size = 1024 * 128;
+    DeferOp defer([&]() {
+        config::max_segment_file_size = 1073741824;
+    });
+
+    {
+        auto res = compact(tablet);
+        ASSERT_TRUE(res.ok());
+
+        ASSERT_EQ(2, tablet->version_count());
+        std::vector<Version> versions;
+        tablet->list_versions(&versions);
+        ASSERT_EQ(2, versions.size());
+        ASSERT_EQ(0, versions[0].first);
+        ASSERT_EQ(0, versions[0].second);
+        ASSERT_EQ(1, versions[1].first);
+        ASSERT_EQ(3, versions[1].second);
+    }
+
+    {
+        auto res = compact(tablet);
+        ASSERT_FALSE(res.ok());
+
+        ASSERT_EQ(2, tablet->version_count());
+    }
+
+    config::max_segment_file_size = 1073741824;
+
+    {
+        auto res = compact(tablet);
+        ASSERT_TRUE(res.ok());
+
+        ASSERT_EQ(1, tablet->version_count());
+    }
+}
+
 } // namespace starrocks