From bfde4a371593327864af2e910d3b90f80306bbe8 Mon Sep 17 00:00:00 2001 From: shuke <37901441+shuke987@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:48:21 +0800 Subject: [PATCH] [fix](regression) Stabilize delete bitmap cache metrics test ### What problem does this PR solve? Issue Number: N/A Related PR: #64515 Problem Summary: The case warmed aggregate delete-bitmap cache entries before iterating over replicas, then fetched the large global brpc metrics payload inside that loop. On a loaded multi-replica P0 cluster, later replicas could be checked tens of seconds after warm-up even though the cache is a capacity-bounded LRU. Pin and query each replica immediately before inspecting its cache. Sort the tablet rows by ReplicaId to match use_fix_replica ordering, restore the session variable in finally, and fetch the global metrics payload once after all per-replica checks. ### Release note None ### Check List (For Author) - Test - [x] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [x] No. - [ ] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../test_delete_bitmap_metrics.groovy | 129 ++++++++++-------- 1 file changed, 70 insertions(+), 59 deletions(-) diff --git a/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy b/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy index 80f7da957ef20e..c4a76cca5f7c64 100644 --- a/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy +++ b/regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy @@ -184,79 +184,90 @@ suite("test_delete_bitmap_metrics", "p0") { def tablets = sql_return_maparray """ show tablets from ${testTable}; """ + tablets.sort { lhs, rhs -> lhs.ReplicaId.toLong() <=> rhs.ReplicaId.toLong() } logger.info("tablets: " + tablets) def local_delete_bitmap_count = 0 def ms_delete_bitmap_count = 0 def local_delete_bitmap_cardinality = 0; def ms_delete_bitmap_cardinality = 0; - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - String trigger_backend_id = tablet.BackendId + try { + for (int ri = 0; ri < tablets.size(); ri++) { + // use_fix_replica selects replicas in ReplicaId order. Populate and + // inspect each replica immediately because the agg cache is an LRU. + sql "set use_fix_replica=${ri};" + sql "select * from ${testTable};" - // before compaction, delete_bitmap_count is (rowsets num - 1) - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - assertTrue(local_delete_bitmap_count == 7) - assertTrue(local_delete_bitmap_cardinality == 7) + def tablet = tablets[ri] + String tablet_id = tablet.TabletId + def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ + logger.info("tablet: " + tablet_info) + String trigger_backend_id = tablet.BackendId - if (isCloudMode()) { - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) - assertTrue(ms_delete_bitmap_count == 7) - assertTrue(ms_delete_bitmap_cardinality == 7) - } + // before compaction, delete_bitmap_count is (rowsets num - 1) + local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) + logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) + assertTrue(local_delete_bitmap_count == 7) + assertTrue(local_delete_bitmap_cardinality == 7) + + if (isCloudMode()) { + ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count + ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality + logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) + logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) + assertTrue(ms_delete_bitmap_count == 7) + assertTrue(ms_delete_bitmap_cardinality == 7) + } - def status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - logger.info("agg cache status: ${status}") - assert status.delete_bitmap_count == 8 - assert status.cardinality == 7 - assert status.size > 0 + def status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) + logger.info("agg cache status: ${status}") + assert status.delete_bitmap_count == 8 + assert status.cardinality == 7 + assert status.size > 0 - status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id, true) - logger.info("agg cache verbose status: ${status}") + status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id, true) + logger.info("agg cache verbose status: ${status}") + } + } finally { + sql "set use_fix_replica=-1;" + } - def tablet_delete_bitmap_count = 0; - def base_rowset_delete_bitmap_count = 0; - int retry_time = 0; - while (retry_time < 10) { - log.info("retry_time: ${retry_time}") - getMetricsMethod.call() { - respCode, body -> - logger.info("test get delete bitmap count resp Code {}", "${respCode}".toString()) - assertEquals("${respCode}".toString(), "200") - String out = "${body}".toString() - def strs = out.split('\n') - for (String line in strs) { - if (line.startsWith("tablet_max_delete_bitmap_score")) { - logger.info("find: {}", line) - tablet_delete_bitmap_count = line.replaceAll("tablet_max_delete_bitmap_score ", "").toInteger() - break - } + def tablet_delete_bitmap_count = 0; + def base_rowset_delete_bitmap_count = 0; + int retry_time = 0; + while (retry_time < 10) { + log.info("retry_time: ${retry_time}") + getMetricsMethod.call() { + respCode, body -> + logger.info("test get delete bitmap count resp Code {}", "${respCode}".toString()) + assertEquals("${respCode}".toString(), "200") + String out = "${body}".toString() + def strs = out.split('\n') + for (String line in strs) { + if (line.startsWith("tablet_max_delete_bitmap_score")) { + logger.info("find: {}", line) + tablet_delete_bitmap_count = line.replaceAll("tablet_max_delete_bitmap_score ", "").toInteger() + break } - for (String line in strs) { - if (line.startsWith("tablet_max_base_rowset_delete_bitmap_score")) { - logger.info("find: {}", line) - base_rowset_delete_bitmap_count = line.replaceAll("tablet_max_base_rowset_delete_bitmap_score ", "").toInteger() - break - } + } + for (String line in strs) { + if (line.startsWith("tablet_max_base_rowset_delete_bitmap_score")) { + logger.info("find: {}", line) + base_rowset_delete_bitmap_count = line.replaceAll("tablet_max_base_rowset_delete_bitmap_score ", "").toInteger() + break } - } - if (tablet_delete_bitmap_count > 0 && base_rowset_delete_bitmap_count > 0) { - break; - } else { - Thread.sleep(10000) - retry_time++; - } + } + } + if (tablet_delete_bitmap_count > 0 && base_rowset_delete_bitmap_count > 0) { + break; + } else { + Thread.sleep(10000) + retry_time++; } - assertTrue(tablet_delete_bitmap_count > 0) - assertTrue(base_rowset_delete_bitmap_count > 0) } + assertTrue(tablet_delete_bitmap_count > 0) + assertTrue(base_rowset_delete_bitmap_count > 0) } finally { reset_be_param("check_tablet_delete_bitmap_interval_seconds") }