Skip to content

[fix](regression) Stabilize delete bitmap cache metrics test #65535

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
shuke987 wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

[fix](regression) Stabilize delete bitmap cache metrics test #65535

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 70 additions & 59 deletions regression-test/suites/metrics_p0/test_delete_bitmap_metrics.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,79 +184,90 @@ suite("test_delete_bitmap_metrics", "p0") {


def tablets = sql_return_maparray """ show tablets from ${testTable}; """
tablets.sort { lhs, rhs -> lhs.ReplicaId.toLong() <=> rhs.ReplicaId.toLong() }
logger.info("tablets: " + tablets)
def local_delete_bitmap_count = 0
def ms_delete_bitmap_count = 0
def local_delete_bitmap_cardinality = 0;
def ms_delete_bitmap_cardinality = 0;
for (def tablet in tablets) {
String tablet_id = tablet.TabletId
def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """
logger.info("tablet: " + tablet_info)
String trigger_backend_id = tablet.BackendId
try {
for (int ri = 0; ri < tablets.size(); ri++) {
// use_fix_replica selects replicas in ReplicaId order. Populate and
// inspect each replica immediately because the agg cache is an LRU.
sql "set use_fix_replica=${ri};"
sql "select * from ${testTable};"

@github-actions github-actions Bot Jul 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This still looks flaky when cloud multi-replica placement is enabled. The test records tablet.BackendId from SHOW TABLETS, then warms with use_fix_replica=${ri} and probes /count_agg_cache on that stored backend. In cloud mode, though, CloudTablet has one logical CloudReplica, and CloudReplica.getBackendIdImpl() chooses a physical BE with rand.nextInt(Config.cloud_replica_num) on each backend-resolution call when enable_cloud_multi_replica is true. The SHOW TABLETS call and the later scan can therefore pick different physical BEs, while /count_agg_cache only snapshots the target BE's local agg cache. Please either pin/disable cloud multi-replica for this per-BE cache assertion, or query the actual BE selected by the warmup scan before checking its cache.


// before compaction, delete_bitmap_count is (rowsets num - 1)
local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality)
assertTrue(local_delete_bitmap_count == 7)
assertTrue(local_delete_bitmap_cardinality == 7)
def tablet = tablets[ri]
String tablet_id = tablet.TabletId
def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """
logger.info("tablet: " + tablet_info)
String trigger_backend_id = tablet.BackendId

if (isCloudMode()) {
ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
assertTrue(ms_delete_bitmap_count == 7)
assertTrue(ms_delete_bitmap_cardinality == 7)
}
// before compaction, delete_bitmap_count is (rowsets num - 1)
local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count)
logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality)
assertTrue(local_delete_bitmap_count == 7)
assertTrue(local_delete_bitmap_cardinality == 7)

if (isCloudMode()) {
ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality)
assertTrue(ms_delete_bitmap_count == 7)
assertTrue(ms_delete_bitmap_cardinality == 7)
}

def status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id)
logger.info("agg cache status: ${status}")
assert status.delete_bitmap_count == 8
assert status.cardinality == 7
assert status.size > 0
def status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id)
logger.info("agg cache status: ${status}")
assert status.delete_bitmap_count == 8
assert status.cardinality == 7
assert status.size > 0

status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id, true)
logger.info("agg cache verbose status: ${status}")
status = getAggCacheDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id, true)
logger.info("agg cache verbose status: ${status}")
}
} finally {
sql "set use_fix_replica=-1;"
}

def tablet_delete_bitmap_count = 0;
def base_rowset_delete_bitmap_count = 0;
int retry_time = 0;
while (retry_time < 10) {
log.info("retry_time: ${retry_time}")
getMetricsMethod.call() {
respCode, body ->
logger.info("test get delete bitmap count resp Code {}", "${respCode}".toString())
assertEquals("${respCode}".toString(), "200")
String out = "${body}".toString()
def strs = out.split('\n')
for (String line in strs) {
if (line.startsWith("tablet_max_delete_bitmap_score")) {
logger.info("find: {}", line)
tablet_delete_bitmap_count = line.replaceAll("tablet_max_delete_bitmap_score ", "").toInteger()
break
}
def tablet_delete_bitmap_count = 0;
def base_rowset_delete_bitmap_count = 0;
int retry_time = 0;
while (retry_time < 10) {
log.info("retry_time: ${retry_time}")
getMetricsMethod.call() {
respCode, body ->
logger.info("test get delete bitmap count resp Code {}", "${respCode}".toString())
assertEquals("${respCode}".toString(), "200")
String out = "${body}".toString()
def strs = out.split('\n')
for (String line in strs) {
if (line.startsWith("tablet_max_delete_bitmap_score")) {
logger.info("find: {}", line)
tablet_delete_bitmap_count = line.replaceAll("tablet_max_delete_bitmap_score ", "").toInteger()
break
}
for (String line in strs) {
if (line.startsWith("tablet_max_base_rowset_delete_bitmap_score")) {
logger.info("find: {}", line)
base_rowset_delete_bitmap_count = line.replaceAll("tablet_max_base_rowset_delete_bitmap_score ", "").toInteger()
break
}
}
for (String line in strs) {
if (line.startsWith("tablet_max_base_rowset_delete_bitmap_score")) {
logger.info("find: {}", line)
base_rowset_delete_bitmap_count = line.replaceAll("tablet_max_base_rowset_delete_bitmap_score ", "").toInteger()
break
}
}
if (tablet_delete_bitmap_count > 0 && base_rowset_delete_bitmap_count > 0) {
break;
} else {
Thread.sleep(10000)
retry_time++;
}
}
}
if (tablet_delete_bitmap_count > 0 && base_rowset_delete_bitmap_count > 0) {
break;
} else {
Thread.sleep(10000)
retry_time++;
}
assertTrue(tablet_delete_bitmap_count > 0)
assertTrue(base_rowset_delete_bitmap_count > 0)
}
assertTrue(tablet_delete_bitmap_count > 0)
assertTrue(base_rowset_delete_bitmap_count > 0)
} finally {
reset_be_param("check_tablet_delete_bitmap_interval_seconds")
}
Expand Down
Loading