Skip to content

[fix](inverted-index) Avoid global null bitmap in sub-reader context #65549

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hoshinojyunn wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[fix](inverted-index) Avoid global null bitmap in sub-reader context #65549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 27 additions & 19 deletions be/src/exprs/function/function_search.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,26 @@ query_v2::QueryPtr make_unknown_query(uint32_t num_rows) {
std::move(null_bitmap));
}

std::shared_ptr<roaring::Roaring> extract_query_null_bitmap(
const query_v2::WeightPtr& weight, const query_v2::QueryExecutionContext& context,
const std::string& binding_key) {
auto null_bitmap = std::make_shared<roaring::Roaring>();
if (context.null_resolver == nullptr) {
return null_bitmap;
}

auto scorer = weight->scorer(context, binding_key);
if (scorer && scorer->has_null_bitmap(context.null_resolver)) {
const auto* bitmap = scorer->get_null_bitmap(context.null_resolver);
if (bitmap != nullptr) {
*null_bitmap = *bitmap;
VLOG_TRACE << "search: Extracted NULL bitmap with " << null_bitmap->cardinality()
<< " NULL documents";
}
}
return null_bitmap;
}

DataTypePtr unwrap_direct_index_value_type(DataTypePtr column_type) {
DataTypePtr value_type = remove_nullable(std::move(column_type));
while (value_type != nullptr &&
Expand Down Expand Up @@ -412,42 +432,30 @@ Status FunctionSearch::evaluate_inverted_index_with_search_param(
}

std::shared_ptr<roaring::Roaring> roaring = std::make_shared<roaring::Roaring>();
std::shared_ptr<roaring::Roaring> null_bitmap = nullptr;
{
int64_t exec_dummy = 0;
const bool is_top_k = enable_scoring && !is_asc && top_k > 0;
SCOPED_RAW_TIMER(stats ? &stats->inverted_index_searcher_search_exec_timer : &exec_dummy);
if (enable_scoring && !is_asc && top_k > 0) {
if (is_top_k) {
bool use_wand = index_query_context->runtime_state != nullptr &&
index_query_context->runtime_state->query_options()
.enable_inverted_index_wand_query;
null_bitmap = extract_query_null_bitmap(weight, exec_ctx, root_binding_key);
query_v2::collect_multi_segment_top_k(
weight, exec_ctx, root_binding_key, top_k, roaring,
index_query_context->collection_similarity, use_wand);
index_query_context->collection_similarity, use_wand, null_bitmap.get());
} else {
query_v2::collect_multi_segment_doc_set(
weight, exec_ctx, root_binding_key, roaring,
index_query_context ? index_query_context->collection_similarity : nullptr,
enable_scoring);
null_bitmap = extract_query_null_bitmap(weight, exec_ctx, root_binding_key);
}
}

DCHECK(null_bitmap != nullptr) << "search: NULL bitmap should not be nullptr after collection";
VLOG_DEBUG << "search: Query completed, matched " << roaring->cardinality() << " documents";

// Extract NULL bitmap from three-valued logic scorer
// The scorer correctly computes which documents evaluate to NULL based on query logic
// For example: TRUE OR NULL = TRUE (not NULL), FALSE OR NULL = NULL
std::shared_ptr<roaring::Roaring> null_bitmap = std::make_shared<roaring::Roaring>();
if (exec_ctx.null_resolver) {
auto scorer = weight->scorer(exec_ctx, root_binding_key);
if (scorer && scorer->has_null_bitmap(exec_ctx.null_resolver)) {
const auto* bitmap = scorer->get_null_bitmap(exec_ctx.null_resolver);
if (bitmap != nullptr) {
*null_bitmap = *bitmap;
VLOG_TRACE << "search: Extracted NULL bitmap with " << null_bitmap->cardinality()
<< " NULL documents";
}
}
}

VLOG_TRACE << "search: Before mask - true_bitmap=" << roaring->cardinality()
<< ", null_bitmap=" << null_bitmap->cardinality();

Expand Down
1 change: 0 additions & 1 deletion be/src/storage/index/inverted/query_v2/collect/multi_segment_util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,6 @@ inline QueryExecutionContext create_segment_context(const QueryExecutionContext&
}

seg_ctx.binding_fields = original_ctx.binding_fields;

@github-actions github-actions Bot Jul 14, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This fixes the full doc-set case, but the same segment context is used by collect_multi_segment_top_k(). In that path we build scorers from seg_ctx, keep only k docs per segment and then globally, and only later does FunctionSearch rebuild a global scorer and call mask_out_null(). Because this context no longer has a resolver, nullable NOT/boolean scorers treat NULL rows as ordinary matches during top-k selection; after the later mask removes those rows, the next non-NULL candidates have already been discarded. A concrete shape is top_k=3 over docs 0..3 where doc 0 is NULL for the field and the query is NOT missing: top-k can retain 0,1,2 on equal scores, the global mask drops 0, and doc 3 is never reconsidered. Please either filter/mark NULL rows before top-k truncation using segment-local null bitmaps, or apply the original global NULL mask before enforcing the final k, and add a scored descending top-k regression.

seg_ctx.null_resolver = original_ctx.null_resolver;

return seg_ctx;
}
Expand Down
12 changes: 9 additions & 3 deletions be/src/storage/index/inverted/query_v2/collect/top_k_collector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,22 @@ namespace doris::segment_v2::inverted_index::query_v2 {
void collect_multi_segment_top_k(const WeightPtr& weight, const QueryExecutionContext& context,
const std::string& binding_key, size_t k,
const std::shared_ptr<roaring::Roaring>& roaring,
const CollectionSimilarityPtr& similarity, bool use_wand) {
const CollectionSimilarityPtr& similarity, bool use_wand,
const roaring::Roaring* excluded_docs) {
TopKCollector final_collector(k);

for_each_index_segment(
context, binding_key, [&](const QueryExecutionContext& seg_ctx, uint32_t seg_base) {
float initial_threshold = final_collector.threshold();

TopKCollector seg_collector(k);
auto callback = [&seg_collector](uint32_t doc_id, float score) -> float {
return seg_collector.collect(doc_id, score);
auto callback = [initial_threshold, &seg_collector, excluded_docs, seg_base](
uint32_t doc_id, float score) -> float {
// Filter global NULL documents before segment Top-K truncation.
if (excluded_docs != nullptr && excluded_docs->contains(doc_id + seg_base)) {
return std::max(initial_threshold, seg_collector.threshold());
}
return std::max(initial_threshold, seg_collector.collect(doc_id, score));
};

if (use_wand) {
Expand Down
3 changes: 2 additions & 1 deletion be/src/storage/index/inverted/query_v2/collect/top_k_collector.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ class TopKCollector {
void collect_multi_segment_top_k(const WeightPtr& weight, const QueryExecutionContext& context,
const std::string& binding_key, size_t k,
const std::shared_ptr<roaring::Roaring>& roaring,
const CollectionSimilarityPtr& similarity, bool use_wand = true);
const CollectionSimilarityPtr& similarity, bool use_wand = true,
const roaring::Roaring* excluded_docs = nullptr);

} // namespace doris::segment_v2::inverted_index::query_v2
18 changes: 18 additions & 0 deletions be/src/storage/index/inverted/query_v2/wand/block_wand.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@

#include <algorithm>
#include <cassert>
#include <limits>
#include <ranges>
#include <vector>

#include "storage/index/inverted/query_v2/term_query/term_scorer.h"
#include "util/debug_points.h"

namespace doris::segment_v2::inverted_index::query_v2 {

Expand Down Expand Up @@ -75,6 +77,10 @@ class BlockWand {
return;
}

// Test probe: only an actual multi-scorer WAND invocation observes this override.
DBUG_EXECUTE_IF("BlockWand.multi_scorer.force_threshold",
{ threshold = dp->param<float>("threshold", threshold); });

std::vector<ScorerWrapper> wrappers;
wrappers.reserve(scorers.size());
for (auto& s : scorers) {
Expand Down Expand Up @@ -107,6 +113,11 @@ class BlockWand {
}

if (block_max_score_upperbound <= threshold) {
DBUG_EXECUTE_IF("BlockWand.multi_scorer.pruned_block", {
LOG(INFO) << "BlockWand pruned block: pivot_doc=" << pivot_doc
<< ", upper_bound=" << block_max_score_upperbound
<< ", threshold=" << threshold;
});
block_max_was_too_low_advance_one_scorer(wrappers, pivot_len);
continue;
}
Expand All @@ -120,6 +131,13 @@ class BlockWand {
score += wrappers[i].score();
}

// Test probe: a score call at or after min_doc makes the Top-K result observable.
DBUG_EXECUTE_IF("BlockWand.multi_scorer.poison_score_from_doc", {
if (pivot_doc >= dp->param<uint32_t>("min_doc", TERMINATED)) {
score = std::numeric_limits<float>::max();
}
});

if (score > threshold) {
threshold = callback(pivot_doc, score);
}
Expand Down
Loading
Loading