diff --git a/be/src/storage/index/index_file_writer.h b/be/src/storage/index/index_file_writer.h index f5a2e4c1d671b2..3a0346b95d428a 100644 --- a/be/src/storage/index/index_file_writer.h +++ b/be/src/storage/index/index_file_writer.h @@ -110,8 +110,8 @@ class IndexFileWriter { // the prx region compresses at snii_prx_zstd_level_direct_load; // compaction / schema change / ADD INDEX keep snii_prx_zstd_level. bool is_direct_load = false; - // 每 doc 一字节的 BM25 norms；为空表示该索引不写 norms（keyword / 不带位置）。 - // 非空时长度必须等于 doc_count，且 posting 会保留 freq 区供打分读取。 + // One byte of BM25 norms per document; empty for keyword or positionless indexes. + // If nonempty, its size must equal doc_count, and postings retain frequencies for scoring. std::vector encoded_norms; }; Status add_snii_index(const TabletIndex* index_meta, uint32_t doc_count, @@ -138,8 +138,8 @@ class IndexFileWriter { doris::snii::format::IndexConfig index_config, std::shared_ptr mem_reporter, doris::snii::writer::SniiStreamedIndexSession** session); - // write_norms=true 的会话在 finish 之前必须通过 set_encoded_norms 交付 norms - // （compaction 在合并 postings 的同一趟里重建它们）。 + // Sessions with write_norms=true must supply norms through set_encoded_norms before finish. + // Compaction rebuilds them in the same pass that merges postings. Status add_snii_index_streamed( const TabletIndex* index_meta, uint32_t doc_count, doris::snii::writer::TrackedNullDocids null_docids, bool write_norms, diff --git a/be/src/storage/index/inverted/inverted_index_cache.h b/be/src/storage/index/inverted/inverted_index_cache.h index 0b53f3c0da810a..e470d88ec6cf4d 100644 --- a/be/src/storage/index/inverted/inverted_index_cache.h +++ b/be/src/storage/index/inverted/inverted_index_cache.h @@ -216,7 +216,7 @@ class InvertedIndexCacheHandle { class InvertedIndexQueryCacheHandle; -// v2：结果缓存键不再携带 CommonGrams 查询计划开关位（该功能已删除）。 +// v2: Result cache keys no longer include the removed CommonGrams query-plan flags. inline constexpr uint32_t INVERTED_INDEX_QUERY_CACHE_SEMANTICS_VERSION = 2; // Stable identity shared by result-cache and row-accurate single-flight. It intentionally contains diff --git a/be/src/storage/index/inverted/similarity/collection_statistics.cpp b/be/src/storage/index/inverted/similarity/collection_statistics.cpp index d8e8d4a2620949..69bd4155389fed 100644 --- a/be/src/storage/index/inverted/similarity/collection_statistics.cpp +++ b/be/src/storage/index/inverted/similarity/collection_statistics.cpp @@ -251,7 +251,7 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, ::doris::snii::reader::DictBlockCache dict_block_cache; for (const auto& logical_term_bytes : collect_info.unique_terms) { - // SNII 的 term 键就是分词后的原始字节，没有任何转义或版本。 + // SNII term keys are the raw analyzed bytes, with no escaping or version prefix. const std::string& physical_term = logical_term_bytes; const auto logical_term = segment_v2::inverted_index::StringHelper::to_wstring(logical_term_bytes); diff --git a/be/src/storage/index/inverted/similarity/collection_statistics.h b/be/src/storage/index/inverted/similarity/collection_statistics.h index d65ee4a35126a7..7e93498508324a 100644 --- a/be/src/storage/index/inverted/similarity/collection_statistics.h +++ b/be/src/storage/index/inverted/similarity/collection_statistics.h @@ -119,8 +119,9 @@ struct SniiScoringSegmentStats { uint64_t token_count = 0; }; -// 一个 SNII 段能参与打分的条件：带位置（词频来自位置）且带 norms（新版 writer 对分词 + 带位置 -// 的索引一律写出）。老段没有 norms → NOT_SUPPORTED，重建索引或等 compaction 补齐。 +// SNII scoring requires positions (which provide term frequencies) and norms. The current writer +// emits norms for every analyzed index with positions. Older segments without norms return +// NOT_SUPPORTED until an index rebuild or compaction supplies them. Result resolve_snii_scoring_segment(uint64_t index_doc_count, uint64_t sum_total_term_freq, bool has_positions, bool has_norms); diff --git a/be/src/storage/index/snii/compaction/eligibility.h b/be/src/storage/index/snii/compaction/eligibility.h index 8c24ea56cb3961..7257258ef3ab9a 100644 --- a/be/src/storage/index/snii/compaction/eligibility.h +++ b/be/src/storage/index/snii/compaction/eligibility.h @@ -52,9 +52,9 @@ using AnalyzerProviderFactory = std::function; struct SniiCompactionEligibility { - // A2：目标索引是"分词 + 带位置"时，合并产物必须带 norms。norms 在合并 postings 的同一趟里 - // 从各源的词频重建（每 doc Σfreq，clamp 到 1..255），所以老段（没有 norms 的 T2）也能 - // 不重分词地升级成带 norms 的段。 + // A2: An analyzed destination index with positions must include norms. The posting merge + // rebuilds them from source term frequencies (sum per document, clamped to 1..255), so older + // T2 segments without norms can gain norms without reanalysis. bool destination_writes_norms = false; }; diff --git a/be/src/storage/index/snii/compaction/posting_run_merger.cpp b/be/src/storage/index/snii/compaction/posting_run_merger.cpp index f4cfaa7ff120be..53a9a34aa41924 100644 --- a/be/src/storage/index/snii/compaction/posting_run_merger.cpp +++ b/be/src/storage/index/snii/compaction/posting_run_merger.cpp @@ -363,8 +363,8 @@ Status MergedPostingRuns::select_run(ActivePostingChunk* active, size_t max_docs : std::span {}; if (!destination_doc_lengths_.empty()) { - // 重建 norms：每篇文档的长度 = 该文档在所有 term 上的词频之和（与 writer 的 - // 词元计数逐字节一致，见 SniiIndexColumnWriter），按 u8 饱和累加，255 封顶。 + // Rebuild norms by summing each document's frequencies across all terms, matching the + // token counts in SniiIndexColumnWriter byte for byte. Saturate the u8 sum at 255. std::vector& lengths = destination_doc_lengths_[*active_destination_]; for (size_t i = 0; i < document_count; ++i) { const uint32_t docid = run->docids[i]; diff --git a/be/src/storage/index/snii/compaction/posting_run_merger.h b/be/src/storage/index/snii/compaction/posting_run_merger.h index 95afa259377213..1562e9d4de23dd 100644 --- a/be/src/storage/index/snii/compaction/posting_run_merger.h +++ b/be/src/storage/index/snii/compaction/posting_run_merger.h @@ -57,8 +57,9 @@ class MergedPostingRuns final : public writer::TermPostingSource { }; public: - // destination_doc_lengths 非空时，每个 (目标段, 目标 docid) 的词频按 u8 饱和累加（0..255）， - // 合并结束后由调用方 encode 成 norms；为空表示目标不写 norms。 + // If destination_doc_lengths is nonempty, accumulate frequencies for each destination + // (segment, docid), saturating at 255. The caller encodes them as norms after the merge. + // An empty vector means the destination does not write norms. MergedPostingRuns(std::vector> cursors, bool retain_positions, std::span destination_doc_counts, std::span> destination_doc_lengths); diff --git a/be/src/storage/index/snii/compaction/snii_index_compaction.cpp b/be/src/storage/index/snii/compaction/snii_index_compaction.cpp index a7c4cdcfb385f8..efa021d2f1e843 100644 --- a/be/src/storage/index/snii/compaction/snii_index_compaction.cpp +++ b/be/src/storage/index/snii/compaction/snii_index_compaction.cpp @@ -412,8 +412,8 @@ Status SniiPlainT2MergePlan::merge_terms( } if (eligibility_.destination_writes_norms) { - // 累加的是原始长度（0..255 饱和）；encode_norm 把 0 映射成 1，与 writer 的 - // encode_norm(len) = clamp(len, 1, 255) 一致。 + // Accumulated raw lengths saturate at 255. encode_norm maps 0 to 1, matching the + // writer's encode_norm(len) = clamp(len, 1, 255). for (size_t destination_ordinal = 0; destination_ordinal < sessions.size(); ++destination_ordinal) { for (uint8_t& value : destination_encoded_norms_[destination_ordinal]) { diff --git a/be/src/storage/index/snii/format/core_metadata.cpp b/be/src/storage/index/snii/format/core_metadata.cpp index 3bc0196965ec35..0f6ef8f5a59cdd 100644 --- a/be/src/storage/index/snii/format/core_metadata.cpp +++ b/be/src/storage/index/snii/format/core_metadata.cpp @@ -72,9 +72,9 @@ Status decode_core_pb(const doris::snii::SniiCoreMetadataPB& input, CoreMetadata !stats.has_null_count()) { return corrupted("core metadata: missing statistics field"); } - // sum_total_term_freq（字段 5）与 norms（字段 5）是后加的可选字段：已上线的生产 - // 3.1 系 writer 不写它们。缺失 = 该段没有打分统计 / 没有 norms， - // 只影响 BM25 打分是否可用，不影响任何过滤查询。 + // sum_total_term_freq (stats field 5) and norms (section_refs field 5) are optional additions + // absent from the deployed 3.1-series writer. Missing fields mean no scoring statistics or + // norms, affecting BM25 availability but not filtering queries. out->stats = {.doc_count = stats.doc_count(), .indexed_doc_count = stats.indexed_doc_count(), .term_count = stats.term_count(), @@ -97,15 +97,16 @@ Status decode_core_pb(const doris::snii::SniiCoreMetadataPB& input, CoreMetadata RETURN_IF_ERROR(decode_region_ref(refs.null_bitmap(), &out->section_refs.null_bitmap)); RETURN_IF_ERROR(decode_region_ref(refs.bsbf(), &out->section_refs.bsbf)); - // 墓碑：CommonGrams 功能已删除。带过字段 4/5 的段只可能是用 CommonGrams analyzer 写出来的 - // （含 gram 词项、键转义或混合 posting 策略），term 键与查询语义都已不可解释，必须重建索引。 - // 生产 writer 从未写过这两个字段，所以升级路径不受影响。 + // Tombstones for the removed CommonGrams feature. Fields 4/5 identify segments written with + // a CommonGrams analyzer (gram terms, escaped keys, or mixed posting policies). Their term + // keys and query semantics are no longer supported, so these indexes must be rebuilt. + // Production writers never emitted these fields, so upgrades are unaffected. if (input.has_legacy_common_grams() || input.has_legacy_common_grams_posting_policy()) { return unsupported( "core metadata: segment was written with CommonGrams, which is no longer " "supported; rebuild the index"); } - // norms（每 doc 一字节的 BM25 文档长度）只对带位置的段有意义：打分的词频来自位置。 + // Norms encode BM25 document lengths in one byte and require positions for term frequencies. if (out->section_refs.norms.length != 0 && !has_positions(out->index_config)) { return corrupted("core metadata: norms require positions"); } @@ -130,7 +131,7 @@ Status encode_core_metadata(const CoreMetadata& metadata, ByteSink* out) { auto* refs = core.mutable_section_refs(); encode_region_ref(metadata.section_refs.dict_region, refs->mutable_dict_region()); encode_region_ref(metadata.section_refs.posting_region, refs->mutable_posting_region()); - // 没有 norms 的段不写字段 5：与生产 writer 的字节形态一致，老 reader 也无需感知。 + // Omit field 5 when norms are absent, matching production bytes without affecting old readers. if (metadata.section_refs.norms.length != 0) { encode_region_ref(metadata.section_refs.norms, refs->mutable_norms()); } diff --git a/be/src/storage/index/snii/format/format_constants.h b/be/src/storage/index/snii/format/format_constants.h index 683acfb16d395e..3073a18930e009 100644 --- a/be/src/storage/index/snii/format/format_constants.h +++ b/be/src/storage/index/snii/format/format_constants.h @@ -68,8 +68,8 @@ enum class SectionType : uint8_t { enum class IndexConfig : uint8_t { kDocsOnly = 0, // docid only: term/match filtering kDocsPositions = 1, // docid+positions: MATCH_PHRASE; BM25 tf = position count - // 2 曾是 kDocsPositionsScoring（CommonGrams 时代的打分 tier），已删除：打分能力由 - // norms region 是否存在决定（见 CoreMetadata::section_refs.norms），reader 拒绝值 2。 + // Value 2 was kDocsPositionsScoring, the removed CommonGrams scoring tier. Scoring now + // depends on the norms region (CoreMetadata::section_refs.norms); readers reject value 2. kPositionsOffsets = 3, // reserved (highlight/RAG), not implemented in this release }; diff --git a/be/src/storage/index/snii/format/phrase_bigram.h b/be/src/storage/index/snii/format/phrase_bigram.h index 743810cc5065d4..0c255e2ba6ba10 100644 --- a/be/src/storage/index/snii/format/phrase_bigram.h +++ b/be/src/storage/index/snii/format/phrase_bigram.h @@ -31,9 +31,9 @@ inline bool is_phrase_bigram_term(std::string_view term) { return term.starts_with(kPhraseBigramTermMarker); } -// SNII 的 term 键就是分词后的原始字节，没有任何转义。唯一的内部命名空间是上面这个以 \x1F -// 开头的 phrase-bigram 标记：用户 term（或前缀展开的前缀）若与它重叠，查询必须绕过 SNII， -// 否则用户词项会命中内部词项。 +// SNII term keys are raw analyzed bytes, without escaping. The only internal namespace is the +// phrase-bigram marker above, starting with \x1F. Queries whose user terms or expansion prefixes +// overlap this marker must bypass SNII to avoid matching internal terms. inline bool term_overlaps_internal_namespace(std::string_view term) { return term.starts_with(kPhraseBigramTermMarker); } diff --git a/be/src/storage/index/snii/format/prx_frame.h b/be/src/storage/index/snii/format/prx_frame.h index 6159330c7bb1a0..84f68381f15527 100644 --- a/be/src/storage/index/snii/format/prx_frame.h +++ b/be/src/storage/index/snii/format/prx_frame.h @@ -35,9 +35,9 @@ struct PrxFrameView { Slice payload; }; -// verify_crc=false 只解析帧头并切出 payload、跳过 crc 比对：给"估算位置工作量"这类 -// 只读元数据的路径用，避免在真正解码之前对整个 prx 窗口多算一遍 crc32c；坏帧仍会在 -// 解码时被 crc 拦下。 +// With verify_crc=false, parse only the header and extract the payload without checking its CRC. +// Metadata-only paths, such as position-work estimation, avoid an extra crc32c pass over the PRX +// window. Decoding still checks the CRC and rejects corrupt frames. Status read_prx_frame(ByteSource* source, PrxFrameView* frame, bool verify_crc = true); } // namespace format diff --git a/be/src/storage/index/snii/query/docid_posting_reader.cpp b/be/src/storage/index/snii/query/docid_posting_reader.cpp index a813334c578982..eaf5560881fc0d 100644 --- a/be/src/storage/index/snii/query/docid_posting_reader.cpp +++ b/be/src/storage/index/snii/query/docid_posting_reader.cpp @@ -110,7 +110,7 @@ Status plan_window_prefix(const LogicalIndexReader& idx, WindowPlan* plan, RETURN_IF_ERROR(validate_windowed_docs_prefix(posting.entry)); uint64_t abs = 0; RETURN_IF_ERROR(prelude_abs(idx, posting.entry, posting.frq_base, &abs)); - // 生产布局：整个 .frq 载荷就是 [prelude][dd-block]，一次 range 读完。 + // Production layout: the entire .frq payload is [prelude][dd-block], read in one range request. plan->prefix_handle = fetcher->add(abs, posting.entry.frq_len); return Status::OK(); } diff --git a/be/src/storage/index/snii/query/internal/phrase_query_split.h b/be/src/storage/index/snii/query/internal/phrase_query_split.h index 822dc00166e4d5..f552543ca6de80 100644 --- a/be/src/storage/index/snii/query/internal/phrase_query_split.h +++ b/be/src/storage/index/snii/query/internal/phrase_query_split.h @@ -173,8 +173,8 @@ struct PhraseTermMapping { PhraseTermMapping build_phrase_term_mapping(const std::vector& terms); -// 把一个已解析的 term 以给定位置偏移追加到 phrase 计划（unique_terms 去重）。定义在 -// phrase_plan.cpp；以前只靠 unity build 的拼接顺序"碰巧"可见。 +// Append a parsed term at the given position offset, deduplicating it in unique_terms. +// Defined in phrase_plan.cpp; visibility previously depended on the unity build's source order. void append_resolved_phrase_clause(ResolvedQueryTerm term, uint32_t position_offset, internal::ResolvedPhrasePlan* plan); diff --git a/be/src/storage/index/snii/query/internal/plain_term_routing.h b/be/src/storage/index/snii/query/internal/plain_term_routing.h index 110aa1950b638d..349fd9754e8c11 100644 --- a/be/src/storage/index/snii/query/internal/plain_term_routing.h +++ b/be/src/storage/index/snii/query/internal/plain_term_routing.h @@ -25,9 +25,9 @@ namespace doris::snii::query::internal { -// SNII 的 term 键就是分词后的原始字节：没有转义、没有版本、没有"物理键 / 逻辑键"之分。 -// 唯一的例外是 \x1F 开头的内部命名空间（phrase-bigram 标记）：用户 term 若与之重叠， -// 整条查询以 INVERTED_INDEX_BYPASS 绕过 SNII，由上层走非索引路径。 +// SNII term keys are raw analyzed bytes, without escaping, versioning, or a physical/logical +// key distinction. The sole exception is the internal phrase-bigram namespace starting with +// \x1F: overlapping user terms return INVERTED_INDEX_BYPASS for execution without the index. inline Status check_term_outside_internal_namespace(std::string_view term) { if (format::term_overlaps_internal_namespace(term)) { return Status::Error( diff --git a/be/src/storage/index/snii/query/phrase_emit.cpp b/be/src/storage/index/snii/query/phrase_emit.cpp index 0d0cf4e46a5323..416a0ea9f7ab5c 100644 --- a/be/src/storage/index/snii/query/phrase_emit.cpp +++ b/be/src/storage/index/snii/query/phrase_emit.cpp @@ -90,7 +90,7 @@ bool should_use_streaming_exact_phrase(const std::vector& plans, } const TermPlan& plan = plans[plan_index]; DORIS_CHECK_NE(plan.df, 0); - // dict entry 不带词频统计：每 doc 的平均位置工作量从已解码的 prx 帧统计得来。 + // Dictionary entries lack term frequencies; estimate position work from decoded PRX frames. DORIS_CHECK_NE(sources[plan_index].logical_position_docs, 0); const uint64_t position_work = sources[plan_index].logical_position_work / sources[plan_index].logical_position_docs; diff --git a/be/src/storage/index/snii/query/phrase_position_source.cpp b/be/src/storage/index/snii/query/phrase_position_source.cpp index f8527a975c668d..c2cabd619149e2 100644 --- a/be/src/storage/index/snii/query/phrase_position_source.cpp +++ b/be/src/storage/index/snii/query/phrase_position_source.cpp @@ -78,7 +78,8 @@ PhraseTermMapping build_phrase_term_mapping(const std::vector& term } namespace { -// 只读帧头估算工作量，不校验 crc：解码时才校验，坏帧在那里被拦下并保留已解码帧的统计。 +// Estimate work from frame headers without checking CRCs. Decoding rejects corrupt frames while +// retaining statistics from frames already decoded. Status accumulate_frame_position_work(Slice frames, uint64_t* work) { ByteSource source(frames); while (!source.eof()) { diff --git a/be/src/storage/index/snii/query/scoring_query.cpp b/be/src/storage/index/snii/query/scoring_query.cpp index e1147cb6d35c17..cd63062cc653c4 100644 --- a/be/src/storage/index/snii/query/scoring_query.cpp +++ b/be/src/storage/index/snii/query/scoring_query.cpp @@ -52,7 +52,7 @@ struct TermPosting { double score = 0.0; }; -// 磁盘上没有词频区：BM25 的 tf 就是该 term 在文档里的位置个数（与 Lucene 系打分定义一致）。 +// With no on-disk frequency region, BM25 tf is the term's position count, as in Lucene scoring. Status require_positions(const LogicalIndexReader& idx) { if (!idx.has_positions()) { return Status::Error( diff --git a/be/src/storage/index/snii/query/term_expansion.cpp b/be/src/storage/index/snii/query/term_expansion.cpp index 4e78c1543e8c9f..ab716ab2ece241 100644 --- a/be/src/storage/index/snii/query/term_expansion.cpp +++ b/be/src/storage/index/snii/query/term_expansion.cpp @@ -45,8 +45,8 @@ Status legacy_raw_prefix_exists(const reader::LogicalIndexReader& idx, std::stri cache); } -// 空前缀会枚举整个词典：段里若存在内部命名空间的词项（phrase-bigram 标记），枚举结果会混入 -// 内部词项，此时整条查询绕过 SNII。 +// An empty prefix enumerates the entire dictionary. If the segment contains internal +// phrase-bigram terms, bypass SNII for the whole query to avoid including those terms. Status prove_no_internal_terms(const reader::LogicalIndexReader& idx, reader::DictBlockCache* cache) { bool exists = false; diff --git a/be/src/storage/index/snii/reader/logical_index_reader.h b/be/src/storage/index/snii/reader/logical_index_reader.h index 1500ebcb117dfd..ee1e90b2571687 100644 --- a/be/src/storage/index/snii/reader/logical_index_reader.h +++ b/be/src/storage/index/snii/reader/logical_index_reader.h @@ -178,7 +178,7 @@ class LogicalIndexReader { const format::StatsBlock& stats() const { return core_.stats; } format::IndexTier tier() const { return tier_; } bool has_positions() const { return has_positions_; } - // 是否带 BM25 norms（分词 + 带位置的索引由新版 writer 写出；老段没有）。 + // Whether BM25 norms exist: current writers emit them for analyzed indexes with positions. bool has_norms() const { return core_.section_refs.norms.length != 0; } LogicalIndexOpenMode open_mode() const { return open_mode_; } io::FileReader* reader() const { return reader_; } diff --git a/be/src/storage/index/snii/snii_index_reader.cpp b/be/src/storage/index/snii/snii_index_reader.cpp index 82d42a7f82da77..bbcb78f7f8449a 100644 --- a/be/src/storage/index/snii/snii_index_reader.cpp +++ b/be/src/storage/index/snii/snii_index_reader.cpp @@ -624,9 +624,10 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::s if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { parse_phrase_slop(&plain_analysis_str, &query_info); } - // 结果缓存只以 (索引文件, 列, 查询类型, 原始查询字节) 为键：分词结果由索引属性与 - // policy 唯一决定（policy 被引用后不可变），因此打开 segment 之前就能判定是否可共享； - // 只有打分查询（结果随集合统计变化）不进缓存，也不走 single-flight 合并。 + // Result cache keys contain only (index file, column, query type, raw query bytes). Analysis + // is determined by index properties and policies, which are immutable once referenced, so + // sharing can be decided before opening the segment. Scoring queries depend on collection + // statistics and use neither the result cache nor single-flight coalescing. const bool allow_result_cache = !actual_similarity; const InvertedIndexRawQuerySemantic raw_semantic {.raw_query_bytes = search_str, .query_type = query_type, diff --git a/be/src/storage/index/snii/snii_index_reader.h b/be/src/storage/index/snii/snii_index_reader.h index 9fb0e188d18678..91996110887854 100644 --- a/be/src/storage/index/snii/snii_index_reader.h +++ b/be/src/storage/index/snii/snii_index_reader.h @@ -37,7 +37,7 @@ struct PhraseMatch; namespace doris::segment_v2 { -// 一次查询在打开 logical reader 之后交给 _compute_query_bitmap 的全部输入。 +// All query inputs passed to _compute_query_bitmap after opening the logical reader. struct SniiQueryBitmapRequest { InvertedIndexQueryType query_type; const InvertedIndexQueryInfo& query_info; diff --git a/be/src/storage/index/snii/snii_index_writer.cpp b/be/src/storage/index/snii/snii_index_writer.cpp index 1720d4deaad45d..94ec6610c0bd42 100644 --- a/be/src/storage/index/snii/snii_index_writer.cpp +++ b/be/src/storage/index/snii/snii_index_writer.cpp @@ -115,9 +115,9 @@ Status SniiIndexColumnWriter::init() { return Status::Error( "SNII create analyzer failed: {}", e.what()); } - // A2：分词 + 带位置的索引一律写 norms（每 doc 的词元数，clamp 到 1..255），与 CLucene 的 - // 打分能力对齐；keyword 或不带位置的索引不写。norms 是 core 元数据里的可选 region， - // 不认识它的老 reader 会原样忽略。 + // A2: Analyzed indexes with positions always write norms (tokens per document, clamped to + // 1..255), matching CLucene's scoring capabilities. Keyword or positionless indexes omit + // them. Norms are an optional core-metadata region ignored by older readers. _writes_norms = _should_analyzer && _has_positions; return Status::OK(); } @@ -253,8 +253,8 @@ Status SniiIndexColumnWriter::add_array_values(size_t field_size, const void* va row_token_count += token_count; } if (_writes_norms) { - // 一行 ARRAY 的文档长度 = 所有元素的词元数之和（NULL 行由 add_array_nulls 声明， - // 但同样经过这里，长度为 0）。 + // An ARRAY row's document length is the total token count across its elements. + // NULL rows also pass here with length 0 and are marked by add_array_nulls. _encoded_norms.push_back(::doris::snii::query::encode_norm(row_token_count)); _report_encoded_norms_capacity(); } diff --git a/be/src/storage/index/snii/snii_index_writer.h b/be/src/storage/index/snii/snii_index_writer.h index 5196e722afeb0f..b17624dbfbfb7c 100644 --- a/be/src/storage/index/snii/snii_index_writer.h +++ b/be/src/storage/index/snii/snii_index_writer.h @@ -93,7 +93,7 @@ class SniiIndexColumnWriter final : public IndexColumnWriter { bool _should_analyzer = false; bool _has_positions = false; const bool _is_char; - // A2：分词 + 带位置的索引一律写 norms（与 CLucene 一致），供 BM25 使用。 + // A2: Analyzed indexes with positions always write BM25 norms, matching CLucene. bool _writes_norms = false; // Latch: set_direct_load() ran. The first call wins; a repeat or late call // is ignored (and logged) so one index keeps one stable compression-tier diff --git a/be/src/storage/index/snii/stats/snii_stats_provider.cpp b/be/src/storage/index/snii/stats/snii_stats_provider.cpp index e13f96a26f6a77..557c9c30417da7 100644 --- a/be/src/storage/index/snii/stats/snii_stats_provider.cpp +++ b/be/src/storage/index/snii/stats/snii_stats_provider.cpp @@ -46,8 +46,9 @@ Status SniiStatsProvider::open(const reader::LogicalIndexReader* idx, SniiStatsP if (idx == nullptr || out == nullptr) { return Status::Error("stats_provider: null argument"); } - // 统计全部来自物理统计：doc_count / indexed_doc_count / sum_total_term_freq 由 writer 对 - // 每个 posting 累加，avgdl 按注释定义用 indexed_doc_count（不含 NULL 行）。 + // All statistics come from physical data collected by the writer: doc_count, + // indexed_doc_count, and sum_total_term_freq. As documented, avgdl uses indexed_doc_count, + // which excludes NULL rows. out->idx_ = idx; const auto& sb = idx->stats(); out->doc_count_ = sb.doc_count; diff --git a/be/src/storage/index/snii/writer/logical_index_writer.h b/be/src/storage/index/snii/writer/logical_index_writer.h index 271975807cd2b2..693d020cc6ede0 100644 --- a/be/src/storage/index/snii/writer/logical_index_writer.h +++ b/be/src/storage/index/snii/writer/logical_index_writer.h @@ -106,8 +106,8 @@ struct SniiIndexInput { // Per-doc 1-byte encoded norm (length doc_count); only consumed when the // config has scoring. May be empty otherwise. std::vector encoded_norms; - // 流式合并会话：norms 在 finish 之前才交付（与 postings 同一趟重建），先用它声明 - // "这个索引会有 norms"，让 writer 在 finalize 时校验长度。 + // Streaming merge sessions declare norms up front but supply them only before finish, + // after rebuilding them alongside postings. The writer validates their size at finalize. bool write_norms = false; // G16-h: zstd levels for the dict-block whole-block compression and the // .prx window auto mode (both default 3 == the historical constants). diff --git a/be/src/storage/index/snii/writer/posting_window_emitter.cpp b/be/src/storage/index/snii/writer/posting_window_emitter.cpp index 130087edbf9766..c2826a83f01bff 100644 --- a/be/src/storage/index/snii/writer/posting_window_emitter.cpp +++ b/be/src/storage/index/snii/writer/posting_window_emitter.cpp @@ -197,8 +197,8 @@ class WindowEmitter::Impl { "window emitter: document frequency overflow"); } stats_.df += static_cast(run.docids.size()); - // 没有 freqs 的 docs-only 输入按每 doc 一次计入总词频；带 freqs 的输入在 - // emit_window_impl 的逐 doc 循环里累加。 + // Docs-only input without freqs contributes one occurrence per document. Input with + // freqs is accumulated in emit_window_impl's per-document loop. if (run.freqs.empty()) { return checked_add(run.docids.size(), &stats_.total_freq); } diff --git a/be/src/storage/index/snii/writer/posting_window_emitter.h b/be/src/storage/index/snii/writer/posting_window_emitter.h index b59e458d9c59da..ed79a92f1bd87e 100644 --- a/be/src/storage/index/snii/writer/posting_window_emitter.h +++ b/be/src/storage/index/snii/writer/posting_window_emitter.h @@ -48,8 +48,9 @@ struct PostingRunView { struct TermAggregateStats { uint32_t df = 0; - // 该 term 的总词频（有位置的 term = 位置个数；docs-only = 文档数或输入 freqs 之和），只进 - // core 元数据的 sum_total_term_freq（BM25 的 avgdl），不进 dict entry。 + // Total frequency of this term: position count, or document count / sum of input freqs for + // docs-only terms. Used only in core metadata's sum_total_term_freq (BM25 avgdl), not in + // dictionary entries. uint64_t total_freq = 0; }; diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.cpp b/be/src/storage/index/snii/writer/snii_compound_writer.cpp index e46a6112f12782..89256f88dd7272 100644 --- a/be/src/storage/index/snii/writer/snii_compound_writer.cpp +++ b/be/src/storage/index/snii/writer/snii_compound_writer.cpp @@ -374,7 +374,7 @@ Status SniiStreamedIndexSession::set_encoded_norms(TrackedEncodedNorms encoded_n "compound: norms length {} differs from doc_count {}", encoded_norms.size(), input_.doc_count); } - // writer_ 持有 input_.encoded_norms 的引用：就地移入即可，finalize 时按引用读取。 + // writer_ references input_.encoded_norms; move into it here for finalize to read by reference. encoded_norms_reservation_ = std::move(encoded_norms.reservation_); input_.encoded_norms = std::move(encoded_norms.norms_); norms_set_ = true; diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.h b/be/src/storage/index/snii/writer/snii_compound_writer.h index f17bcd1482de6f..cb3092d780db81 100644 --- a/be/src/storage/index/snii/writer/snii_compound_writer.h +++ b/be/src/storage/index/snii/writer/snii_compound_writer.h @@ -131,8 +131,9 @@ class SniiStreamedIndexSession { // every rejection is terminal here because posting bytes may already have // entered the compound output; all later calls return the first error. Status push_term(StreamedTermPostings&& tp); - // 交付本目标段的 norms（compaction 在合并 postings 的同一趟里重建）。声明了 - // write_norms 的会话在 finish 之前必须恰好调用一次；长度必须等于 doc_count。 + // Supply this destination segment's norms, rebuilt alongside postings during compaction. + // Sessions declaring write_norms must call this exactly once before finish, with doc_count + // entries. Status set_encoded_norms(TrackedEncodedNorms encoded_norms); // Seals this index: flushes the trailing DICT block, streams the DICT region // right after the posting region and records the placements. A failed finish @@ -222,7 +223,7 @@ class SniiCompoundWriter { // rejected while a session is unfinished, as is finish(). The returned // handle is owned by this writer and valid for its lifetime. Status begin_streamed_index(SniiIndexInput in, SniiStreamedIndexSession** session); - // in.write_norms=true 的会话在 finish 之前必须通过 set_encoded_norms 交付 norms。 + // Sessions with in.write_norms=true must supply norms through set_encoded_norms before finish. Status begin_streamed_index(SniiIndexInput in, TrackedNullDocids null_docids, SniiStreamedIndexSession** session); diff --git a/be/src/storage/index/snii/writer/spill_run_codec.cpp b/be/src/storage/index/snii/writer/spill_run_codec.cpp index f70b2303012fa4..1a669789361467 100644 --- a/be/src/storage/index/snii/writer/spill_run_codec.cpp +++ b/be/src/storage/index/snii/writer/spill_run_codec.cpp @@ -49,8 +49,8 @@ constexpr size_t kWriteFlushBytes = 1u << 22; // 4 MiB // merge-phase peak RSS at low spill thresholds (each reader holds a window). constexpr size_t kReadChunkBytes = 1u << 16; // 64 KiB -// 形状值 0 曾是 CommonGrams 的无频次 docs-only 记录，已删除；run 文件是构建期临时文件， -// 读到 0 一律按损坏处理。 +// Shape 0 was the removed CommonGrams docs-only record without frequencies. Run files are +// temporary build artifacts, so encountering shape 0 is always treated as corruption. enum class RunPostingShape : uint8_t { kDocsAndFreqs = 1, kPositioned = 2, diff --git a/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp b/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp index 50f1ae3346f335..e0e9620e964411 100644 --- a/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp +++ b/be/test/storage/index/inverted/inverted_index_reader_analysis_purpose_test.cpp @@ -303,7 +303,7 @@ class InvertedIndexReaderAnalysisPurposeTest : public testing::Test { EXPECT_EQ(execution.stats.inverted_index_query_cache_miss, 0); EXPECT_EQ(execution.stats.inverted_index_query_cache_lookup, 1); EXPECT_EQ(execution.stats.inverted_index_query_cache_insert, 0); - // 命中发生在打开 logical reader 之前：searcher cache 完全没有被触碰。 + // The hit occurs before opening the logical reader, without touching the searcher cache. EXPECT_EQ(execution.stats.inverted_index_searcher_cache_hit, 0); EXPECT_EQ(execution.stats.inverted_index_searcher_cache_miss, 0); } @@ -365,8 +365,8 @@ TEST(InvertedIndexRawQuerySemanticTest, CacheEnvelopeSeparatesSlashAndNulBoundar EXPECT_NE(nul_left.encode(), nul_right.encode()); } -// 结果缓存的键只有 (索引文件, 列, 查询类型, 原始查询字节)：命中发生在打开 segment 与 -// 任何分词之前，所以即使 analyzer provider 会失败，缓存命中也照常返回。 +// Result cache keys contain only (index file, column, query type, raw query bytes). Hits precede +// opening the segment and analysis, so they succeed even if the analyzer provider would fail. TEST_F(InvertedIndexReaderAnalysisPurposeTest, SniiRawCacheHitHappensBeforeSegmentOpenAndAnalysis) { preload_legacy_searcher_cache_entries(); expect_raw_cache_hit_after_segment_admission( diff --git a/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp b/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp index cb57ea12b03ac7..21b62a02f1898f 100644 --- a/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp +++ b/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp @@ -453,7 +453,7 @@ class CollectionStatisticsTest : public ::testing::Test { return file_writer.finish_close(); } - // 分词 + 带位置 + norms 的普通 SNII 段：这就是新 writer 对可打分索引写出的形态。 + // A normal analyzed SNII segment with positions and norms, as emitted for scoring indexes. Status write_snii_scoring_segment(const std::string& segment_path) { const std::string index_path_prefix { segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; @@ -603,7 +603,7 @@ class CollectionStatisticsTest : public ::testing::Test { return tablet_schema; } - // 落在 SNII 内部命名空间（\x1f 开头）里的词项，对 V3（CLucene）索引只是普通字节。 + // Terms in SNII's internal namespace (starting with \x1f) are ordinary bytes in V3 (CLucene). VExprContextSPtrs create_reserved_exact_search_contexts() { return create_search_contexts("EXACT", std::string(snii::format::kPhraseBigramTermMarker) + "user"); @@ -1090,7 +1090,7 @@ class CollectionStatisticsDetailedTest : public ::testing::Test { std::unique_ptr stats_; }; -// 一个 SNII 段能参与打分的条件只有两个物理事实：带位置、带 norms；统计量直接取 stats 块。 +// SNII scoring requires only positions and norms; statistics come directly from the stats block. TEST(CollectionStatisticsSniiScoringTest, ResolveUsesPhysicalDocAndTokenCounts) { auto result = resolve_snii_scoring_segment(3, 7, /*has_positions=*/true, /*has_norms=*/true); @@ -1132,7 +1132,7 @@ TEST_F(CollectionStatisticsTest, CollectionStatisticsInstancesKeepAdmissionState EXPECT_FLOAT_EQ(second.get_or_calculate_avg_dl(L"1"), 5.0F); } -// 老段（没有 norms）混进来就整体拒绝打分，已收集的统计量一并清空。 +// An older segment without norms disables scoring for the whole collection and clears its stats. TEST_F(CollectionStatisticsTest, SegmentWithoutNormsRejectsWholeCollection) { ASSERT_TRUE(admit_snii_segment_for_test(stats_.get(), L"1", 2, 6).ok()); diff --git a/be/test/storage/index/snii/compaction/snii_compaction_eligibility_test.cpp b/be/test/storage/index/snii/compaction/snii_compaction_eligibility_test.cpp index 007e836d7272e4..bbf5c0493d308a 100644 --- a/be/test/storage/index/snii/compaction/snii_compaction_eligibility_test.cpp +++ b/be/test/storage/index/snii/compaction/snii_compaction_eligibility_test.cpp @@ -211,7 +211,7 @@ TEST(SniiCompactionEligibilityTest, RejectsPhysicalShapesOutsidePlainT2) { } } -// 带 norms 的段（新 writer 对分词 + 带位置索引的产物）与不带 norms 的老段都是合法的 T2 源。 +// Both current segments with norms and older segments without norms are valid T2 sources. TEST(SniiCompactionEligibilityTest, AcceptsSourcesWithOrWithoutNorms) { auto with_norms = open_index(IndexShape {.has_norms = true}); auto without_norms = open_index({}); @@ -223,8 +223,8 @@ TEST(SniiCompactionEligibilityTest, AcceptsSourcesWithOrWithoutNorms) { EXPECT_TRUE(compaction::validate_plain_t2_compaction_eligibility(sources, *destination).ok()); } -// A2：目标索引分词就写 norms（哪怕所有源都是没有 norms 的老段——合并时从 postings 重建）； -// 不分词的 keyword 索引不写。 +// A2: An analyzed destination writes norms, rebuilt from postings even if all sources lack them. +// Keyword indexes without analysis do not write norms. TEST(SniiCompactionEligibilityTest, DestinationWritesNormsExactlyWhenAnalyzed) { auto legacy = open_index({}); diff --git a/be/test/storage/index/snii/compaction/snii_index_compaction_test.cpp b/be/test/storage/index/snii/compaction/snii_index_compaction_test.cpp index 60f2a3cd4efdaa..5cee6dcf1fea9e 100644 --- a/be/test/storage/index/snii/compaction/snii_index_compaction_test.cpp +++ b/be/test/storage/index/snii/compaction/snii_index_compaction_test.cpp @@ -78,8 +78,8 @@ SniiIndexInput make_input(uint32_t doc_count, std::vector null_docids, return input; } -// 带 norms 的 T2 输入（A2：分词 + 带位置的索引一律写 norms）。norms 由调用方给出，必须与 -// postings 的每 doc 词频一致，这样 compaction 从 postings 重建出的 norms 才能与之逐字节相同。 +// T2 input with norms (A2: analyzed indexes with positions always write norms). Caller-supplied +// norms must match per-document posting frequencies so compaction can rebuild identical bytes. SniiIndexInput make_norms_input(uint32_t doc_count, std::vector null_docids, std::vector norms, std::vector terms) { SniiIndexInput input = make_input(doc_count, std::move(null_docids), std::move(terms)); @@ -509,19 +509,20 @@ TEST(SniiIndexCompactionTest, MergesTwentyFourRunSourcesByteIdenticallyToReferen } } -// A2 验收：目标写 norms 时，compaction 不重分词，而是在合并 postings 的同一趟里按每 doc Σfreq -// 重建 norms；结果必须与"用同样 postings 与一致 norms 重新构建"的段逐字节相同。 +// Verify A2: compaction rebuilds destination norms from per-document frequency sums in the +// posting merge pass, without reanalysis. The result must be byte-identical to a segment rebuilt +// from the same postings and matching norms. TEST(SniiIndexCompactionTest, NormsMergeMatchesRebuildAfterDeletesAndRemap) { OpenedIndex source_zero; OpenedIndex source_one; - // 源 0：doc0 = alpha×2，doc1 = alpha×1，doc2（null 但带 posting）= beta×1 → norms {2, 1, 1} + // Source 0: doc0 = alpha*2, doc1 = alpha*1, doc2 (null with a posting) = beta*1; norms {2, 1, 1}. build_index(make_norms_input( /*doc_count=*/3, /*null_docids=*/ {2}, /*norms=*/ {2, 1, 1}, {make_term("alpha", {{.docid = 0, .positions = {0, 2}}, {.docid = 1, .positions = {0}}}), make_term("beta", {{.docid = 2, .positions = {0}}})}), &source_zero, reader::LogicalIndexOpenMode::kCompaction); - // 源 1：doc0 = alpha×1，doc1 = gamma×2 → norms {1, 2} + // Source 1: doc0 = alpha*1, doc1 = gamma*2; norms {1, 2}. build_index(make_norms_input( /*doc_count=*/2, /*null_docids=*/ {}, /*norms=*/ {1, 2}, {make_term("alpha", {{.docid = 0, .positions = {0}}}), @@ -561,7 +562,8 @@ TEST(SniiIndexCompactionTest, NormsMergeMatchesRebuildAfterDeletesAndRemap) { assert_ok(compound->finish()); } - // 目标 0 = [源0 doc0, 源1 doc0, 源1 doc1] → norms {2, 1, 2}；目标 1 = [源0 doc2] → {1} + // Destination 0 = [source0 doc0, source1 doc0, source1 doc1], norms {2, 1, 2}. + // Destination 1 = [source0 doc2], norms {1}. std::array rebuilt; build_index(make_norms_input( /*doc_count=*/3, /*null_docids=*/ {}, /*norms=*/ {2, 1, 2}, @@ -603,8 +605,9 @@ TEST(SniiIndexCompactionTest, NormsMergeMatchesRebuildAfterDeletesAndRemap) { EXPECT_EQ(merged_docs, rebuilt_docs); } -// 老段（没有 norms 的 T2）参与 compaction 时同样能重建出 norms：这是生产升级后无需重建索引就 -// 获得打分能力的路径。空文档（无任何 token）编码为 1，超过 255 个 token 饱和到 255。 +// Compaction also rebuilds norms from older T2 segments without norms, enabling scoring after +// a production upgrade without rebuilding indexes. Empty documents encode as 1; token counts +// above 255 saturate at 255. TEST(SniiIndexCompactionTest, NormsAreReconstructedFromLegacySourcesWithSaturation) { std::vector long_positions(300); std::iota(long_positions.begin(), long_positions.end(), 0U); @@ -678,7 +681,7 @@ TEST(SniiIndexCompactionTest, NormsMergeReclaimsResidentDictBeforeLargePlainTerm source_terms.push_back(make_term( "zeta", {{.docid = 0, .positions = make_positions(kPlainPositions, /*salt=*/99)}})); - // 单 doc 里的 token 远超 255，norm 饱和到 255。 + // The document contains far more than 255 tokens, so its norm saturates at 255. SniiIndexInput source_input = make_norms_input(/*doc_count=*/1, /*null_docids=*/ {}, /*norms=*/ {255}, std::move(source_terms)); source_input.target_dict_block_bytes = 1; diff --git a/be/test/storage/index/snii/compaction/snii_posting_cursor_test.cpp b/be/test/storage/index/snii/compaction/snii_posting_cursor_test.cpp index 9e05de6502387c..effcb6d90d068c 100644 --- a/be/test/storage/index/snii/compaction/snii_posting_cursor_test.cpp +++ b/be/test/storage/index/snii/compaction/snii_posting_cursor_test.cpp @@ -57,7 +57,7 @@ static_assert(!std::is_constructible_v< constexpr uint64_t kIndexId = 9; constexpr std::string_view kIndexSuffix = "body"; -constexpr uint32_t kDocCount = 20000; // slim 项的 docid 间距不规则，最大约 1.6 万 +constexpr uint32_t kDocCount = 20000; // Slim docids have irregular gaps, reaching about 16,000. constexpr uint32_t kFreqDroppedDocCount = 65536; constexpr auto kDeleted = std::pair {std::numeric_limits::max(), std::numeric_limits::max()}; @@ -95,8 +95,8 @@ Status build_source(std::vector terms, uint32_t doc_count, return fixture->segment.open_index(kIndexId, kIndexSuffix, &fixture->index); } -// 500 个 docid，间距 1..62 不规则：dd 区（PFOR）超过 256B 的 inline 阈值， -// 且 df < 512，仍是 slim pod_ref。docid 0 一定在列表里。 +// 500 docids with irregular gaps of 1..62: the PFOR dd region exceeds the 256-byte inline +// threshold, while df < 512 still selects a slim pod_ref. The list always includes docid 0. std::vector slim_docids() { std::vector docids; docids.reserve(500); @@ -110,7 +110,8 @@ std::vector slim_docids() { return docids; } -// slim 项每个 doc 的位置只由 docid 决定（哈希取 1..97 个位置），测试据此重算期望。 +// Each slim document's positions depend only on its docid (hashed to 1..97 positions), allowing +// the test to recompute the expected values. std::vector slim_positions(uint32_t docid) { uint32_t mixed = docid * 2654435761U; mixed ^= mixed >> 16; diff --git a/be/test/storage/index/snii/compaction/snii_streamed_session_test.cpp b/be/test/storage/index/snii/compaction/snii_streamed_session_test.cpp index ba424c5c6d3a52..eeb6d6e1f454b5 100644 --- a/be/test/storage/index/snii/compaction/snii_streamed_session_test.cpp +++ b/be/test/storage/index/snii/compaction/snii_streamed_session_test.cpp @@ -384,7 +384,7 @@ Status begin_scoring_session_from_local_input(SniiCompoundWriter* compound, input.config = format::IndexConfig::kDocsPositions; input.doc_count = 4; input.null_docids = {1, 3}; - // 流式会话的 norms 只能晚绑定（compaction 在合并 postings 之后才知道每 doc 长度）。 + // Streaming norms are bound late: compaction learns document lengths after merging postings. input.write_norms = true; return compound->begin_streamed_index(std::move(input), session); } @@ -1125,14 +1125,15 @@ TEST(SniiStreamedWriterSessionTest, ActiveAndFinishedSessionLifecycleIsEnforced) EXPECT_FALSE(exists); } -// A2：compaction 的目标会话先声明 write_norms，postings 合并完毕后才把重建出来的 norms 交给 -// 会话；必须恰好交付一次、长度等于 doc_count，finish 之前缺失则整个 compound 中毒。 +// A2: A compaction destination declares write_norms before merging postings, then receives the +// rebuilt norms exactly once, with doc_count entries. Missing norms at finish poison the entire +// compound writer. TEST(SniiStreamedWriterSessionTest, EncodedNormsAreLateBoundExactlyOnceBeforeFinish) { MemoryFile file; SniiCompoundWriter compound(&file); SniiStreamedIndexSession* session = nullptr; SniiIndexInput input = empty_input(105, "late_norms", /*doc_count=*/2); - // norms 需要词频（BM25 的 tf 来自位置计数，存储上要求 freq 区存在）。 + // Norms require frequencies: BM25 tf comes from position counts and requires a freq region. input.write_norms = true; assert_ok(compound.begin_streamed_index(std::move(input), &session)); ASSERT_NE(session, nullptr); diff --git a/be/test/storage/index/snii/format/core_metadata_test.cpp b/be/test/storage/index/snii/format/core_metadata_test.cpp index 809a9426da02bc..ce717fe9477d72 100644 --- a/be/test/storage/index/snii/format/core_metadata_test.cpp +++ b/be/test/storage/index/snii/format/core_metadata_test.cpp @@ -46,7 +46,7 @@ CoreMetadata sample_core(IndexConfig index_config = IndexConfig::kDocsOnly) { .norms = {}, .null_bitmap = {.offset = 41, .length = 42}, .bsbf = {.offset = 51, .length = 52}}; - // norms 只对带位置的段合法（BM25 的词频来自位置）；docs-only 样本不带 norms。 + // Norms require positions for BM25 frequencies; docs-only samples omit them. if (has_positions(index_config)) { metadata.section_refs.norms = {.offset = 31, .length = 32}; } @@ -109,7 +109,7 @@ void expect_core_eq(const CoreMetadata& expected, const CoreMetadata& actual) { EXPECT_EQ(expected.section_refs.bsbf.length, actual.section_refs.bsbf.length); } -// docs-only 段不能带 norms（norms 需要位置）：这是 A2 之后 core 元数据的一条硬约束。 +// Since A2, core metadata forbids norms on docs-only segments: norms require positions. TEST(SniiCoreMetadata, RejectsNormsOnDocsOnlyIndex) { auto metadata = sample_core(); metadata.section_refs.norms = {.offset = 31, .length = 32}; @@ -132,11 +132,11 @@ TEST(SniiCoreMetadata, RoundTripsPositions) { expect_core_eq(expected, actual); } -// CommonGrams 已删除：写过字段 4（CommonGrams 元数据）或字段 5（posting 策略）的段是墓碑， -// 必须重建索引。生产 writer 从未写过这两个字段。 +// CommonGrams was removed: fields 4 (CommonGrams metadata) and 5 (posting policy) mark unsupported +// segments that require an index rebuild. Production writers never emitted these fields. TEST(SniiCoreMetadata, RejectsLegacyCommonGramsMetadataFieldAsUnsupported) { auto payload = payload_of(encode(sample_core(IndexConfig::kDocsPositions))); - // 字段 4，length-delimited，任意内容。 + // Field 4, length-delimited, with arbitrary contents. payload.push_back(static_cast((4u << 3) | 2u)); payload.push_back(3); payload.insert(payload.end(), {'c', 'g', '1'}); @@ -193,8 +193,9 @@ TEST(SniiCoreMetadata, RejectsUnsupportedIndexConfig) { EXPECT_TRUE(status.is()) << status; } -// master 开发期曾用 index_config=2 表示"带打分 tier"；打分能力现在由 norms region 表达， -// 该取值不再有意义，按不支持拒绝（从未进入生产，不存在兼容负担）。 +// During development on master, index_config=2 denoted a scoring tier. The norms region now +// determines scoring capability, so reject this obsolete value. It was never deployed to +// production and has no compatibility requirements. TEST(SniiCoreMetadata, RejectsLegacyScoringIndexConfigAsUnsupported) { auto payload = payload_of(encode(sample_core())); ByteSink field; @@ -207,8 +208,8 @@ TEST(SniiCoreMetadata, RejectsLegacyScoringIndexConfigAsUnsupported) { EXPECT_TRUE(status.is()) << status; } -// 已上线的 3.1 系生产 writer 不写 stats.sum_total_term_freq 与 -// section_refs.norms：这种形态必须能打开，缺失的统计按 0、norms 按空 region 处理。 +// The deployed 3.1-series writer omits stats.sum_total_term_freq and section_refs.norms. Such +// segments must remain readable, with missing statistics set to 0 and an empty norms region. TEST(SniiCoreMetadata, AcceptsProductionShapeWithoutNormsAndTotalTermFreq) { const auto metadata = sample_core(IndexConfig::kDocsPositions); const auto payload = mutate_core_payload(metadata, [](auto* core) { @@ -226,7 +227,7 @@ TEST(SniiCoreMetadata, AcceptsProductionShapeWithoutNormsAndTotalTermFreq) { EXPECT_EQ(actual.section_refs.bsbf.length, metadata.section_refs.bsbf.length); } -// 没有 norms 的段编码时不写 section_refs.norms（与生产 writer 的字节形态一致）。 +// Segments without norms omit section_refs.norms, matching the production writer's bytes. TEST(SniiCoreMetadata, OmitsEmptyNormsRefOnEncode) { auto metadata = sample_core(IndexConfig::kDocsPositions); metadata.section_refs.norms = {}; @@ -244,7 +245,7 @@ TEST(SniiCoreMetadata, OmitsEmptyNormsRefOnEncode) { TEST(SniiCoreMetadata, RejectsMissingEachStatsField) { const auto metadata = sample_core(); - // sum_total_term_freq 是可选字段（生产 writer 不写），不在必填之列。 + // sum_total_term_freq is optional and omitted by the production writer. for (const auto clear : std::array { &doris::snii::SniiStatsPB::clear_doc_count, &doris::snii::SniiStatsPB::clear_indexed_doc_count, diff --git a/be/test/storage/index/snii/query/phrase_prefix_query_test.cpp b/be/test/storage/index/snii/query/phrase_prefix_query_test.cpp index 0fd31b5d80f15a..2ce64fb114e8fa 100644 --- a/be/test/storage/index/snii/query/phrase_prefix_query_test.cpp +++ b/be/test/storage/index/snii/query/phrase_prefix_query_test.cpp @@ -333,15 +333,15 @@ Corpus BuildCrossWindowTailCorpus() { // is byte-wise, matching the index's dict enumeration order. Corpus BuildCjkTailCorpus() { Corpus c; - const char* const tails[] = {"\xE7\xBB\x93\xE6\x9E\x9C\xE7\x94\xB2", // 结果甲 - "\xE7\xBB\x93\xE6\x9E\x9C\xE4\xB9\x99", // 结果乙 - "\xE7\xBB\x93\xE6\x9E\x9C\xE4\xB8\x99"}; // 结果丙 - const std::string lead = "\xE8\xBF\x9E\xE6\x8E\xA5"; // 连接 + const char* const tails[] = {"\xE7\xBB\x93\xE6\x9E\x9C\xE7\x94\xB2", // Result A + "\xE7\xBB\x93\xE6\x9E\x9C\xE4\xB9\x99", // Result B + "\xE7\xBB\x93\xE6\x9E\x9C\xE4\xB8\x99"}; // Result C + const std::string lead = "\xE8\xBF\x9E\xE6\x8E\xA5"; // Join c.docs.resize(120); for (uint32_t d = 0; d < c.docs.size(); ++d) { if (d % 20 == 3) { c.docs[d] = {lead, "\xE9\x97\xB4\xE9\x9A\x94", - tails[d % 3]}; // 间隔 filler, not adjacent + tails[d % 3]}; // Separating filler, not adjacent } else { c.docs[d] = {lead, tails[d % 3]}; } @@ -966,8 +966,8 @@ TEST(SniiPhrasePrefixMerge, CjkUnicodeTailsMatchOracle) { SniiSegmentReader segment; LogicalIndexReader idx = OpenIndex(&file, &segment, path); - const std::vector terms = {"\xE8\xBF\x9E\xE6\x8E\xA5", // 连接 - "\xE7\xBB\x93\xE6\x9E\x9C"}; // 结果 + const std::vector terms = {"\xE8\xBF\x9E\xE6\x8E\xA5", // Join + "\xE7\xBB\x93\xE6\x9E\x9C"}; // Result std::vector got; ASSERT_TRUE(query::phrase_prefix_query(idx, terms, &got).ok()); EXPECT_TRUE(std::ranges::is_sorted(got)); diff --git a/be/test/storage/index/snii/query/query_operator_error_test.cpp b/be/test/storage/index/snii/query/query_operator_error_test.cpp index 7773fab90afe44..4083c222f8b08f 100644 --- a/be/test/storage/index/snii/query/query_operator_error_test.cpp +++ b/be/test/storage/index/snii/query/query_operator_error_test.cpp @@ -128,7 +128,7 @@ void BuildIndexBytes(const Corpus& corpus, doris::snii::format::IndexConfig conf in.config = config; in.doc_count = static_cast(corpus.docs.size()); if (config == doris::snii::format::IndexConfig::kDocsPositions) { - // 带位置的索引一并写 norms（A2），让打分查询在这些边界用例里也可用。 + // Indexes with positions also write norms (A2), enabling scoring in these edge cases. in.encoded_norms.assign(corpus.docs.size(), 1); } in.terms = buf.finalize_sorted(); diff --git a/be/test/storage/index/snii/query/query_profile_test.cpp b/be/test/storage/index/snii/query/query_profile_test.cpp index 4a44493c43de9e..ec8d32db3538b5 100644 --- a/be/test/storage/index/snii/query/query_profile_test.cpp +++ b/be/test/storage/index/snii/query/query_profile_test.cpp @@ -206,7 +206,7 @@ void WriteCorpus(const Corpus& c, const std::string& path, int prx_zstd_level = in.index_suffix = "body"; in.config = doris::snii::format::IndexConfig::kDocsPositions; in.doc_count = static_cast(c.docs.size()); - // 分词 + 带位置的索引一律带 norms（A2），这样才能打分。 + // Analyzed indexes with positions always include norms (A2) to enable scoring. in.encoded_norms.assign(c.docs.size(), 1); in.terms = buf.finalize_sorted(); in.target_dict_block_bytes = 512; diff --git a/be/test/storage/index/snii/snii_golden_corpus_test.cpp b/be/test/storage/index/snii/snii_golden_corpus_test.cpp index dadead111062ac..aaabaeefe97d49 100644 --- a/be/test/storage/index/snii/snii_golden_corpus_test.cpp +++ b/be/test/storage/index/snii/snii_golden_corpus_test.cpp @@ -15,20 +15,20 @@ // specific language governing permissions and limitations // under the License. -// SNII 黄金语料回读测试（跨版本 / 跨改动的查询语义锁定）。 +// SNII golden-corpus readback test: preserve query semantics across versions and changes. // -// 用途：把一组「普通」SNII 段（无 CommonGrams、无打分扩展）用生产写入栈写到磁盘， -// 并把每条查询在写入时的结果（docid 集合、null bitmap、状态码）记成期望文件； -// 之后任何改动（删 CommonGrams、格式对齐、norms 写入……）都必须能原样回读这些段， -// 且每条查询结果逐条一致。段文件也可以由其它版本的 writer（例如生产分支）写出， -// 只要目录里有同名的 .expect 即可校验。 +// Write ordinary SNII segments without CommonGrams or scoring extensions through the production +// writer stack. Record each query's docids, null bitmap, and status codes as expectations. +// Later changes, including CommonGrams removal, format alignment, or norms support, must preserve +// readback and query results. Other writer versions, such as a production branch, may also supply +// segment files with matching .expect files in the same directory. // -// SNII_GOLDEN_DIR= SNII_GOLDEN_MODE=write 写段 + 写期望 -// SNII_GOLDEN_DIR= 回读校验（默认） -// 未设置 SNII_GOLDEN_DIR 跳过 +// SNII_GOLDEN_DIR= SNII_GOLDEN_MODE=write Write segments and expectations. +// SNII_GOLDEN_DIR= Verify readback (default). +// SNII_GOLDEN_DIR unset Skip the test. // -// 校验模式下每条查询跑三遍：结果缓存关闭（冷）、缓存开启（冷）、缓存开启（热）， -// 三遍都必须等于期望。 +// Verification runs each query with the result cache disabled, enabled but cold, and warm. +// All three results must match the expectations. #include #include @@ -69,30 +69,30 @@ namespace { constexpr int64_t kIndexId = 7001; constexpr const char* kColumn = "c1"; -// ---------------------------------------------------------------- 语料 +// ---------------------------------------------------------------- Corpus using ScalarRow = std::optional; // nullopt = NULL std::vector scalar_corpus() { std::vector rows; - rows.emplace_back("hello world hello doris"); // 0 - rows.emplace_back(""); // 1 空串 - rows.emplace_back("The QUICK brown-fox; jumped!! over_the lazy dog 42 times"); // 2 + rows.emplace_back("hello world hello doris"); // 0 + rows.emplace_back(""); // 1 Empty string + rows.emplace_back("The QUICK brown-fox; jumped!! over_the lazy dog 42 times"); // 2 rows.emplace_back("重复 重复 重复 词元 Doris 数据库 全文检索 mixed 中英 tokens"); // 3 rows.emplace_back(std::nullopt); // 4 NULL rows.emplace_back(std::nullopt); // 5 NULL rows.emplace_back(std::nullopt); // 6 NULL rows.emplace_back(std::string(300, 'x')); // 7 > ignore_above rows.emplace_back("single"); // 8 - rows.emplace_back("!!! ??? ,,,"); // 9 无词元 + rows.emplace_back("!!! ??? ,,,"); // 9 No tokens rows.emplace_back("hello world again and again and again"); // 10 rows.emplace_back(std::nullopt); // 11 NULL - rows.emplace_back(std::string("\x1f") + "hidden term inside"); // 12 内部命名空间前缀 - rows.emplace_back(std::string("\x1e") + "escaped start"); // 13 转义前缀 + rows.emplace_back(std::string("\x1f") + "hidden term inside"); // 12 Control-byte prefix + rows.emplace_back(std::string("\x1e") + "escaped start"); // 13 Escape control byte rows.emplace_back("prefix prefixes prefixing prefab"); // 14 rows.emplace_back("alpha beta gamma alpha beta alpha"); // 15 { - std::string long_doc; // 16 超过 255 个词元（norm 饱和） + std::string long_doc; // 16 More than 255 tokens (norm saturation) for (int i = 0; i < 300; ++i) long_doc += (i ? " tok" : "tok"); rows.emplace_back(std::move(long_doc)); } @@ -162,7 +162,7 @@ struct Sample { std::string name; std::map properties; bool keyword_lane; - bool array = false; // ARRAY 列：经 add_array_values / add_array_nulls 写入 + bool array = false; // ARRAY, written via add_array_values / add_array_nulls. }; std::vector samples() { @@ -185,27 +185,27 @@ std::vector samples() { }; } -// ARRAY 语料：每行是若干元素；nullopt 行 = 整行 NULL；元素级 NULL 用 std::nullopt 元素表示。 +// ARRAY corpus: an absent row is SQL NULL; absent elements represent element-level NULLs. using ArrayRow = std::optional>>; std::vector array_corpus() { std::vector rows; rows.emplace_back(std::vector> {"hello world", "hello doris"}); // 0 - rows.emplace_back(std::vector> {}); // 1 空数组 - rows.emplace_back(std::nullopt); // 2 NULL 行 + rows.emplace_back(std::vector> {}); // 1 Empty array + rows.emplace_back(std::nullopt); // 2 NULL row rows.emplace_back(std::vector> {"single"}); // 3 - rows.emplace_back(std::vector> {"alpha beta", std::nullopt, - "gamma alpha"}); // 4 含元素 NULL - rows.emplace_back( - std::vector> {"world", "hello"}); // 5 跨元素不成短语 + rows.emplace_back(std::vector> { + "alpha beta", std::nullopt, "gamma alpha"}); // 4 Contains a NULL element + rows.emplace_back(std::vector> { + "world", "hello"}); // 5 Phrases do not span elements rows.emplace_back(std::vector> {"重复 词元", "Doris 数据库"}); // 6 - rows.emplace_back(std::nullopt); // 7 NULL 行 + rows.emplace_back(std::nullopt); // 7 NULL row rows.emplace_back(std::vector> {"prefix prefixes", "", "hello world hello doris"}); // 8 return rows; } -// ---------------------------------------------------------------- 工具 +// ---------------------------------------------------------------- Helpers TabletIndex make_meta(const Sample& sample) { TabletIndexPB pb; @@ -281,8 +281,8 @@ struct Observation { InvertedIndexQueryType type; std::string text; int status_code = 0; - std::string docids; // 逗号分隔 - std::string null_docids; // 逗号分隔 + std::string docids; // Comma-separated. + std::string null_docids; // Comma-separated. int null_status_code = 0; std::string line() const { @@ -319,7 +319,7 @@ std::optional parse_line(const std::string& sample, const std::stri return o; } -// 查询上下文：仿生产的 IndexQueryContext 装配（enable_query_cache 可选）。 +// Assemble IndexQueryContext as in production, with optional query-cache support. struct QueryEnv { explicit QueryEnv(bool enable_query_cache) { TQueryOptions options; @@ -337,7 +337,7 @@ struct QueryEnv { IndexQueryContextPtr context = std::make_shared(); }; -// ---------------------------------------------------------------- 写入 +// ---------------------------------------------------------------- Writing Status write_sample(const std::string& dir, const Sample& sample, const TabletIndex& meta) { const std::string prefix = dir + "/" + sample.name; @@ -354,8 +354,8 @@ Status write_sample(const std::string& dir, const Sample& sample, const TabletIn SniiIndexColumnWriter writer(&index_file_writer, &meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); RETURN_IF_ERROR(writer.init()); if (sample.array) { - // 仿 ArrayColumnWriter::append_nullable：所有行（含 NULL 行）都喂给 add_array_values， - // NULL 行是空数组；行级 NULL 再通过 add_array_nulls 声明。 + // Follow ArrayColumnWriter::append_nullable: pass all rows to add_array_values, + // representing NULL rows as empty arrays, then mark them with add_array_nulls. const auto rows = array_corpus(); std::vector storage; std::vector element_nulls; @@ -409,7 +409,7 @@ Status write_sample(const std::string& dir, const Sample& sample, const TabletIn return Status::OK(); } -// ---------------------------------------------------------------- 读取 + 查询 +// ---------------------------------------------------------------- Reading and querying struct OpenedSample { std::shared_ptr file_reader; @@ -459,7 +459,7 @@ class SniiGoldenCorpus : public testing::Test { void SetUp() override { const char* dir = std::getenv("SNII_GOLDEN_DIR"); if (dir == nullptr || *dir == '\0') { - GTEST_SKIP() << "SNII_GOLDEN_DIR 未设置"; + GTEST_SKIP() << "SNII_GOLDEN_DIR is not set"; } _dir = dir; const char* mode = std::getenv("SNII_GOLDEN_MODE"); @@ -506,16 +506,16 @@ TEST_F(SniiGoldenCorpus, WriteOrVerify) { continue; } - // 校验模式：期望文件缺失 = 该样本不存在（可能由其它版本 writer 未写出），跳过并提示。 + // Report missing expectations as a failure, then continue checking the remaining samples. std::ifstream in(expect_path, std::ios::binary); if (!in.good()) { - ADD_FAILURE() << "缺少期望文件: " << expect_path; + ADD_FAILURE() << "Missing expectation file: " << expect_path; continue; } OpenedSample opened; const Status os = open_sample(_dir, sample, meta, &opened); if (!os.ok()) { - ADD_FAILURE() << sample.name << " 打开失败: " << os.to_string(); + ADD_FAILURE() << sample.name << " failed to open: " << os.to_string(); ++mismatches; continue; } @@ -524,7 +524,8 @@ TEST_F(SniiGoldenCorpus, WriteOrVerify) { while (std::getline(in, line)) { if (line.empty() || line[0] == '#') continue; auto parsed = parse_line(sample.name, line); - ASSERT_TRUE(parsed.has_value()) << "期望文件格式错误: " << expect_path << ": " << line; + ASSERT_TRUE(parsed.has_value()) + << "Invalid expectation file format: " << expect_path << ": " << line; expected.push_back(*parsed); } for (const Observation& want : expected) { @@ -540,14 +541,14 @@ TEST_F(SniiGoldenCorpus, WriteOrVerify) { got->null_docids == want.null_docids; if (!same) { ++mismatches; - ADD_FAILURE() << want.describe() << "\n 期望: " << want.line() - << "\n 实际: " << got->line(); + ADD_FAILURE() << want.describe() << "\n Expected: " << want.line() + << "\n Actual: " << got->line(); } } } } if (!_write_mode) { - EXPECT_EQ(mismatches, 0) << "共校验 " << checked << " 条"; + EXPECT_EQ(mismatches, 0) << "Checked " << checked << " observations"; std::cout << "[golden] checked=" << checked << " mismatches=" << mismatches << std::endl; } } diff --git a/be/test/storage/index/snii/writer/phase_a_readback_test.cpp b/be/test/storage/index/snii/writer/phase_a_readback_test.cpp index 5e841157cce1d7..471d0d04d098f3 100644 --- a/be/test/storage/index/snii/writer/phase_a_readback_test.cpp +++ b/be/test/storage/index/snii/writer/phase_a_readback_test.cpp @@ -114,7 +114,7 @@ SniiIndexInput MakeIndex(const Corpus& c) { SniiIndexInput in; in.index_id = 1; in.index_suffix = "body"; - in.config = IndexConfig::kDocsPositions; // norms 随 encoded_norms 一起写出 + in.config = IndexConfig::kDocsPositions; // Norms are written from encoded_norms. in.doc_count = c.doc_count; in.target_dict_block_bytes = 1; // one block per term in.encoded_norms.resize(c.doc_count); diff --git a/be/test/storage/index/snii/writer/snii_compound_writer_test.cpp b/be/test/storage/index/snii/writer/snii_compound_writer_test.cpp index 6c9dd56d1f700a..1fd58df5042565 100644 --- a/be/test/storage/index/snii/writer/snii_compound_writer_test.cpp +++ b/be/test/storage/index/snii/writer/snii_compound_writer_test.cpp @@ -1537,9 +1537,10 @@ void ExpectFixtureReadable(const std::vector& file) { } // namespace -// 夹具尺寸随格式演进漂移（core 元数据字段的增减就会改动几十字节）。从"跨 2*kMinPaddingLeverage -// 个块"的导出值往下找第一个"确实要补齐、且补齐便宜（2*pad < block）"的块大小：块越小跨的块越多， -// 阈值门只会更宽松，所以门不会成为这些用例的决定因素。 +// Fixture sizes vary with the format; core-metadata fields alone can shift them by dozens of +// bytes. Starting from a size spanning 2*kMinPaddingLeverage blocks, search downward for a block +// size that needs padding and satisfies 2*pad < block. Smaller blocks only relax the block-count +// threshold, keeping that threshold from determining these test results. int64_t CheapPaddingBlockSize(size_t unpadded) { auto block = static_cast(unpadded / (2 * kMinPaddingLeverage)); while (block >= 2) { diff --git a/be/test/storage/index/snii/writer/spill_run_codec_test.cpp b/be/test/storage/index/snii/writer/spill_run_codec_test.cpp index 5f1be4a0670c2e..dbefdc7f77b976 100644 --- a/be/test/storage/index/snii/writer/spill_run_codec_test.cpp +++ b/be/test/storage/index/snii/writer/spill_run_codec_test.cpp @@ -358,7 +358,7 @@ TEST(SniiSpillRunCodec, RunWriterStreamsWideTermWithinAccountedBound) { TempRun run; std::vector docids(kDocs); std::iota(docids.begin(), docids.end(), 0U); - // 生产 SPIMI 落盘的 postings 一定带 freqs（to_postings 总会填充），run 记录没有无频次形状。 + // SPIMI spill postings always contain freqs populated by to_postings; run records require them. TermPostings postings = MakeTerm(std::move(docids), std::vector(kDocs, 1)); int64_t observed = 0; diff --git a/be/test/storage/index/snii_query_test.cpp b/be/test/storage/index/snii_query_test.cpp index e16b7c577a4aba..4ccf4628462add 100644 --- a/be/test/storage/index/snii_query_test.cpp +++ b/be/test/storage/index/snii_query_test.cpp @@ -901,8 +901,9 @@ struct OpaqueMatcherPlanCase { std::vector common; }; -// 内部命名空间（\x1f 开头）里的不透明词项：这些用例直接驱动 phrase 计划执行器， -// 验证"覆盖两个位置的不透明词项"与普通位置匹配的等价性，与任何具体分析器无关。 +// Opaque terms in the internal namespace (starting with \x1f). These tests drive the phrase-plan +// executor directly to verify that an opaque term spanning two positions matches ordinary +// positions equivalently, independently of any particular analyzer. constexpr std::string_view kOpaqueInternalTermPrefix = "\x1f" "SNII_TEST_OPAQUE" diff --git a/be/test/storage/index/snii_spimi_intern_test.cpp b/be/test/storage/index/snii_spimi_intern_test.cpp index 54381a1afb4e67..756f8c9f2a41cd 100644 --- a/be/test/storage/index/snii_spimi_intern_test.cpp +++ b/be/test/storage/index/snii_spimi_intern_test.cpp @@ -110,7 +110,7 @@ TEST(SniiSpimiTermBufferTest, InternHashUsesFastStringViewHash) { } TEST(SniiSpimiTermBufferTest, OrdinaryDocsOnlyMarkerTermRetainsFrequency) { - // 落在内部命名空间（\x1f 开头）里的普通词项也只是普通词项：照常记录频次。 + // Ordinary terms starting with the internal \x1f namespace still record frequencies normally. const std::string literal_marker_term = std::string("\x1f") + "SNII_TEST_MARKER\x1f" + "literal"; SpimiTermBuffer ordinary(/*has_positions=*/false); diff --git a/be/test/storage/index/snii_writer_test.cpp b/be/test/storage/index/snii_writer_test.cpp index 77f08840065759..2451c8aa578ff6 100644 --- a/be/test/storage/index/snii_writer_test.cpp +++ b/be/test/storage/index/snii_writer_test.cpp @@ -96,8 +96,8 @@ void init_failure_index_meta(doris::TabletIndex* index_meta, int64_t index_id) { index_meta->init_from_pb(index_pb); } -// 分词过程中抛 INVERTED_INDEX_ANALYZER_ERROR 的分析器：模拟任何 token filter 的运行期失败 -// （以前由某个词元过滤器的 UTF-8 校验扮演这个角色）。 +// An analyzer that throws INVERTED_INDEX_ANALYZER_ERROR to simulate a token filter's runtime +// failure. Previously, a token filter's UTF-8 validation provided this failure path. class ThrowingTokenStream final : public lucene::analysis::TokenStream { public: lucene::analysis::Token* next(lucene::analysis::Token*) override { diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicy.java index 1630442c63ede1..31d51a06c9917d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicy.java @@ -123,8 +123,8 @@ public List getShowInfo() { GsonUtils.GSON.toJson(this.properties)); } - // 已从 BE 删除、但老版本镜像/edit log 里可能仍持久化着的 token filter 类型。 - // 这类策略可以被加载（否则 FE 起不来），但不可再被 analyzer 引用。 + // Token filter types removed from BE but possibly retained in older images or edit logs. + // Load these policies so FE can start, but reject analyzers that reference them. public static final Set LEGACY_UNSUPPORTED_TOKEN_FILTER_TYPES = ImmutableSet.of("common_grams"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java index 9f71a140c136c1..b547c482d81452 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyMgr.java @@ -117,9 +117,10 @@ public void validateAnalyzerExists(String analyzerName) throws DdlException { } /** - * 安全网：老版本可能持久化了 BE 已不再支持的 token filter 类型（例如已删除的 common_grams）。 - * 这类策略仍会被加载（不能让 FE 因为镜像里的一条策略起不来），但任何引用它的 analyzer - * 都必须在使用时被明确拒绝，而不是等到 BE 建索引/查询时才报"未知 token filter"。 + * Older metadata may retain token filter types no longer supported by BE, such as common_grams. + * Load these policies so a single obsolete policy cannot prevent FE startup, but reject any + * analyzer that references them at use time, before BE reports an unknown token filter during + * index construction or querying. */ private void validateReferencedTokenFiltersUsableLocked(String analyzerName, IndexPolicy analyzer) throws DdlException { diff --git a/gensrc/proto/snii.proto b/gensrc/proto/snii.proto index 188ddc9e5ea3d9..1dc6793b1e2798 100644 --- a/gensrc/proto/snii.proto +++ b/gensrc/proto/snii.proto @@ -77,9 +77,9 @@ message SniiCoreMetadataPB { optional uint32 index_config = 1; optional SniiStatsPB stats = 2; optional SniiSectionRefsPB section_refs = 3; - // 字段 4/5 曾是 CommonGrams 元数据（功能已删除）。字段号保留、类型降级成不透明 - // 字节 / 整数：reader 只用它们识别"这段是用 CommonGrams 写出来的"并拒绝读取（墓碑）， - // 不再解析内容。旧 reader（生产 4.1.7 系）本来就忽略这两个字段。 + // Fields 4/5 belonged to the removed CommonGrams feature. Keep their numbers as opaque + // bytes/integers so readers can identify and reject CommonGrams segments without parsing + // the contents. Older production 4.1.7-series readers already ignore these fields. optional bytes legacy_common_grams = 4; optional uint32 legacy_common_grams_posting_policy = 5; }