apache · airborne12 · Sep 11, 2026 · Sep 10, 2026
@@ -110,8 +110,8 @@ class IndexFileWriter {
         // the prx region compresses at snii_prx_zstd_level_direct_load;
         // compaction / schema change / ADD INDEX keep snii_prx_zstd_level.
         bool is_direct_load = false;
-        // 每 doc 一字节的 BM25 norms；为空表示该索引不写 norms（keyword / 不带位置）。
-        // 非空时长度必须等于 doc_count，且 posting 会保留 freq 区供打分读取。
+        // One byte of BM25 norms per document; empty for keyword or positionless indexes.
+        // If nonempty, its size must equal doc_count, and postings retain frequencies for scoring.
         std::vector<uint8_t> encoded_norms;
     };
     Status add_snii_index(const TabletIndex* index_meta, uint32_t doc_count,
@@ -138,8 +138,8 @@ class IndexFileWriter {
             doris::snii::format::IndexConfig index_config,
             std::shared_ptr<doris::snii::writer::MemoryReporter> mem_reporter,
             doris::snii::writer::SniiStreamedIndexSession** session);
-    // write_norms=true 的会话在 finish 之前必须通过 set_encoded_norms 交付 norms
-    // （compaction 在合并 postings 的同一趟里重建它们）。
+    // Sessions with write_norms=true must supply norms through set_encoded_norms before finish.
+    // Compaction rebuilds them in the same pass that merges postings.
     Status add_snii_index_streamed(
             const TabletIndex* index_meta, uint32_t doc_count,
             doris::snii::writer::TrackedNullDocids null_docids, bool write_norms,

@@ -216,7 +216,7 @@ class InvertedIndexCacheHandle {
 
 class InvertedIndexQueryCacheHandle;
 
-// v2：结果缓存键不再携带 CommonGrams 查询计划开关位（该功能已删除）。
+// v2: Result cache keys no longer include the removed CommonGrams query-plan flags.
 inline constexpr uint32_t INVERTED_INDEX_QUERY_CACHE_SEMANTICS_VERSION = 2;
 
 // Stable identity shared by result-cache and row-accurate single-flight. It intentionally contains

@@ -251,7 +251,7 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset,
 
             ::doris::snii::reader::DictBlockCache dict_block_cache;
             for (const auto& logical_term_bytes : collect_info.unique_terms) {
-                // SNII 的 term 键就是分词后的原始字节，没有任何转义或版本。
+                // SNII term keys are the raw analyzed bytes, with no escaping or version prefix.
                 const std::string& physical_term = logical_term_bytes;
                 const auto logical_term =
                         segment_v2::inverted_index::StringHelper::to_wstring(logical_term_bytes);

@@ -119,8 +119,9 @@ struct SniiScoringSegmentStats {
     uint64_t token_count = 0;
 };
 
-// 一个 SNII 段能参与打分的条件：带位置（词频来自位置）且带 norms（新版 writer 对分词 + 带位置
-// 的索引一律写出）。老段没有 norms → NOT_SUPPORTED，重建索引或等 compaction 补齐。
+// SNII scoring requires positions (which provide term frequencies) and norms. The current writer
+// emits norms for every analyzed index with positions. Older segments without norms return
+// NOT_SUPPORTED until an index rebuild or compaction supplies them.
 Result<SniiScoringSegmentStats> resolve_snii_scoring_segment(uint64_t index_doc_count,
                                                              uint64_t sum_total_term_freq,
                                                              bool has_positions, bool has_norms);

@@ -52,9 +52,9 @@ using AnalyzerProviderFactory = std::function<segment_v2::inverted_index::Analyz
         const InvertedIndexAnalyzerConfig&)>;
 
 struct SniiCompactionEligibility {
-    // A2：目标索引是"分词 + 带位置"时，合并产物必须带 norms。norms 在合并 postings 的同一趟里
-    // 从各源的词频重建（每 doc Σfreq，clamp 到 1..255），所以老段（没有 norms 的 T2）也能
-    // 不重分词地升级成带 norms 的段。
+    // A2: An analyzed destination index with positions must include norms. The posting merge
+    // rebuilds them from source term frequencies (sum per document, clamped to 1..255), so older
+    // T2 segments without norms can gain norms without reanalysis.
     bool destination_writes_norms = false;
 };
 

@@ -363,8 +363,8 @@ Status MergedPostingRuns::select_run(ActivePostingChunk* active, size_t max_docs
                               : std::span<const uint32_t> {};
 
     if (!destination_doc_lengths_.empty()) {
-        // 重建 norms：每篇文档的长度 = 该文档在所有 term 上的词频之和（与 writer 的
-        // 词元计数逐字节一致，见 SniiIndexColumnWriter），按 u8 饱和累加，255 封顶。
+        // Rebuild norms by summing each document's frequencies across all terms, matching the
+        // token counts in SniiIndexColumnWriter byte for byte. Saturate the u8 sum at 255.
         std::vector<uint8_t>& lengths = destination_doc_lengths_[*active_destination_];
         for (size_t i = 0; i < document_count; ++i) {
             const uint32_t docid = run->docids[i];

@@ -57,8 +57,9 @@ class MergedPostingRuns final : public writer::TermPostingSource {
     };
 
 public:
-    // destination_doc_lengths 非空时，每个 (目标段, 目标 docid) 的词频按 u8 饱和累加（0..255），
-    // 合并结束后由调用方 encode 成 norms；为空表示目标不写 norms。
+    // If destination_doc_lengths is nonempty, accumulate frequencies for each destination
+    // (segment, docid), saturating at 255. The caller encodes them as norms after the merge.
+    // An empty vector means the destination does not write norms.
     MergedPostingRuns(std::vector<std::unique_ptr<SniiPostingCursor>> cursors,
                       bool retain_positions, std::span<const uint32_t> destination_doc_counts,
                       std::span<std::vector<uint8_t>> destination_doc_lengths);

@@ -412,8 +412,8 @@ Status SniiPlainT2MergePlan::merge_terms(
     }
 
     if (eligibility_.destination_writes_norms) {
-        // 累加的是原始长度（0..255 饱和）；encode_norm 把 0 映射成 1，与 writer 的
-        // encode_norm(len) = clamp(len, 1, 255) 一致。
+        // Accumulated raw lengths saturate at 255. encode_norm maps 0 to 1, matching the
+        // writer's encode_norm(len) = clamp(len, 1, 255).
         for (size_t destination_ordinal = 0; destination_ordinal < sessions.size();
              ++destination_ordinal) {
             for (uint8_t& value : destination_encoded_norms_[destination_ordinal]) {

@@ -72,9 +72,9 @@ Status decode_core_pb(const doris::snii::SniiCoreMetadataPB& input, CoreMetadata
         !stats.has_null_count()) {
         return corrupted("core metadata: missing statistics field");
     }
-    // sum_total_term_freq（字段 5）与 norms（字段 5）是后加的可选字段：已上线的生产
-    // 3.1 系 writer 不写它们。缺失 = 该段没有打分统计 / 没有 norms，
-    // 只影响 BM25 打分是否可用，不影响任何过滤查询。
+    // sum_total_term_freq (stats field 5) and norms (section_refs field 5) are optional additions
+    // absent from the deployed 3.1-series writer. Missing fields mean no scoring statistics or
+    // norms, affecting BM25 availability but not filtering queries.
     out->stats = {.doc_count = stats.doc_count(),
                   .indexed_doc_count = stats.indexed_doc_count(),
                   .term_count = stats.term_count(),
@@ -97,15 +97,16 @@ Status decode_core_pb(const doris::snii::SniiCoreMetadataPB& input, CoreMetadata
     RETURN_IF_ERROR(decode_region_ref(refs.null_bitmap(), &out->section_refs.null_bitmap));
     RETURN_IF_ERROR(decode_region_ref(refs.bsbf(), &out->section_refs.bsbf));
 
-    // 墓碑：CommonGrams 功能已删除。带过字段 4/5 的段只可能是用 CommonGrams analyzer 写出来的
-    // （含 gram 词项、键转义或混合 posting 策略），term 键与查询语义都已不可解释，必须重建索引。
-    // 生产 writer 从未写过这两个字段，所以升级路径不受影响。
+    // Tombstones for the removed CommonGrams feature. Fields 4/5 identify segments written with
+    // a CommonGrams analyzer (gram terms, escaped keys, or mixed posting policies). Their term
+    // keys and query semantics are no longer supported, so these indexes must be rebuilt.
+    // Production writers never emitted these fields, so upgrades are unaffected.
     if (input.has_legacy_common_grams() || input.has_legacy_common_grams_posting_policy()) {
         return unsupported(
                 "core metadata: segment was written with CommonGrams, which is no longer "
                 "supported; rebuild the index");
     }
-    // norms（每 doc 一字节的 BM25 文档长度）只对带位置的段有意义：打分的词频来自位置。
+    // Norms encode BM25 document lengths in one byte and require positions for term frequencies.
     if (out->section_refs.norms.length != 0 && !has_positions(out->index_config)) {
         return corrupted("core metadata: norms require positions");
     }
@@ -130,7 +131,7 @@ Status encode_core_metadata(const CoreMetadata& metadata, ByteSink* out) {
     auto* refs = core.mutable_section_refs();
     encode_region_ref(metadata.section_refs.dict_region, refs->mutable_dict_region());
     encode_region_ref(metadata.section_refs.posting_region, refs->mutable_posting_region());
-    // 没有 norms 的段不写字段 5：与生产 writer 的字节形态一致，老 reader 也无需感知。
+    // Omit field 5 when norms are absent, matching production bytes without affecting old readers.
     if (metadata.section_refs.norms.length != 0) {
         encode_region_ref(metadata.section_refs.norms, refs->mutable_norms());
     }

@@ -68,8 +68,8 @@ enum class SectionType : uint8_t {
 enum class IndexConfig : uint8_t {
     kDocsOnly = 0,      // docid only: term/match filtering
     kDocsPositions = 1, // docid+positions: MATCH_PHRASE; BM25 tf = position count
-    // 2 曾是 kDocsPositionsScoring（CommonGrams 时代的打分 tier），已删除：打分能力由
-    // norms region 是否存在决定（见 CoreMetadata::section_refs.norms），reader 拒绝值 2。
+    // Value 2 was kDocsPositionsScoring, the removed CommonGrams scoring tier. Scoring now
+    // depends on the norms region (CoreMetadata::section_refs.norms); readers reject value 2.
     kPositionsOffsets = 3, // reserved (highlight/RAG), not implemented in this release
 };
 

@@ -31,9 +31,9 @@ inline bool is_phrase_bigram_term(std::string_view term) {
     return term.starts_with(kPhraseBigramTermMarker);
 }
 
-// SNII 的 term 键就是分词后的原始字节，没有任何转义。唯一的内部命名空间是上面这个以 \x1F
-// 开头的 phrase-bigram 标记：用户 term（或前缀展开的前缀）若与它重叠，查询必须绕过 SNII，
-// 否则用户词项会命中内部词项。
+// SNII term keys are raw analyzed bytes, without escaping. The only internal namespace is the
+// phrase-bigram marker above, starting with \x1F. Queries whose user terms or expansion prefixes
+// overlap this marker must bypass SNII to avoid matching internal terms.
 inline bool term_overlaps_internal_namespace(std::string_view term) {
     return term.starts_with(kPhraseBigramTermMarker);
 }

@@ -35,9 +35,9 @@ struct PrxFrameView {
     Slice payload;
 };
 
-// verify_crc=false 只解析帧头并切出 payload、跳过 crc 比对：给"估算位置工作量"这类
-// 只读元数据的路径用，避免在真正解码之前对整个 prx 窗口多算一遍 crc32c；坏帧仍会在
-// 解码时被 crc 拦下。
+// With verify_crc=false, parse only the header and extract the payload without checking its CRC.
+// Metadata-only paths, such as position-work estimation, avoid an extra crc32c pass over the PRX
+// window. Decoding still checks the CRC and rejects corrupt frames.
 Status read_prx_frame(ByteSource* source, PrxFrameView* frame, bool verify_crc = true);
 
 } // namespace format

@@ -110,7 +110,7 @@ Status plan_window_prefix(const LogicalIndexReader& idx, WindowPlan* plan,
     RETURN_IF_ERROR(validate_windowed_docs_prefix(posting.entry));
     uint64_t abs = 0;
     RETURN_IF_ERROR(prelude_abs(idx, posting.entry, posting.frq_base, &abs));
-    // 生产布局：整个 .frq 载荷就是 [prelude][dd-block]，一次 range 读完。
+    // Production layout: the entire .frq payload is [prelude][dd-block], read in one range request.
     plan->prefix_handle = fetcher->add(abs, posting.entry.frq_len);
     return Status::OK();
 }

@@ -173,8 +173,8 @@ struct PhraseTermMapping {
 
 PhraseTermMapping build_phrase_term_mapping(const std::vector<std::string>& terms);
 
-// 把一个已解析的 term 以给定位置偏移追加到 phrase 计划（unique_terms 去重）。定义在
-// phrase_plan.cpp；以前只靠 unity build 的拼接顺序"碰巧"可见。
+// Append a parsed term at the given position offset, deduplicating it in unique_terms.
+// Defined in phrase_plan.cpp; visibility previously depended on the unity build's source order.
 void append_resolved_phrase_clause(ResolvedQueryTerm term, uint32_t position_offset,
                                    internal::ResolvedPhrasePlan* plan);
 

@@ -25,9 +25,9 @@
 
 namespace doris::snii::query::internal {
 
-// SNII 的 term 键就是分词后的原始字节：没有转义、没有版本、没有"物理键 / 逻辑键"之分。
-// 唯一的例外是 \x1F 开头的内部命名空间（phrase-bigram 标记）：用户 term 若与之重叠，
-// 整条查询以 INVERTED_INDEX_BYPASS 绕过 SNII，由上层走非索引路径。
+// SNII term keys are raw analyzed bytes, without escaping, versioning, or a physical/logical
+// key distinction. The sole exception is the internal phrase-bigram namespace starting with
+// \x1F: overlapping user terms return INVERTED_INDEX_BYPASS for execution without the index.
 inline Status check_term_outside_internal_namespace(std::string_view term) {
     if (format::term_overlaps_internal_namespace(term)) {
         return Status::Error<ErrorCode::INVERTED_INDEX_BYPASS>(

@@ -90,7 +90,7 @@ bool should_use_streaming_exact_phrase(const std::vector<TermPlan>& plans,
         }
         const TermPlan& plan = plans[plan_index];
         DORIS_CHECK_NE(plan.df, 0);
-        // dict entry 不带词频统计：每 doc 的平均位置工作量从已解码的 prx 帧统计得来。
+        // Dictionary entries lack term frequencies; estimate position work from decoded PRX frames.
         DORIS_CHECK_NE(sources[plan_index].logical_position_docs, 0);
         const uint64_t position_work = sources[plan_index].logical_position_work /
                                        sources[plan_index].logical_position_docs;

@@ -78,7 +78,8 @@ PhraseTermMapping build_phrase_term_mapping(const std::vector<std::string>& term
 }
 
 namespace {
-// 只读帧头估算工作量，不校验 crc：解码时才校验，坏帧在那里被拦下并保留已解码帧的统计。
+// Estimate work from frame headers without checking CRCs. Decoding rejects corrupt frames while
+// retaining statistics from frames already decoded.
 Status accumulate_frame_position_work(Slice frames, uint64_t* work) {
     ByteSource source(frames);
     while (!source.eof()) {

@@ -52,7 +52,7 @@ struct TermPosting {
     double score = 0.0;
 };
 
-// 磁盘上没有词频区：BM25 的 tf 就是该 term 在文档里的位置个数（与 Lucene 系打分定义一致）。
+// With no on-disk frequency region, BM25 tf is the term's position count, as in Lucene scoring.
 Status require_positions(const LogicalIndexReader& idx) {
     if (!idx.has_positions()) {
         return Status::Error<ErrorCode::INVALID_ARGUMENT, false>(

@@ -45,8 +45,8 @@ Status legacy_raw_prefix_exists(const reader::LogicalIndexReader& idx, std::stri
             cache);
 }
 
-// 空前缀会枚举整个词典：段里若存在内部命名空间的词项（phrase-bigram 标记），枚举结果会混入
-// 内部词项，此时整条查询绕过 SNII。
+// An empty prefix enumerates the entire dictionary. If the segment contains internal
+// phrase-bigram terms, bypass SNII for the whole query to avoid including those terms.
 Status prove_no_internal_terms(const reader::LogicalIndexReader& idx,
                                reader::DictBlockCache* cache) {
     bool exists = false;

@@ -178,7 +178,7 @@ class LogicalIndexReader {
     const format::StatsBlock& stats() const { return core_.stats; }
     format::IndexTier tier() const { return tier_; }
     bool has_positions() const { return has_positions_; }
-    // 是否带 BM25 norms（分词 + 带位置的索引由新版 writer 写出；老段没有）。
+    // Whether BM25 norms exist: current writers emit them for analyzed indexes with positions.
     bool has_norms() const { return core_.section_refs.norms.length != 0; }
     LogicalIndexOpenMode open_mode() const { return open_mode_; }
     io::FileReader* reader() const { return reader_; }

@@ -624,9 +624,10 @@ Status SniiIndexReader::_query(const IndexQueryContextPtr& context, const std::s
     if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) {
         parse_phrase_slop(&plain_analysis_str, &query_info);
     }
-    // 结果缓存只以 (索引文件, 列, 查询类型, 原始查询字节) 为键：分词结果由索引属性与
-    // policy 唯一决定（policy 被引用后不可变），因此打开 segment 之前就能判定是否可共享；
-    // 只有打分查询（结果随集合统计变化）不进缓存，也不走 single-flight 合并。
+    // Result cache keys contain only (index file, column, query type, raw query bytes). Analysis
+    // is determined by index properties and policies, which are immutable once referenced, so
+    // sharing can be decided before opening the segment. Scoring queries depend on collection
+    // statistics and use neither the result cache nor single-flight coalescing.
     const bool allow_result_cache = !actual_similarity;
     const InvertedIndexRawQuerySemantic raw_semantic {.raw_query_bytes = search_str,
                                                       .query_type = query_type,

@@ -37,7 +37,7 @@ struct PhraseMatch;
 
 namespace doris::segment_v2 {
 
-// 一次查询在打开 logical reader 之后交给 _compute_query_bitmap 的全部输入。
+// All query inputs passed to _compute_query_bitmap after opening the logical reader.
 struct SniiQueryBitmapRequest {
     InvertedIndexQueryType query_type;
     const InvertedIndexQueryInfo& query_info;

@@ -115,9 +115,9 @@ Status SniiIndexColumnWriter::init() {
         return Status::Error<ErrorCode::INVERTED_INDEX_ANALYZER_ERROR>(
                 "SNII create analyzer failed: {}", e.what());
     }
-    // A2：分词 + 带位置的索引一律写 norms（每 doc 的词元数，clamp 到 1..255），与 CLucene 的
-    // 打分能力对齐；keyword 或不带位置的索引不写。norms 是 core 元数据里的可选 region，
-    // 不认识它的老 reader 会原样忽略。
+    // A2: Analyzed indexes with positions always write norms (tokens per document, clamped to
+    // 1..255), matching CLucene's scoring capabilities. Keyword or positionless indexes omit
+    // them. Norms are an optional core-metadata region ignored by older readers.
     _writes_norms = _should_analyzer && _has_positions;
     return Status::OK();
 }
@@ -253,8 +253,8 @@ Status SniiIndexColumnWriter::add_array_values(size_t field_size, const void* va
             row_token_count += token_count;
         }
         if (_writes_norms) {
-            // 一行 ARRAY 的文档长度 = 所有元素的词元数之和（NULL 行由 add_array_nulls 声明，
-            // 但同样经过这里，长度为 0）。
+            // An ARRAY row's document length is the total token count across its elements.
+            // NULL rows also pass here with length 0 and are marked by add_array_nulls.
             _encoded_norms.push_back(::doris::snii::query::encode_norm(row_token_count));
             _report_encoded_norms_capacity();
         }