Skip to content

Commit 5985916

Browse files
authored
[style](index) Translate SNII comments and test diagnostics to English (apache#67808)
### What problem does this PR solve? Related PR: apache#67537 Problem Summary: The SNII code and tests touched by apache#67537 contain Chinese comments and golden-corpus diagnostics. Translate that explanatory text to English, including the related FE policy and protobuf comments. Preserve all Unicode test inputs, executable logic, query expectations, and the on-disk format. Also clarify two comments against the existing behavior: the control-byte sample is not the full internal marker, and missing golden expectations fail verification.
1 parent 9125fd6 commit 5985916

51 files changed

Lines changed: 197 additions & 179 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/storage/index/index_file_writer.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,8 @@ class IndexFileWriter {
110110
// the prx region compresses at snii_prx_zstd_level_direct_load;
111111
// compaction / schema change / ADD INDEX keep snii_prx_zstd_level.
112112
bool is_direct_load = false;
113-
// 每 doc 一字节的 BM25 norms;为空表示该索引不写 norms(keyword / 不带位置)。
114-
// 非空时长度必须等于 doc_count,且 posting 会保留 freq 区供打分读取。
113+
// One byte of BM25 norms per document; empty for keyword or positionless indexes.
114+
// If nonempty, its size must equal doc_count, and postings retain frequencies for scoring.
115115
std::vector<uint8_t> encoded_norms;
116116
};
117117
Status add_snii_index(const TabletIndex* index_meta, uint32_t doc_count,
@@ -138,8 +138,8 @@ class IndexFileWriter {
138138
doris::snii::format::IndexConfig index_config,
139139
std::shared_ptr<doris::snii::writer::MemoryReporter> mem_reporter,
140140
doris::snii::writer::SniiStreamedIndexSession** session);
141-
// write_norms=true 的会话在 finish 之前必须通过 set_encoded_norms 交付 norms
142-
// (compaction 在合并 postings 的同一趟里重建它们)。
141+
// Sessions with write_norms=true must supply norms through set_encoded_norms before finish.
142+
// Compaction rebuilds them in the same pass that merges postings.
143143
Status add_snii_index_streamed(
144144
const TabletIndex* index_meta, uint32_t doc_count,
145145
doris::snii::writer::TrackedNullDocids null_docids, bool write_norms,

‎be/src/storage/index/inverted/inverted_index_cache.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ class InvertedIndexCacheHandle {
216216

217217
class InvertedIndexQueryCacheHandle;
218218

219-
// v2:结果缓存键不再携带 CommonGrams 查询计划开关位(该功能已删除)。
219+
// v2: Result cache keys no longer include the removed CommonGrams query-plan flags.
220220
inline constexpr uint32_t INVERTED_INDEX_QUERY_CACHE_SEMANTICS_VERSION = 2;
221221

222222
// Stable identity shared by result-cache and row-accurate single-flight. It intentionally contains

‎be/src/storage/index/inverted/similarity/collection_statistics.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset,
251251

252252
::doris::snii::reader::DictBlockCache dict_block_cache;
253253
for (const auto& logical_term_bytes : collect_info.unique_terms) {
254-
// SNII 的 term 键就是分词后的原始字节,没有任何转义或版本。
254+
// SNII term keys are the raw analyzed bytes, with no escaping or version prefix.
255255
const std::string& physical_term = logical_term_bytes;
256256
const auto logical_term =
257257
segment_v2::inverted_index::StringHelper::to_wstring(logical_term_bytes);

‎be/src/storage/index/inverted/similarity/collection_statistics.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,9 @@ struct SniiScoringSegmentStats {
119119
uint64_t token_count = 0;
120120
};
121121

122-
// 一个 SNII 段能参与打分的条件:带位置(词频来自位置)且带 norms(新版 writer 对分词 + 带位置
123-
// 的索引一律写出)。老段没有 norms → NOT_SUPPORTED,重建索引或等 compaction 补齐。
122+
// SNII scoring requires positions (which provide term frequencies) and norms. The current writer
123+
// emits norms for every analyzed index with positions. Older segments without norms return
124+
// NOT_SUPPORTED until an index rebuild or compaction supplies them.
124125
Result<SniiScoringSegmentStats> resolve_snii_scoring_segment(uint64_t index_doc_count,
125126
uint64_t sum_total_term_freq,
126127
bool has_positions, bool has_norms);

‎be/src/storage/index/snii/compaction/eligibility.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,9 @@ using AnalyzerProviderFactory = std::function<segment_v2::inverted_index::Analyz
5252
const InvertedIndexAnalyzerConfig&)>;
5353

5454
struct SniiCompactionEligibility {
55-
// A2:目标索引是"分词 + 带位置"时,合并产物必须带 norms。norms 在合并 postings 的同一趟里
56-
// 从各源的词频重建(每 doc Σfreq,clamp 到 1..255),所以老段(没有 norms 的 T2)也能
57-
// 不重分词地升级成带 norms 的段。
55+
// A2: An analyzed destination index with positions must include norms. The posting merge
56+
// rebuilds them from source term frequencies (sum per document, clamped to 1..255), so older
57+
// T2 segments without norms can gain norms without reanalysis.
5858
bool destination_writes_norms = false;
5959
};
6060

‎be/src/storage/index/snii/compaction/posting_run_merger.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -363,8 +363,8 @@ Status MergedPostingRuns::select_run(ActivePostingChunk* active, size_t max_docs
363363
: std::span<const uint32_t> {};
364364

365365
if (!destination_doc_lengths_.empty()) {
366-
// 重建 norms:每篇文档的长度 = 该文档在所有 term 上的词频之和(与 writer 的
367-
// 词元计数逐字节一致,见 SniiIndexColumnWriter),按 u8 饱和累加,255 封顶。
366+
// Rebuild norms by summing each document's frequencies across all terms, matching the
367+
// token counts in SniiIndexColumnWriter byte for byte. Saturate the u8 sum at 255.
368368
std::vector<uint8_t>& lengths = destination_doc_lengths_[*active_destination_];
369369
for (size_t i = 0; i < document_count; ++i) {
370370
const uint32_t docid = run->docids[i];

‎be/src/storage/index/snii/compaction/posting_run_merger.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,9 @@ class MergedPostingRuns final : public writer::TermPostingSource {
5757
};
5858

5959
public:
60-
// destination_doc_lengths 非空时,每个 (目标段, 目标 docid) 的词频按 u8 饱和累加(0..255),
61-
// 合并结束后由调用方 encode 成 norms;为空表示目标不写 norms。
60+
// If destination_doc_lengths is nonempty, accumulate frequencies for each destination
61+
// (segment, docid), saturating at 255. The caller encodes them as norms after the merge.
62+
// An empty vector means the destination does not write norms.
6263
MergedPostingRuns(std::vector<std::unique_ptr<SniiPostingCursor>> cursors,
6364
bool retain_positions, std::span<const uint32_t> destination_doc_counts,
6465
std::span<std::vector<uint8_t>> destination_doc_lengths);

‎be/src/storage/index/snii/compaction/snii_index_compaction.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -412,8 +412,8 @@ Status SniiPlainT2MergePlan::merge_terms(
412412
}
413413

414414
if (eligibility_.destination_writes_norms) {
415-
// 累加的是原始长度(0..255 饱和);encode_norm 把 0 映射成 1,与 writer 的
416-
// encode_norm(len) = clamp(len, 1, 255) 一致。
415+
// Accumulated raw lengths saturate at 255. encode_norm maps 0 to 1, matching the
416+
// writer's encode_norm(len) = clamp(len, 1, 255).
417417
for (size_t destination_ordinal = 0; destination_ordinal < sessions.size();
418418
++destination_ordinal) {
419419
for (uint8_t& value : destination_encoded_norms_[destination_ordinal]) {

‎be/src/storage/index/snii/format/core_metadata.cpp‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,9 @@ Status decode_core_pb(const doris::snii::SniiCoreMetadataPB& input, CoreMetadata
7272
!stats.has_null_count()) {
7373
return corrupted("core metadata: missing statistics field");
7474
}
75-
// sum_total_term_freq(字段 5)与 norms(字段 5)是后加的可选字段:已上线的生产
76-
// 3.1 系 writer 不写它们。缺失 = 该段没有打分统计 / 没有 norms,
77-
// 只影响 BM25 打分是否可用,不影响任何过滤查询。
75+
// sum_total_term_freq (stats field 5) and norms (section_refs field 5) are optional additions
76+
// absent from the deployed 3.1-series writer. Missing fields mean no scoring statistics or
77+
// norms, affecting BM25 availability but not filtering queries.
7878
out->stats = {.doc_count = stats.doc_count(),
7979
.indexed_doc_count = stats.indexed_doc_count(),
8080
.term_count = stats.term_count(),
@@ -97,15 +97,16 @@ Status decode_core_pb(const doris::snii::SniiCoreMetadataPB& input, CoreMetadata
9797
RETURN_IF_ERROR(decode_region_ref(refs.null_bitmap(), &out->section_refs.null_bitmap));
9898
RETURN_IF_ERROR(decode_region_ref(refs.bsbf(), &out->section_refs.bsbf));
9999

100-
// 墓碑:CommonGrams 功能已删除。带过字段 4/5 的段只可能是用 CommonGrams analyzer 写出来的
101-
// (含 gram 词项、键转义或混合 posting 策略),term 键与查询语义都已不可解释,必须重建索引。
102-
// 生产 writer 从未写过这两个字段,所以升级路径不受影响。
100+
// Tombstones for the removed CommonGrams feature. Fields 4/5 identify segments written with
101+
// a CommonGrams analyzer (gram terms, escaped keys, or mixed posting policies). Their term
102+
// keys and query semantics are no longer supported, so these indexes must be rebuilt.
103+
// Production writers never emitted these fields, so upgrades are unaffected.
103104
if (input.has_legacy_common_grams() || input.has_legacy_common_grams_posting_policy()) {
104105
return unsupported(
105106
"core metadata: segment was written with CommonGrams, which is no longer "
106107
"supported; rebuild the index");
107108
}
108-
// norms(每 doc 一字节的 BM25 文档长度)只对带位置的段有意义:打分的词频来自位置。
109+
// Norms encode BM25 document lengths in one byte and require positions for term frequencies.
109110
if (out->section_refs.norms.length != 0 && !has_positions(out->index_config)) {
110111
return corrupted("core metadata: norms require positions");
111112
}
@@ -130,7 +131,7 @@ Status encode_core_metadata(const CoreMetadata& metadata, ByteSink* out) {
130131
auto* refs = core.mutable_section_refs();
131132
encode_region_ref(metadata.section_refs.dict_region, refs->mutable_dict_region());
132133
encode_region_ref(metadata.section_refs.posting_region, refs->mutable_posting_region());
133-
// 没有 norms 的段不写字段 5:与生产 writer 的字节形态一致,老 reader 也无需感知。
134+
// Omit field 5 when norms are absent, matching production bytes without affecting old readers.
134135
if (metadata.section_refs.norms.length != 0) {
135136
encode_region_ref(metadata.section_refs.norms, refs->mutable_norms());
136137
}

‎be/src/storage/index/snii/format/format_constants.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ enum class SectionType : uint8_t {
6868
enum class IndexConfig : uint8_t {
6969
kDocsOnly = 0, // docid only: term/match filtering
7070
kDocsPositions = 1, // docid+positions: MATCH_PHRASE; BM25 tf = position count
71-
// 2 曾是 kDocsPositionsScoring(CommonGrams 时代的打分 tier),已删除:打分能力由
72-
// norms region 是否存在决定(见 CoreMetadata::section_refs.norms),reader 拒绝值 2。
71+
// Value 2 was kDocsPositionsScoring, the removed CommonGrams scoring tier. Scoring now
72+
// depends on the norms region (CoreMetadata::section_refs.norms); readers reject value 2.
7373
kPositionsOffsets = 3, // reserved (highlight/RAG), not implemented in this release
7474
};
7575

0 commit comments

Comments
 (0)