Skip to content

Commit c5aadc4

Browse files
author
qiye
committed
[fix](Index)Make column unique ids in index dynamically computed (apache#48988)
Issue Number: close apache#48989 Related PR: apache#46648 apache#42882 Problem Summary: We introduced `Index.columnUniqueIds` in 3.0.3, and make sure `Index.columnUniqueIds` is not empty in 3.0.4. But when we upgrade Doris from 3.0.2 and before to 3.0.4, the `Index.columnUniqueIds` will be `null`. Versions 302 and earlier do not have this variable, and gson deserialization will assign it to null. We need to initialize the column unique ids associated with the index correctly. Key Changes Removed stored `columnUniqueIds` from `Index` class: Previously, column unique IDs were stored as a field in the `Index` class Now they're dynamically computed when needed rather than stored redundantly Added dynamic computation method: New method `getColumnUniqueIds(List<Column> schema)` computes IDs at runtime Column IDs are looked up by matching index column names with schema columns Updated method signatures: Changed `toThrift()` to `toThrift(List<Integer> indexColumnUniqueIds)` Changed `toPb()` to `toPb(Map<Integer, Column> columnMap, List<Integer> indexColumnUniqueIds)` Updated all callers to pass column IDs as parameters
1 parent 17ffd30 commit c5aadc4

16 files changed

Lines changed: 197 additions & 47 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2594,7 +2594,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
25942594
indexDef.checkColumn(column, olapTable.getKeysType(),
25952595
olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(),
25962596
disableInvertedIndexV1ForVariant);
2597-
indexDef.getColumnUniqueIds().add(column.getUniqueId());
25982597
} else {
25992598
throw new DdlException("index column does not exist in table. invalid column: " + col);
26002599
}
@@ -2605,7 +2604,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
26052604
// so here update column name in CreateIndexClause after checkColumn for indexDef,
26062605
// there will use the column name in olapTable instead of the column name in CreateIndexClause.
26072606
alterIndex.setColumns(indexDef.getColumns());
2608-
alterIndex.setColumnUniqueIds(indexDef.getColumnUniqueIds());
26092607
newIndexes.add(alterIndex);
26102608
return false;
26112609
}

‎fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
7373
indexDef.analyze();
7474
this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(),
7575
indexDef.getColumns(), indexDef.getIndexType(),
76-
indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds());
76+
indexDef.getProperties(), indexDef.getComment());
7777
}
7878

7979
@Override

‎fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
7373
indexDef.analyze();
7474
this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(),
7575
indexDef.getColumns(), indexDef.getIndexType(),
76-
indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds());
76+
indexDef.getProperties(), indexDef.getComment());
7777
}
7878

7979
@Override

‎fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -581,8 +581,7 @@ public void analyze(Analyzer analyzer) throws UserException {
581581
}
582582
}
583583
indexes.add(new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(),
584-
indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment(),
585-
indexDef.getColumnUniqueIds()));
584+
indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment()));
586585
distinct.add(indexDef.getIndexName());
587586
distinctCol.add(Pair.of(indexDef.getIndexType(),
588587
indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList())));

‎fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ public class IndexDef {
4343
private Map<String, String> properties;
4444
private boolean isBuildDeferred = false;
4545
private PartitionNames partitionNames;
46-
private List<Integer> columnUniqueIds = Lists.newArrayList();
4746
public static final int MIN_NGRAM_SIZE = 1;
4847
public static final int MAX_NGRAM_SIZE = 255;
4948
public static final int MIN_BF_SIZE = 64;
@@ -203,10 +202,6 @@ public List<String> getPartitionNames() {
203202
return partitionNames == null ? Lists.newArrayList() : partitionNames.getPartitionNames();
204203
}
205204

206-
public List<Integer> getColumnUniqueIds() {
207-
return columnUniqueIds;
208-
}
209-
210205
public enum IndexType {
211206
BITMAP,
212207
INVERTED,

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -64,19 +64,15 @@ public class Index implements Writable {
6464
private Map<String, String> properties;
6565
@SerializedName(value = "comment")
6666
private String comment;
67-
@SerializedName(value = "cui", alternate = {"columnUniqueIds"})
68-
private List<Integer> columnUniqueIds;
6967

7068
public Index(long indexId, String indexName, List<String> columns,
71-
IndexDef.IndexType indexType, Map<String, String> properties, String comment,
72-
List<Integer> columnUniqueIds) {
69+
IndexDef.IndexType indexType, Map<String, String> properties, String comment) {
7370
this.indexId = indexId;
7471
this.indexName = indexName;
7572
this.columns = columns == null ? Lists.newArrayList() : Lists.newArrayList(columns);
7673
this.indexType = indexType;
7774
this.properties = properties == null ? Maps.newHashMap() : Maps.newHashMap(properties);
7875
this.comment = comment;
79-
this.columnUniqueIds = columnUniqueIds == null ? Lists.newArrayList() : Lists.newArrayList(columnUniqueIds);
8076
if (indexType == IndexDef.IndexType.INVERTED) {
8177
if (this.properties != null && !this.properties.isEmpty()) {
8278
if (this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
@@ -100,7 +96,6 @@ public Index() {
10096
this.indexType = null;
10197
this.properties = null;
10298
this.comment = null;
103-
this.columnUniqueIds = null;
10499
}
105100

106101
public long getIndexId() {
@@ -190,14 +185,6 @@ public void setComment(String comment) {
190185
this.comment = comment;
191186
}
192187

193-
public List<Integer> getColumnUniqueIds() {
194-
return columnUniqueIds;
195-
}
196-
197-
public void setColumnUniqueIds(List<Integer> columnUniqueIds) {
198-
this.columnUniqueIds = columnUniqueIds;
199-
}
200-
201188
@Override
202189
public void write(DataOutput out) throws IOException {
203190
Text.writeString(out, GsonUtils.GSON.toJson(this));
@@ -215,7 +202,7 @@ public int hashCode() {
215202

216203
public Index clone() {
217204
return new Index(indexId, indexName, new ArrayList<>(columns),
218-
indexType, new HashMap<>(properties), comment, columnUniqueIds);
205+
indexType, new HashMap<>(properties), comment);
219206
}
220207

221208
@Override
@@ -250,7 +237,21 @@ public String toSql() {
250237
return sb.toString();
251238
}
252239

253-
public TOlapTableIndex toThrift() {
240+
public List<Integer> getColumnUniqueIds(List<Column> schema) {
241+
List<Integer> columnUniqueIds = new ArrayList<>();
242+
if (schema != null) {
243+
for (String columnName : columns) {
244+
for (Column column : schema) {
245+
if (columnName.equalsIgnoreCase(column.getName())) {
246+
columnUniqueIds.add(column.getUniqueId());
247+
}
248+
}
249+
}
250+
}
251+
return columnUniqueIds;
252+
}
253+
254+
public TOlapTableIndex toThrift(List<Integer> indexColumnUniqueIds) {
254255
TOlapTableIndex tIndex = new TOlapTableIndex();
255256
tIndex.setIndexId(indexId);
256257
tIndex.setIndexName(indexName);
@@ -259,7 +260,7 @@ public TOlapTableIndex toThrift() {
259260
if (properties != null) {
260261
tIndex.setProperties(properties);
261262
}
262-
tIndex.setColumnUniqueIds(columnUniqueIds);
263+
tIndex.setColumnUniqueIds(indexColumnUniqueIds);
263264
return tIndex;
264265
}
265266

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -387,13 +387,6 @@ public void initSchemaColumnUniqueId() {
387387
maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
388388
this.schema.forEach(column -> {
389389
column.setUniqueId(incAndGetMaxColUniqueId());
390-
this.indexes.forEach(index -> {
391-
index.getColumns().forEach(col -> {
392-
if (col.equalsIgnoreCase(column.getName())) {
393-
index.getColumnUniqueIds().add(column.getUniqueId());
394-
}
395-
});
396-
});
397390
if (LOG.isDebugEnabled()) {
398391
LOG.debug("indexId: {}, column:{}, uniqueId:{}",
399392
indexId, column, column.getUniqueId());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,6 @@ public IndexType getIndexType() {
213213

214214
public Index translateToCatalogStyle() {
215215
return new Index(Env.getCurrentEnv().getNextId(), name, cols, indexType, properties,
216-
comment, null);
216+
comment);
217217
}
218218
}

‎fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1481,7 +1481,7 @@ protected void toThrift(TPlanNode msg) {
14811481
}
14821482

14831483
for (Index index : olapTable.getIndexes()) {
1484-
TOlapTableIndex tIndex = index.toThrift();
1484+
TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(olapTable.getBaseSchema()));
14851485
indexDesc.add(tIndex);
14861486
}
14871487

‎fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -305,7 +305,7 @@ public TOlapTableSchemaParam createSchema(long dbId, OlapTable table, Analyzer a
305305
indexes = table.getIndexes();
306306
}
307307
for (Index index : indexes) {
308-
TOlapTableIndex tIndex = index.toThrift();
308+
TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(table.getBaseSchema()));
309309
indexDesc.add(tIndex);
310310
}
311311
TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns,

0 commit comments

Comments
 (0)