Skip to content

Commit 4e40a4d

Browse files
author
qiye
committed
[fix](Index)Make column unique ids in index dynamically computed (apache#48988)
Issue Number: close apache#48989 Related PR: apache#46648 apache#42882 Problem Summary: We introduced `Index.columnUniqueIds` in 3.0.3, and make sure `Index.columnUniqueIds` is not empty in 3.0.4. But when we upgrade Doris from 3.0.2 and before to 3.0.4, the `Index.columnUniqueIds` will be `null`. Versions 302 and earlier do not have this variable, and gson deserialization will assign it to null. We need to initialize the column unique ids associated with the index correctly. Key Changes Removed stored `columnUniqueIds` from `Index` class: Previously, column unique IDs were stored as a field in the `Index` class Now they're dynamically computed when needed rather than stored redundantly Added dynamic computation method: New method `getColumnUniqueIds(List<Column> schema)` computes IDs at runtime Column IDs are looked up by matching index column names with schema columns Updated method signatures: Changed `toThrift()` to `toThrift(List<Integer> indexColumnUniqueIds)` Changed `toPb()` to `toPb(Map<Integer, Column> columnMap, List<Integer> indexColumnUniqueIds)` Updated all callers to pass column IDs as parameters
1 parent 4e24298 commit 4e40a4d

17 files changed

Lines changed: 202 additions & 52 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2712,7 +2712,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
27122712
indexDef.checkColumn(column, olapTable.getKeysType(),
27132713
olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(),
27142714
disableInvertedIndexV1ForVariant);
2715-
indexDef.getColumnUniqueIds().add(column.getUniqueId());
27162715
} else {
27172716
throw new DdlException("index column does not exist in table. invalid column: " + col);
27182717
}
@@ -2723,7 +2722,6 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
27232722
// so here update column name in CreateIndexClause after checkColumn for indexDef,
27242723
// there will use the column name in olapTable instead of the column name in CreateIndexClause.
27252724
alterIndex.setColumns(indexDef.getColumns());
2726-
alterIndex.setColumnUniqueIds(indexDef.getColumnUniqueIds());
27272725
newIndexes.add(alterIndex);
27282726
return false;
27292727
}

‎fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
7777
indexDef.analyze();
7878
this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(),
7979
indexDef.getColumns(), indexDef.getIndexType(),
80-
indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds());
80+
indexDef.getProperties(), indexDef.getComment());
8181
}
8282

8383
@Override

‎fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
7373
indexDef.analyze();
7474
this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(),
7575
indexDef.getColumns(), indexDef.getIndexType(),
76-
indexDef.getProperties(), indexDef.getComment(), indexDef.getColumnUniqueIds());
76+
indexDef.getProperties(), indexDef.getComment());
7777
}
7878

7979
@Override

‎fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -592,8 +592,7 @@ public void analyze(Analyzer analyzer) throws UserException {
592592
}
593593
}
594594
indexes.add(new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(),
595-
indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment(),
596-
indexDef.getColumnUniqueIds()));
595+
indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment()));
597596
distinct.add(indexDef.getIndexName());
598597
distinctCol.add(Pair.of(indexDef.getIndexType(),
599598
indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList())));

‎fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ public class IndexDef {
4242
private Map<String, String> properties;
4343
private boolean isBuildDeferred = false;
4444
private PartitionNames partitionNames;
45-
private List<Integer> columnUniqueIds = Lists.newArrayList();
4645
public static final int MIN_NGRAM_SIZE = 1;
4746
public static final int MAX_NGRAM_SIZE = 255;
4847
public static final int MIN_BF_SIZE = 64;
@@ -202,10 +201,6 @@ public List<String> getPartitionNames() {
202201
return partitionNames == null ? Lists.newArrayList() : partitionNames.getPartitionNames();
203202
}
204203

205-
public List<Integer> getColumnUniqueIds() {
206-
return columnUniqueIds;
207-
}
208-
209204
public enum IndexType {
210205
BITMAP,
211206
INVERTED,

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -65,19 +65,15 @@ public class Index implements Writable {
6565
private Map<String, String> properties;
6666
@SerializedName(value = "ct", alternate = {"comment"})
6767
private String comment;
68-
@SerializedName(value = "cui", alternate = {"columnUniqueIds"})
69-
private List<Integer> columnUniqueIds;
7068

7169
public Index(long indexId, String indexName, List<String> columns,
72-
IndexDef.IndexType indexType, Map<String, String> properties, String comment,
73-
List<Integer> columnUniqueIds) {
70+
IndexDef.IndexType indexType, Map<String, String> properties, String comment) {
7471
this.indexId = indexId;
7572
this.indexName = indexName;
7673
this.columns = columns == null ? Lists.newArrayList() : Lists.newArrayList(columns);
7774
this.indexType = indexType;
7875
this.properties = properties == null ? Maps.newHashMap() : Maps.newHashMap(properties);
7976
this.comment = comment;
80-
this.columnUniqueIds = columnUniqueIds == null ? Lists.newArrayList() : Lists.newArrayList(columnUniqueIds);
8177
if (indexType == IndexDef.IndexType.INVERTED) {
8278
if (this.properties != null && !this.properties.isEmpty()) {
8379
if (this.properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
@@ -101,7 +97,6 @@ public Index() {
10197
this.indexType = null;
10298
this.properties = null;
10399
this.comment = null;
104-
this.columnUniqueIds = null;
105100
}
106101

107102
public long getIndexId() {
@@ -191,14 +186,6 @@ public void setComment(String comment) {
191186
this.comment = comment;
192187
}
193188

194-
public List<Integer> getColumnUniqueIds() {
195-
return columnUniqueIds;
196-
}
197-
198-
public void setColumnUniqueIds(List<Integer> columnUniqueIds) {
199-
this.columnUniqueIds = columnUniqueIds;
200-
}
201-
202189
@Override
203190
public void write(DataOutput out) throws IOException {
204191
Text.writeString(out, GsonUtils.GSON.toJson(this));
@@ -216,7 +203,7 @@ public int hashCode() {
216203

217204
public Index clone() {
218205
return new Index(indexId, indexName, new ArrayList<>(columns),
219-
indexType, new HashMap<>(properties), comment, columnUniqueIds);
206+
indexType, new HashMap<>(properties), comment);
220207
}
221208

222209
@Override
@@ -251,7 +238,21 @@ public String toSql() {
251238
return sb.toString();
252239
}
253240

254-
public TOlapTableIndex toThrift() {
241+
public List<Integer> getColumnUniqueIds(List<Column> schema) {
242+
List<Integer> columnUniqueIds = new ArrayList<>();
243+
if (schema != null) {
244+
for (String columnName : columns) {
245+
for (Column column : schema) {
246+
if (columnName.equalsIgnoreCase(column.getName())) {
247+
columnUniqueIds.add(column.getUniqueId());
248+
}
249+
}
250+
}
251+
}
252+
return columnUniqueIds;
253+
}
254+
255+
public TOlapTableIndex toThrift(List<Integer> indexColumnUniqueIds) {
255256
TOlapTableIndex tIndex = new TOlapTableIndex();
256257
tIndex.setIndexId(indexId);
257258
tIndex.setIndexName(indexName);
@@ -260,15 +261,16 @@ public TOlapTableIndex toThrift() {
260261
if (properties != null) {
261262
tIndex.setProperties(properties);
262263
}
263-
tIndex.setColumnUniqueIds(columnUniqueIds);
264+
tIndex.setColumnUniqueIds(indexColumnUniqueIds);
264265
return tIndex;
265266
}
266267

267-
public OlapFile.TabletIndexPB toPb(Map<Integer, Column> columnMap) {
268+
public OlapFile.TabletIndexPB toPb(Map<Integer, Column> columnMap, List<Integer> indexColumnUniqueIds) {
268269
OlapFile.TabletIndexPB.Builder builder = OlapFile.TabletIndexPB.newBuilder();
269270
builder.setIndexId(indexId);
270271
builder.setIndexName(indexName);
271-
for (Integer columnUniqueId : columnUniqueIds) {
272+
273+
for (Integer columnUniqueId : indexColumnUniqueIds) {
272274
Column column = columnMap.get(columnUniqueId);
273275
if (column != null) {
274276
builder.addColUniqueId(column.getUniqueId());

‎fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -391,13 +391,6 @@ public void initSchemaColumnUniqueId() {
391391
maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
392392
this.schema.forEach(column -> {
393393
column.setUniqueId(incAndGetMaxColUniqueId());
394-
this.indexes.forEach(index -> {
395-
index.getColumns().forEach(col -> {
396-
if (col.equalsIgnoreCase(column.getName())) {
397-
index.getColumnUniqueIds().add(column.getUniqueId());
398-
}
399-
});
400-
});
401394
if (LOG.isDebugEnabled()) {
402395
LOG.debug("indexId: {}, column:{}, uniqueId:{}",
403396
indexId, column, column.getUniqueId());

‎fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -321,9 +321,8 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId,
321321
columnMap.put(column.getUniqueId(), column);
322322
}
323323
if (indexes != null) {
324-
for (int i = 0; i < indexes.size(); i++) {
325-
Index index = indexes.get(i);
326-
schemaBuilder.addIndex(index.toPb(columnMap));
324+
for (Index index : indexes) {
325+
schemaBuilder.addIndex(index.toPb(columnMap, index.getColumnUniqueIds(schemaColumns)));
327326
}
328327
}
329328

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,6 @@ public IndexType getIndexType() {
209209

210210
public Index translateToCatalogStyle() {
211211
return new Index(Env.getCurrentEnv().getNextId(), name, cols, indexType, properties,
212-
comment, null);
212+
comment);
213213
}
214214
}

‎fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1518,7 +1518,7 @@ protected void toThrift(TPlanNode msg) {
15181518
}
15191519

15201520
for (Index index : olapTable.getIndexes()) {
1521-
TOlapTableIndex tIndex = index.toThrift();
1521+
TOlapTableIndex tIndex = index.toThrift(index.getColumnUniqueIds(olapTable.getBaseSchema()));
15221522
indexDesc.add(tIndex);
15231523
}
15241524

0 commit comments

Comments
 (0)