Skip to content

Commit d1070cb

Browse files
[fix](fe) block inverted index V1 creation in FE (#64522)
### What problem does this PR solve? Problem Summary: When users copy historical DDL statements that explicitly set inverted index format to V1, new tables are silently created with V1. Inverted index V1 does not support small file merging and bypasses the packed file path, significantly amplifying PUT requests to object storage. Observed impact: clusters with high write QPS are seeing300–400 PUT/s, with 70–80% attributed to inverted index V1. Block new inverted index V1 creation at the FE level: - Explicitly specifying `inverted_index_storage_format = v1` on a new indexnow returns an error immediately. - Users are guided to use V2 (the default). - Existing V1 tables remain fully compatible — metadata loading and read/write are unaffected. No forced migration. Error message:Inverted index V1 is deprecated and no longer allowed for new index creation.Please use inverted index V2. Coverage - `CREATE TABLE` with inline index definition - `ALTER TABLE ADD INDEX` - All other code paths that create a new inverted index ### Release note
1 parent 60c5a8d commit d1070cb

57 files changed

Lines changed: 258 additions & 1207 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2822,8 +2822,9 @@ public class Config extends ConfigBase {
28222822
@ConfField(mutable = true)
28232823
public static boolean fix_tablet_partition_id_eq_0 = false;
28242824

2825-
@ConfField(mutable = true, masterOnly = true, description = "Default storage format of inverted index, the "
2826-
+ "default value is V3.")
2825+
@ConfField(mutable = true, masterOnly = true,
2826+
callback = InvertedIndexStorageFormatValidator.RuntimeConfigHandler.class,
2827+
description = "Default storage format of inverted index, the default value is V3.")
28272828
public static String inverted_index_storage_format = "V3";
28282829

28292830
@ConfField(mutable = true, masterOnly = true, description = "Enable the 'delete predicate' for DELETE statements. "
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.common;
19+
20+
import java.lang.reflect.Field;
21+
22+
public final class InvertedIndexStorageFormatValidator {
23+
private InvertedIndexStorageFormatValidator() {
24+
}
25+
26+
public static void rejectRuntimeV1(String confVal) throws ConfigException {
27+
String normalizedConfVal = confVal.trim();
28+
if ("V1".equalsIgnoreCase(normalizedConfVal)) {
29+
throw new ConfigException("Inverted index V1 is deprecated and no longer allowed"
30+
+ " for new index creation. Please use inverted index V2.");
31+
}
32+
}
33+
34+
public static void rejectStartupV1(String confVal) throws ConfigException {
35+
String normalizedConfVal = confVal.trim();
36+
if ("V1".equalsIgnoreCase(normalizedConfVal)) {
37+
throw new ConfigException(
38+
"inverted_index_storage_format=V1 is no longer supported. "
39+
+ "Please update fe.conf (or fe_custom.conf): set inverted_index_storage_format=V2.");
40+
}
41+
}
42+
43+
public static class RuntimeConfigHandler extends ConfigBase.DefaultConfHandler {
44+
@Override
45+
public void handle(Field field, String confVal) throws Exception {
46+
rejectRuntimeV1(confVal);
47+
super.handle(field, confVal.trim());
48+
}
49+
}
50+
}

‎fe/fe-common/src/test/java/org/apache/doris/common/ConfigTest.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,4 +131,24 @@ public void testSecurityPathConfigsAreNotRuntimeMutable() {
131131
Assert.assertTrue(e.getMessage().contains("is not mutable"));
132132
}
133133
}
134+
135+
@Test
136+
public void testRejectDeprecatedInvertedIndexV1WithWhitespace() throws Exception {
137+
String originFormat = Config.inverted_index_storage_format;
138+
try {
139+
ConfigBase.setMutableConfig("inverted_index_storage_format", "V2");
140+
ConfigException dynamicException = Assert.assertThrows(ConfigException.class,
141+
() -> ConfigBase.setMutableConfig("inverted_index_storage_format", " V1 "));
142+
Assert.assertTrue(dynamicException.getMessage().contains("Inverted index V1 is deprecated"));
143+
Assert.assertEquals("V2", Config.inverted_index_storage_format);
144+
145+
Config.inverted_index_storage_format = "V2";
146+
ConfigException startupException = Assert.assertThrows(ConfigException.class,
147+
() -> InvertedIndexStorageFormatValidator.rejectStartupV1(" V1 "));
148+
Assert.assertTrue(startupException.getMessage().contains("inverted_index_storage_format=V1"));
149+
Assert.assertEquals("V2", Config.inverted_index_storage_format);
150+
} finally {
151+
Config.inverted_index_storage_format = originFormat;
152+
}
153+
}
134154
}

‎fe/fe-core/src/main/java/org/apache/doris/DorisFE.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.doris.common.Config;
2424
import org.apache.doris.common.FeConstants;
2525
import org.apache.doris.common.FeMetaVersion;
26+
import org.apache.doris.common.InvertedIndexStorageFormatValidator;
2627
import org.apache.doris.common.LdapConfig;
2728
import org.apache.doris.common.Log4jConfig;
2829
import org.apache.doris.common.LogUtils;
@@ -149,6 +150,10 @@ public static void start(String dorisHomeDir, String pidDir, String[] args, Star
149150
// Must init custom config after init config, separately.
150151
// Because the path of custom config file is defined in fe.conf
151152
config.initCustom(Config.custom_config_dir + "/fe_custom.conf");
153+
// inverted_index_storage_format's runtime callback is not invoked while parsing
154+
// fe.conf/fe_custom.conf, so validate the loaded value here after both files are loaded
155+
// and merged, to reject a "V1" left over in the config files at startup.
156+
InvertedIndexStorageFormatValidator.rejectStartupV1(Config.inverted_index_storage_format);
152157

153158
// The command line value overrides fe.conf, so this can only run once Config is loaded.
154159
resolveLocalResourceGroup(commandLine);

‎fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3290,6 +3290,13 @@ private boolean processAddIndex(CreateIndexOp createIndexOp, OlapTable olapTable
32903290
AnnIndexPropertiesChecker.checkProperties(indexDef.getProperties());
32913291
}
32923292

3293+
if (indexDef.getIndexType() == IndexType.INVERTED
3294+
&& olapTable.getInvertedIndexFileStorageFormat() == TInvertedIndexFileStorageFormat.V1) {
3295+
throw new DdlException("Inverted index V1 is deprecated and no longer allowed for new index creation."
3296+
+ " Upgrading inverted_index_storage_format via ALTER TABLE is not supported;"
3297+
+ " recreate the table with inverted_index_storage_format = V2.");
3298+
}
3299+
32933300
for (String col : indexDef.getColumnNames()) {
32943301
Column column = olapTable.getColumn(col);
32953302
if (column != null) {

‎fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1228,9 +1228,7 @@ public static TInvertedIndexFileStorageFormat analyzeInvertedIndexFileStorageFor
12281228
invertedIndexFileStorageFormat = properties.get(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT);
12291229
properties.remove(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT);
12301230
} else {
1231-
if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
1232-
return TInvertedIndexFileStorageFormat.V1;
1233-
} else if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) {
1231+
if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) {
12341232
return TInvertedIndexFileStorageFormat.V2;
12351233
} else if (Config.inverted_index_storage_format.equalsIgnoreCase("SNII")) {
12361234
return TInvertedIndexFileStorageFormat.SNII;
@@ -1240,17 +1238,17 @@ public static TInvertedIndexFileStorageFormat analyzeInvertedIndexFileStorageFor
12401238
}
12411239

12421240
if (invertedIndexFileStorageFormat.equalsIgnoreCase("v1")) {
1243-
return TInvertedIndexFileStorageFormat.V1;
1241+
throw new AnalysisException(
1242+
"Inverted index V1 is deprecated and no longer allowed for new index creation."
1243+
+ " Please use inverted index V2.");
12441244
} else if (invertedIndexFileStorageFormat.equalsIgnoreCase("v2")) {
12451245
return TInvertedIndexFileStorageFormat.V2;
12461246
} else if (invertedIndexFileStorageFormat.equalsIgnoreCase("v3")) {
12471247
return TInvertedIndexFileStorageFormat.V3;
12481248
} else if (invertedIndexFileStorageFormat.equalsIgnoreCase("snii")) {
12491249
return TInvertedIndexFileStorageFormat.SNII;
12501250
} else if (invertedIndexFileStorageFormat.equalsIgnoreCase("default")) {
1251-
if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
1252-
return TInvertedIndexFileStorageFormat.V1;
1253-
} else if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) {
1251+
if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) {
12541252
return TInvertedIndexFileStorageFormat.V2;
12551253
} else if (Config.inverted_index_storage_format.equalsIgnoreCase("SNII")) {
12561254
return TInvertedIndexFileStorageFormat.SNII;

‎fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -339,14 +339,17 @@ public void testAnalyzeInvertedIndexFileStorageFormat() throws AnalysisException
339339
TInvertedIndexFileStorageFormat result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(null);
340340
Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3, result);
341341

342+
// Config=V1 with no explicit property: V1 config is ignored, falls through to V3
342343
Config.inverted_index_storage_format = "V1";
343344
result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(new HashMap<>());
344-
Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result);
345+
Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3, result);
345346

346347
Map<String, String> propertiesWithV1 = new HashMap<>();
347348
propertiesWithV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "v1");
348-
result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV1);
349-
Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result);
349+
AnalysisException v1Ex = Assertions.assertThrows(AnalysisException.class,
350+
() -> PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV1));
351+
Assertions.assertTrue(v1Ex.getMessage().contains(
352+
"Inverted index V1 is deprecated and no longer allowed for new index creation."));
350353

351354
Map<String, String> propertiesWithV2 = new HashMap<>();
352355
propertiesWithV2.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "v2");
@@ -358,11 +361,12 @@ public void testAnalyzeInvertedIndexFileStorageFormat() throws AnalysisException
358361
result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithV3);
359362
Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3, result);
360363

364+
// "default" + Config=V1: falls through to V3 (current default when config is not V2)
361365
Config.inverted_index_storage_format = "V1";
362366
Map<String, String> propertiesWithDefaultV1 = new HashMap<>();
363367
propertiesWithDefaultV1.put(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, "default");
364368
result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(propertiesWithDefaultV1);
365-
Assertions.assertEquals(TInvertedIndexFileStorageFormat.V1, result);
369+
Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3, result);
366370

367371
Config.inverted_index_storage_format = "V2";
368372
Map<String, String> propertiesWithDefaultV2 = new HashMap<>();

‎regression-test/data/fault_injection_p0/test_index_io_context.out‎

Lines changed: 0 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -8,51 +8,6 @@
88
-- !sql --
99
177
1010

11-
-- !sql --
12-
177
13-
14-
-- !sql --
15-
177
16-
17-
-- !sql --
18-
177
19-
20-
-- !sql --
21-
2
22-
23-
-- !sql --
24-
2
25-
26-
-- !sql --
27-
2
28-
29-
-- !sql --
30-
2
31-
32-
-- !sql --
33-
2
34-
35-
-- !sql --
36-
2
37-
38-
-- !sql --
39-
2
40-
41-
-- !sql --
42-
2
43-
44-
-- !sql --
45-
2
46-
47-
-- !sql --
48-
2
49-
50-
-- !sql --
51-
2
52-
53-
-- !sql --
54-
2
55-
5611
-- !sql --
5712
2
5813

@@ -98,18 +53,6 @@
9853
-- !sql --
9954
0
10055

101-
-- !sql --
102-
0
103-
104-
-- !sql --
105-
0
106-
107-
-- !sql --
108-
0
109-
110-
-- !sql --
111-
4
112-
11356
-- !sql --
11457
4
11558

@@ -119,42 +62,6 @@
11962
-- !sql --
12063
4
12164

122-
-- !sql --
123-
4
124-
125-
-- !sql --
126-
4
127-
128-
-- !sql --
129-
844
130-
131-
-- !sql --
132-
152
133-
134-
-- !sql --
135-
99
136-
137-
-- !sql --
138-
13
139-
140-
-- !sql --
141-
152
142-
143-
-- !sql --
144-
844
145-
146-
-- !sql --
147-
152
148-
149-
-- !sql --
150-
99
151-
152-
-- !sql --
153-
13
154-
155-
-- !sql --
156-
152
157-
15865
-- !sql --
15966
844
16067

0 commit comments

Comments
 (0)