Skip to content

Commit 6363ee8

Browse files
[core][flink][spark][python][vector] Support vector indexes on DV tables
Allow deletion-vector tables to build vector global indexes whose readers can filter deleted row IDs. Opt in both Lumina and every paimon-vector native index factory while keeping other generic index types rejected.\n\nPin Spark and PyPaimon vector reads to the snapshot used for planning. Thread that snapshot through live-row filtering, scalar-index filtering, and raw fallback reads so materialized-deletion compaction cannot mix old index plans with new row-id or DV state.\n\nRefresh global-index deletions on every materialized-compaction commit attempt so indexes committed after preparation are still removed.
1 parent b35e19f commit 6363ee8

16 files changed

Lines changed: 425 additions & 24 deletions

File tree

‎paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexerFactory.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ default boolean supportsFullTextSearch() {
3333
return false;
3434
}
3535

36+
/**
37+
* Whether the read path filters DV-deleted rows, so building over all physical rows is safe.
38+
*/
39+
default boolean supportsDeletionVectors() {
40+
return false;
41+
}
42+
3643
GlobalIndexer create(DataField indexField, Options options);
3744

3845
/**

‎paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java‎

Lines changed: 63 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,9 @@ public int commit(ManifestCommittable committable, boolean checkAppendFiles) {
339339
int generatedSnapshot = 0;
340340
int attempts = 0;
341341

342-
ManifestEntryChanges changes = collectChanges(committable.fileCommittables());
342+
List<CommitMessage> commitMessages = committable.fileCommittables();
343+
ManifestEntryChanges changes = collectChanges(commitMessages);
344+
Set<BinaryRow> materializedPartitions = materializedPartitions(commitMessages);
343345
try {
344346
List<SimpleFileEntry> appendSimpleEntries =
345347
SimpleFileEntry.from(changes.appendTableFiles);
@@ -392,10 +394,7 @@ public int commit(ManifestCommittable committable, boolean checkAppendFiles) {
392394
|| !changes.compactIndexFiles.isEmpty()) {
393395
attempts +=
394396
tryCommit(
395-
CommitChangesProvider.provider(
396-
changes.compactTableFiles,
397-
changes.compactChangelog,
398-
changes.compactIndexFiles),
397+
compactChangesProvider(changes, materializedPartitions),
399398
committable.identifier(),
400399
committable.watermark(),
401400
committable.properties(),
@@ -766,6 +765,65 @@ private ManifestEntryChanges collectChanges(List<CommitMessage> commitMessages)
766765
return changes;
767766
}
768767

768+
private Set<BinaryRow> materializedPartitions(List<CommitMessage> commitMessages) {
769+
if (!options.dataEvolutionEnabled() || !options.deletionVectorsEnabled()) {
770+
return Collections.emptySet();
771+
}
772+
773+
Set<BinaryRow> result = new HashSet<>();
774+
for (CommitMessage message : commitMessages) {
775+
CommitMessageImpl commitMessage = (CommitMessageImpl) message;
776+
if (commitMessage.compactIncrement().compactBefore().stream()
777+
.noneMatch(
778+
file ->
779+
!isBlobFile(file.fileName())
780+
&& !isVectorStoreFile(file.fileName()))
781+
|| commitMessage.compactIncrement().compactAfter().stream()
782+
.anyMatch(file -> file.firstRowId() != null)) {
783+
continue;
784+
}
785+
result.add(commitMessage.partition());
786+
}
787+
return result;
788+
}
789+
790+
private CommitChangesProvider compactChangesProvider(
791+
ManifestEntryChanges changes, Set<BinaryRow> materializedPartitions) {
792+
if (materializedPartitions.isEmpty()) {
793+
return CommitChangesProvider.provider(
794+
changes.compactTableFiles, changes.compactChangelog, changes.compactIndexFiles);
795+
}
796+
797+
return latestSnapshot -> {
798+
List<IndexManifestEntry> indexFiles =
799+
changes.compactIndexFiles.stream()
800+
// Replace global-index deletions prepared against an older snapshot.
801+
.filter(
802+
entry ->
803+
entry.kind() != FileKind.DELETE
804+
|| entry.indexFile().globalIndexMeta() == null
805+
|| !materializedPartitions.contains(
806+
entry.partition()))
807+
.collect(Collectors.toList());
808+
809+
// This provider is invoked again after every optimistic-commit conflict. Scanning the
810+
// latest snapshot here guarantees that an index committed concurrently is either
811+
// deleted by this attempt or makes this attempt retry and is deleted by the next one.
812+
if (latestSnapshot != null && latestSnapshot.indexManifest() != null) {
813+
for (IndexManifestEntry entry :
814+
indexManifestFile.read(latestSnapshot.indexManifest())) {
815+
if (entry.indexFile().globalIndexMeta() != null
816+
&& materializedPartitions.contains(entry.partition())) {
817+
indexFiles.add(entry.toDeleteEntry());
818+
}
819+
}
820+
}
821+
822+
return new CommitChanges(
823+
changes.compactTableFiles, changes.compactChangelog, indexFiles);
824+
};
825+
}
826+
769827
private int tryCommit(
770828
CommitChangesProvider changesProvider,
771829
long identifier,

‎paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.apache.paimon.format.blob.BlobFileFormat;
3838
import org.apache.paimon.globalindex.IndexedSplit;
3939
import org.apache.paimon.index.DeletionVectorMeta;
40+
import org.apache.paimon.index.GlobalIndexMeta;
4041
import org.apache.paimon.index.IndexFileMeta;
4142
import org.apache.paimon.io.CompactIncrement;
4243
import org.apache.paimon.io.DataFileMeta;
@@ -737,6 +738,31 @@ public void testStaleMaterializeCompactionAllowsNonOverlappingConcurrentUpdate()
737738
"b|9|updated-9");
738739
}
739740

741+
@Test
742+
public void testMaterializeCompactionDropsGlobalIndexCommittedAfterPreparation()
743+
throws Exception {
744+
createTableDefault();
745+
FileStoreTable table = getTableDefault();
746+
writeBaseRows(table);
747+
updateStructuredColumn(table);
748+
commitDeletionVectors(table, DEFAULT_DV_SPECS);
749+
750+
Map<String, String> dynamicOptions = new HashMap<>();
751+
dynamicOptions.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2");
752+
dynamicOptions.put(CoreOptions.DATA_EVOLUTION_COMPACTION_REWRITE_ROW_IDS.key(), "true");
753+
FileStoreTable compactTable = table.copy(dynamicOptions);
754+
List<CommitMessage> compactMessages = prepareDataEvolutionCompaction(compactTable, false);
755+
756+
String indexFile = "concurrent-global-index";
757+
commitGlobalIndex(table, indexFile, FULL_RANGE);
758+
assertThat(liveGlobalIndexFileNames(table)).containsExactly(indexFile);
759+
760+
commit(compactTable, compactMessages);
761+
762+
assertThat(liveGlobalIndexFileNames(table)).isEmpty();
763+
assertThat(normalFileRowRanges(table)).containsExactly(new Range(15, 24));
764+
}
765+
740766
@Test
741767
public void testMaterializeCompactionUsesRemainingSizeForLargeDeletedRange() throws Exception {
742768
createTableDefault();
@@ -1137,6 +1163,11 @@ private void updateStructuredColumn(FileStoreTable table) throws Exception {
11371163

11381164
private void compactDataEvolutionTable(FileStoreTable table, boolean compactBlob)
11391165
throws Exception {
1166+
commit(table, prepareDataEvolutionCompaction(table, compactBlob));
1167+
}
1168+
1169+
private List<CommitMessage> prepareDataEvolutionCompaction(
1170+
FileStoreTable table, boolean compactBlob) throws Exception {
11401171
Snapshot snapshot = table.latestSnapshot().get();
11411172
DataEvolutionCompactCoordinator coordinator =
11421173
new DataEvolutionCompactCoordinator(table, compactBlob, false, snapshot);
@@ -1154,7 +1185,36 @@ private void compactDataEvolutionTable(FileStoreTable table, boolean compactBlob
11541185
commitMessages.addAll(
11551186
new DataEvolutionCompactionCommitPreparation(table, snapshot)
11561187
.prepare(commitMessages));
1157-
commit(table, commitMessages);
1188+
return commitMessages;
1189+
}
1190+
1191+
private void commitGlobalIndex(FileStoreTable table, String fileName, Range rowRange)
1192+
throws Exception {
1193+
IndexFileMeta indexFile =
1194+
new IndexFileMeta(
1195+
"test-global-index",
1196+
fileName,
1197+
1,
1198+
rowRange.count(),
1199+
new GlobalIndexMeta(rowRange.from, rowRange.to, 0, null, null),
1200+
null);
1201+
commit(
1202+
table,
1203+
Collections.singletonList(
1204+
new CommitMessageImpl(
1205+
BinaryRow.EMPTY_ROW,
1206+
UNAWARE_BUCKET,
1207+
null,
1208+
DataIncrement.indexIncrement(Collections.singletonList(indexFile)),
1209+
CompactIncrement.emptyIncrement())));
1210+
}
1211+
1212+
private static List<String> liveGlobalIndexFileNames(FileStoreTable table) {
1213+
return table.store().newIndexFileHandler().scanEntries().stream()
1214+
.filter(entry -> entry.indexFile().globalIndexMeta() != null)
1215+
.map(entry -> entry.indexFile().fileName())
1216+
.sorted()
1217+
.collect(Collectors.toList());
11581218
}
11591219

11601220
private void commitDeletionVectors(FileStoreTable table, List<DvSpec> deletionVectorSpecs)

‎paimon-flink/paimon-flink-common/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,13 @@ under the License.
239239
<scope>test</scope>
240240
</dependency>
241241

242+
<dependency>
243+
<groupId>org.apache.paimon</groupId>
244+
<artifactId>paimon-vector</artifactId>
245+
<version>${project.version}</version>
246+
<scope>test</scope>
247+
</dependency>
248+
242249
<dependency>
243250
<groupId>org.xerial</groupId>
244251
<artifactId>sqlite-jdbc</artifactId>

‎paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericGlobalIndexBuilder.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.paimon.flink.globalindex;
2020

2121
import org.apache.paimon.Snapshot;
22+
import org.apache.paimon.globalindex.GlobalIndexerFactoryUtils;
2223
import org.apache.paimon.manifest.IndexManifestEntry;
2324
import org.apache.paimon.manifest.ManifestEntry;
2425
import org.apache.paimon.partition.PartitionPredicate;
@@ -41,6 +42,7 @@ public class GenericGlobalIndexBuilder implements Serializable {
4142

4243
@Nullable protected PartitionPredicate partitionPredicate;
4344
@Nullable private Snapshot scanSnapshot;
45+
@Nullable private String indexType;
4446

4547
public GenericGlobalIndexBuilder(FileStoreTable table) {
4648
this.table = table;
@@ -51,6 +53,11 @@ public GenericGlobalIndexBuilder withPartitionPredicate(PartitionPredicate parti
5153
return this;
5254
}
5355

56+
public GenericGlobalIndexBuilder withIndexType(String indexType) {
57+
this.indexType = indexType;
58+
return this;
59+
}
60+
5461
public FileStoreTable table() {
5562
return table;
5663
}
@@ -67,11 +74,14 @@ public List<ManifestEntry> scan() {
6774
+ "but table '%s' has bucket = %d.",
6875
table.name(),
6976
table.coreOptions().bucket());
77+
// Allow DV only for index types that opt in (read path filters DV-deleted
78+
// rows via the live-row pre-filter, e.g. Lumina); reject all others.
7079
checkArgument(
71-
!table.coreOptions().deletionVectorsEnabled(),
72-
"Generic global index does not support tables with deletion vectors enabled. "
80+
!table.coreOptions().deletionVectorsEnabled() || supportsDeletionVectors(),
81+
"Global index type '%s' does not support tables with deletion vectors enabled. "
7382
+ "Table '%s' has 'deletion-vectors.enabled' = true, which may cause "
7483
+ "deleted rows to be indexed.",
84+
indexType,
7585
table.name());
7686

7787
scanSnapshot = table.snapshotManager().latestSnapshot();
@@ -87,6 +97,11 @@ public List<ManifestEntry> scan() {
8797
.files();
8898
}
8999

100+
private boolean supportsDeletionVectors() {
101+
return indexType != null
102+
&& GlobalIndexerFactoryUtils.load(indexType).supportsDeletionVectors();
103+
}
104+
90105
@Nullable
91106
public Snapshot scanSnapshot() {
92107
return scanSnapshot;

‎paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,7 @@ private static boolean buildIndexInternal(
298298
boolean autoIncremental)
299299
throws Exception {
300300
GenericGlobalIndexBuilder indexBuilder = indexBuilderSupplier.get();
301+
indexBuilder.withIndexType(indexType);
301302
if (partitionPredicate != null) {
302303
indexBuilder.withPartitionPredicate(partitionPredicate);
303304
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.paimon.flink.globalindex;
20+
21+
import org.apache.paimon.CoreOptions;
22+
import org.apache.paimon.table.FileStoreTable;
23+
import org.apache.paimon.utils.SnapshotManager;
24+
25+
import org.junit.jupiter.api.Test;
26+
27+
import java.util.Arrays;
28+
29+
import static org.assertj.core.api.Assertions.assertThatCode;
30+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
31+
import static org.mockito.Mockito.mock;
32+
import static org.mockito.Mockito.when;
33+
34+
/** Tests the deletion-vector guard in {@link GenericGlobalIndexBuilder}. */
35+
class GenericGlobalIndexBuilderDeletionVectorTest {
36+
37+
private static FileStoreTable table(boolean deletionVectorsEnabled) {
38+
FileStoreTable table = mock(FileStoreTable.class);
39+
CoreOptions options = mock(CoreOptions.class);
40+
when(options.bucket()).thenReturn(-1);
41+
when(options.deletionVectorsEnabled()).thenReturn(deletionVectorsEnabled);
42+
when(table.coreOptions()).thenReturn(options);
43+
when(table.name()).thenReturn("T");
44+
SnapshotManager snapshotManager = mock(SnapshotManager.class);
45+
when(snapshotManager.latestSnapshot()).thenReturn(null);
46+
when(table.snapshotManager()).thenReturn(snapshotManager);
47+
return table;
48+
}
49+
50+
@Test
51+
void testLuminaAllowedOnDeletionVectorTable() {
52+
assertThatCode(
53+
() ->
54+
new GenericGlobalIndexBuilder(table(true))
55+
.withIndexType("lumina")
56+
.scan())
57+
.doesNotThrowAnyException();
58+
}
59+
60+
@Test
61+
void testLegacyLuminaIdentifierAllowedOnDeletionVectorTable() {
62+
assertThatCode(
63+
() ->
64+
new GenericGlobalIndexBuilder(table(true))
65+
.withIndexType("lumina-vector-ann")
66+
.scan())
67+
.doesNotThrowAnyException();
68+
}
69+
70+
@Test
71+
void testNativeVectorIndexesAllowedOnDeletionVectorTable() {
72+
for (String indexType :
73+
Arrays.asList("ivf-flat", "ivf-pq", "ivf-sq", "ivf-rq", "diskann")) {
74+
assertThatCode(
75+
() ->
76+
new GenericGlobalIndexBuilder(table(true))
77+
.withIndexType(indexType)
78+
.scan())
79+
.as(indexType)
80+
.doesNotThrowAnyException();
81+
}
82+
}
83+
84+
@Test
85+
void testNonLuminaRejectedOnDeletionVectorTable() {
86+
assertThatThrownBy(
87+
() ->
88+
new GenericGlobalIndexBuilder(table(true))
89+
.withIndexType("bitmap")
90+
.scan())
91+
.hasMessageContaining("deletion vectors");
92+
}
93+
94+
@Test
95+
void testNonLuminaAllowedWithoutDeletionVectors() {
96+
assertThatCode(
97+
() ->
98+
new GenericGlobalIndexBuilder(table(false))
99+
.withIndexType("bitmap")
100+
.scan())
101+
.doesNotThrowAnyException();
102+
}
103+
}

‎paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexerFactory.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ public String identifier() {
3333
return IDENTIFIER;
3434
}
3535

36+
@Override
37+
public boolean supportsDeletionVectors() {
38+
// Vector search filters DV-deleted rows at read time (live-row pre-filter).
39+
return true;
40+
}
41+
3642
@Override
3743
public GlobalIndexer create(DataField field, Options options) {
3844
Options fieldOptions = LuminaVectorIndexOptions.resolveFieldOptions(field.name(), options);

0 commit comments

Comments
 (0)