Skip to content

Commit 4f887ac

Browse files
committed
[fix](lance) Source fixed-list contract facts from the reconstructed Arrow view
Verified against the pinned lance-core 9.1.0-beta.3 JNI bindings: - Dataset.getLanceSchema() returns LanceFields whose children are always empty for every fixed-size list (float32/float16, element nullable or not): the element type is collapsed into the manifest logical-type string (for example fixed_size_list:float:4). The previous contract path required exactly one Lance child, so it rejected every real vector column before the element facts were ever read. - Element nullability has no slot in the manifest. It is dropped on write and every read-side view synthesizes nullable=true (Java-side LanceField.childrenForFixedSizeList() hardcodes true). Existing datasets are equally affected; an SDK upgrade cannot backfill them. - The element type is recoverable: LanceField.asArrowField() synthesizes the element as the first child of the reconstructed Arrow view. The contract builder now takes vectorElementType and vectorElementNullable from that synthesized child, copying the synthesized nullability flag instead of pinning true so the contract improves automatically if a future SDK preserves element nullability. A missing Arrow view, an empty child list or a null child fails closed with the bounded ERR_LANCE_INDEX_INVALID error, matching the other malformed-provider-fact rejections. Tests: the mock fixtures stub asArrowField() with a synthetic always-nullable child exactly as the SDK produces it, and a new real JNI fixture creates a schema-only dataset via Dataset.create (f32[4] not-null top level, f16[3] nullable top level, int not null), reopens it, and pins the reconstructed-schema contract: fixed_size_list with dimensions 4/3, float32/float16 elements, vectorElementNullable=true, and preserved top-level nullability.
1 parent a53722f commit 4f887ac

5 files changed

Lines changed: 251 additions & 52 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceSchemaContractBuilder.java‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.arrow.vector.types.FloatingPointPrecision;
2727
import org.apache.arrow.vector.types.TimeUnit;
2828
import org.apache.arrow.vector.types.pojo.ArrowType;
29+
import org.apache.arrow.vector.types.pojo.Field;
2930
import org.lance.schema.LanceField;
3031

3132
import java.util.Arrays;
@@ -47,6 +48,18 @@
4748
* field id (0 is legal) or a non-positive dimension is a malformed provider fact and fails
4849
* closed the same way. The canonical vocabulary defined here is the Java-side authority the
4950
* Rust worker's golden fixtures align to (design section 4.2).
51+
*
52+
* <p>Fixed-size-list element facts are sourced from the reconstructed Arrow view, not from
53+
* the LanceField tree. The pinned SDK's {@code Dataset.getLanceSchema()} collapses the element
54+
* into the manifest logical-type string (for example {@code fixed_size_list:float:4}), so the
55+
* field's children are always empty, while {@link LanceField#asArrowField()} synthesizes the
56+
* element back as the first child of the Arrow view. The element type is recovered from that
57+
* synthesized child; its nullability is copied as synthesized — the manifest has no slot for
58+
* element nullability, so the reconstruction reports {@code true} today regardless of what was
59+
* written, and existing datasets are as affected as new ones. The contract therefore records
60+
* reconstructed-schema facts, which is exactly what the worker observes through the same SDK;
61+
* copying the synthesized flag instead of pinning {@code true} lets the contract improve
62+
* automatically if a future SDK preserves element nullability.
5063
*/
5164
final class LanceSchemaContractBuilder {
5265
/** Timezones that fit the canonical {@code tz="…"} slot without any escaping (IANA names). */
@@ -104,12 +117,19 @@ private static LanceIndexSchemaContract.IndexedField indexedField(LanceField fie
104117
"unsupported schema contract: fixed-size list dimension must be positive");
105118
}
106119
fixedSizeListDimension = listSize;
107-
List<LanceField> children = field.getChildren();
108-
if (children == null || children.size() != 1 || children.get(0) == null) {
109-
throw new IllegalArgumentException(
110-
"Lance fixed-size list field must have exactly one child");
120+
// The pinned SDK collapses the element into the manifest logical-type string, so the
121+
// LanceField tree never carries children for a fixed-size list; asArrowField()
122+
// re-synthesizes the element as the first child of the reconstructed Arrow view. The
123+
// synthesized child's nullability is always true — the manifest has no slot for
124+
// element nullability — and it is copied as-is so the contract tracks whatever the
125+
// SDK can reconstruct rather than pinning the limitation.
126+
Field arrowView = field.asArrowField();
127+
List<Field> elements = arrowView == null ? null : arrowView.getChildren();
128+
if (elements == null || elements.isEmpty() || elements.get(0) == null) {
129+
ErrorReport.reportAnalysisException(ErrorCode.ERR_LANCE_INDEX_INVALID,
130+
"unsupported schema contract: fixed-size list element must be present");
111131
}
112-
LanceField element = children.get(0);
132+
Field element = elements.get(0);
113133
vectorElementType = canonicalType(element.getType());
114134
vectorElementNullable = element.isNullable();
115135
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceIndexAdmissionTest.java‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939

4040
import org.apache.arrow.vector.types.FloatingPointPrecision;
4141
import org.apache.arrow.vector.types.pojo.ArrowType;
42+
import org.apache.arrow.vector.types.pojo.Field;
43+
import org.apache.arrow.vector.types.pojo.FieldType;
4244
import org.junit.jupiter.api.AfterEach;
4345
import org.junit.jupiter.api.Assertions;
4446
import org.junit.jupiter.api.BeforeEach;
@@ -48,7 +50,6 @@
4850
import org.mockito.MockedStatic;
4951
import org.mockito.Mockito;
5052

51-
import java.lang.reflect.Field;
5253
import java.util.ArrayList;
5354
import java.util.Arrays;
5455
import java.util.Collections;
@@ -114,7 +115,7 @@ public void setUp() throws Exception {
114115
Mockito.when(catalog.getId()).thenReturn(CATALOG_ID);
115116
Mockito.when(catalog.getProperties()).thenReturn(new HashMap<>());
116117
CatalogMgr catalogMgr = new CatalogMgr();
117-
Field catalogs = CatalogMgr.class.getDeclaredField("idToCatalog");
118+
java.lang.reflect.Field catalogs = CatalogMgr.class.getDeclaredField("idToCatalog");
118119
catalogs.setAccessible(true);
119120
@SuppressWarnings("unchecked")
120121
Map<Long, CatalogIf> registered = (Map<Long, CatalogIf>) catalogs.get(catalogMgr);
@@ -178,19 +179,20 @@ private static Column notNullColumn(String name, Type type) {
178179
}
179180

180181
private static LanceField vectorField(String name, int id) {
181-
LanceField element = Mockito.mock(LanceField.class);
182-
Mockito.when(element.getId()).thenReturn(id * 100 + 1);
183-
Mockito.when(element.getName()).thenReturn("item");
184-
Mockito.when(element.getType())
185-
.thenReturn(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE));
186-
Mockito.when(element.isNullable()).thenReturn(false);
187-
Mockito.when(element.getChildren()).thenReturn(Collections.emptyList());
182+
// Mirrors the pinned SDK: the LanceField tree carries no children for a fixed-size
183+
// list; the element lives only in the synthesized child of the reconstructed Arrow
184+
// view, always nullable.
185+
Field synthesizedElement = Field.nullable("item",
186+
new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE));
187+
Field arrowView = new Field(name, FieldType.notNullable(new ArrowType.FixedSizeList(4)),
188+
Collections.singletonList(synthesizedElement));
188189
LanceField field = Mockito.mock(LanceField.class);
189190
Mockito.when(field.getId()).thenReturn(id);
190191
Mockito.when(field.getName()).thenReturn(name);
191192
Mockito.when(field.getType()).thenReturn(new ArrowType.FixedSizeList(4));
192193
Mockito.when(field.isNullable()).thenReturn(false);
193-
Mockito.when(field.getChildren()).thenReturn(Collections.singletonList(element));
194+
Mockito.when(field.getChildren()).thenReturn(Collections.emptyList());
195+
Mockito.when(field.asArrowField()).thenReturn(arrowView);
194196
return field;
195197
}
196198

@@ -342,7 +344,9 @@ public void plainCreateOnEmptySnapshotIsAdmitted() throws Exception {
342344
Assertions.assertFalse(field.isNullable());
343345
Assertions.assertEquals(4, field.getFixedSizeListDimension());
344346
Assertions.assertEquals("float32", field.getVectorElementType());
345-
Assertions.assertEquals(Boolean.FALSE, field.getVectorElementNullable());
347+
// The synthesized element child is always nullable — the manifest has no slot for
348+
// element nullability — so the contract records the reconstructed-schema fact.
349+
Assertions.assertEquals(Boolean.TRUE, field.getVectorElementNullable());
346350

347351
Assertions.assertEquals(1, manager.getJobCount());
348352
Assertions.assertEquals(1, manager.editLog.size());
Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.datasource.lance;
19+
20+
import org.apache.doris.datasource.lance.job.LanceIndexSchemaContract;
21+
22+
import org.apache.arrow.memory.BufferAllocator;
23+
import org.apache.arrow.memory.RootAllocator;
24+
import org.apache.arrow.vector.types.FloatingPointPrecision;
25+
import org.apache.arrow.vector.types.pojo.ArrowType;
26+
import org.apache.arrow.vector.types.pojo.Field;
27+
import org.apache.arrow.vector.types.pojo.FieldType;
28+
import org.apache.arrow.vector.types.pojo.Schema;
29+
import org.junit.jupiter.api.AfterEach;
30+
import org.junit.jupiter.api.Assertions;
31+
import org.junit.jupiter.api.Test;
32+
import org.junit.jupiter.api.io.TempDir;
33+
import org.lance.Dataset;
34+
import org.lance.WriteParams;
35+
import org.lance.schema.LanceField;
36+
37+
import java.nio.file.Path;
38+
import java.util.Arrays;
39+
import java.util.Collections;
40+
import java.util.List;
41+
42+
/**
43+
* Pins the fixed-size-list contract semantics against a real on-disk dataset created through
44+
* the pinned lance-core JNI bindings (the jar bundles the native library, and the surefire
45+
* argLine already carries the opens the bindings need). A schema-only dataset is enough:
46+
* the manifest round trip behaves identically with and without data. What this fixture pins:
47+
* the LanceField tree of a fixed-size list never carries children (the element is collapsed
48+
* into the manifest logical-type string), the reconstructed Arrow view synthesizes the
49+
* element back, and the contract copies the element facts from that synthesized child —
50+
* including the always-true element nullability the manifest cannot store. Top-level
51+
* nullability and the scalar columns round trip exactly.
52+
*/
53+
public class LanceSchemaContractBuilderRealDatasetTest {
54+
@TempDir
55+
private Path tempDir;
56+
57+
private BufferAllocator allocator;
58+
59+
@AfterEach
60+
public void tearDown() {
61+
// The allocator backs the JNI export during create; close it deterministically.
62+
if (allocator != null) {
63+
allocator.close();
64+
allocator = null;
65+
}
66+
}
67+
68+
@Test
69+
public void fixedListContractMatchesTheReconstructedSchemaOfARealDataset() throws Exception {
70+
Schema schema = new Schema(Arrays.asList(
71+
new Field("v_f32", FieldType.notNullable(new ArrowType.FixedSizeList(4)),
72+
Collections.singletonList(Field.nullable("item",
73+
new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)))),
74+
new Field("v_f16", FieldType.nullable(new ArrowType.FixedSizeList(3)),
75+
Collections.singletonList(Field.nullable("item",
76+
new ArrowType.FloatingPoint(FloatingPointPrecision.HALF)))),
77+
Field.notNullable("c", new ArrowType.Int(32, true))));
78+
String uri = "file://" + tempDir.resolve("contract.lance").toAbsolutePath();
79+
80+
allocator = new RootAllocator(Long.MAX_VALUE);
81+
try (Dataset created = Dataset.create(allocator, uri, schema,
82+
new WriteParams.Builder().build())) {
83+
Assertions.assertTrue(created.version() >= 1);
84+
}
85+
86+
try (Dataset dataset = Dataset.open(uri)) {
87+
List<LanceField> fields = dataset.getLanceSchema().fields();
88+
89+
// The storage-format fact the contract works around: fixed-size lists come back
90+
// with an empty LanceField tree — the element exists only in the logical-type
91+
// string and in the synthesized child of the reconstructed Arrow view.
92+
for (LanceField field : fields) {
93+
if (field.getType() instanceof ArrowType.FixedSizeList) {
94+
Assertions.assertTrue(field.getChildren() == null
95+
|| field.getChildren().isEmpty(),
96+
"fixed-size list field must not carry Lance children");
97+
}
98+
}
99+
100+
LanceIndexSchemaContract.IndexedField f32 = indexedField(fields, "v_f32");
101+
Assertions.assertEquals("fixed_size_list", f32.getNormalizedType());
102+
Assertions.assertFalse(f32.isNullable());
103+
Assertions.assertEquals(4, f32.getFixedSizeListDimension());
104+
Assertions.assertEquals("float32", f32.getVectorElementType());
105+
Assertions.assertEquals(Boolean.TRUE, f32.getVectorElementNullable());
106+
107+
LanceIndexSchemaContract.IndexedField f16 = indexedField(fields, "v_f16");
108+
Assertions.assertEquals("fixed_size_list", f16.getNormalizedType());
109+
Assertions.assertTrue(f16.isNullable());
110+
Assertions.assertEquals(3, f16.getFixedSizeListDimension());
111+
Assertions.assertEquals("float16", f16.getVectorElementType());
112+
Assertions.assertEquals(Boolean.TRUE, f16.getVectorElementNullable());
113+
114+
LanceIndexSchemaContract.IndexedField scalar = indexedField(fields, "c");
115+
Assertions.assertEquals("int<32>", scalar.getNormalizedType());
116+
Assertions.assertFalse(scalar.isNullable());
117+
Assertions.assertNull(scalar.getFixedSizeListDimension());
118+
Assertions.assertNull(scalar.getVectorElementType());
119+
Assertions.assertNull(scalar.getVectorElementNullable());
120+
}
121+
}
122+
123+
private static LanceIndexSchemaContract.IndexedField indexedField(List<LanceField> fields,
124+
String storedColumnName) throws Exception {
125+
LanceIndexSchemaContract contract = LanceSchemaContractBuilder.build(fields,
126+
storedColumnName);
127+
Assertions.assertEquals(1, contract.getFields().size());
128+
return contract.getFields().get(0);
129+
}
130+
}

0 commit comments

Comments
 (0)