Skip to content

Commit 712e06b

Browse files
ARROW-17431: [Java] MapBinder to bind Arrow Map type to DB column (#13941)
Typical real life Arrow datasets contain map of primitive type. This PR introduce MapBinder mapping of primitive types map entries Authored-by: igor.suhorukov <[email protected]> Signed-off-by: David Li <[email protected]>
1 parent 965e761 commit 712e06b

3 files changed

Lines changed: 276 additions & 1 deletion

File tree

‎java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import org.apache.arrow.vector.VarBinaryVector;
4646
import org.apache.arrow.vector.VarCharVector;
4747
import org.apache.arrow.vector.complex.ListVector;
48+
import org.apache.arrow.vector.complex.MapVector;
4849
import org.apache.arrow.vector.types.pojo.ArrowType;
4950

5051
/**
@@ -99,7 +100,7 @@ public ColumnBinder visit(ArrowType.Union type) {
99100

100101
@Override
101102
public ColumnBinder visit(ArrowType.Map type) {
102-
throw new UnsupportedOperationException("No column binder implemented for type " + type);
103+
return new MapBinder((MapVector) vector);
103104
}
104105

105106
@Override
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.arrow.adapter.jdbc.binder;
19+
20+
import java.sql.PreparedStatement;
21+
import java.sql.SQLException;
22+
import java.sql.Types;
23+
import java.util.LinkedHashMap;
24+
import java.util.List;
25+
import java.util.Objects;
26+
27+
import org.apache.arrow.vector.complex.MapVector;
28+
import org.apache.arrow.vector.complex.impl.UnionMapReader;
29+
import org.apache.arrow.vector.types.pojo.ArrowType;
30+
import org.apache.arrow.vector.types.pojo.Field;
31+
import org.apache.arrow.vector.util.JsonStringHashMap;
32+
33+
/**
34+
* A column binder for map of primitive values.
35+
*/
36+
public class MapBinder extends BaseColumnBinder<MapVector> {
37+
38+
private UnionMapReader reader;
39+
private final boolean isTextKey;
40+
private final boolean isTextValue;
41+
42+
public MapBinder(MapVector vector) {
43+
this(vector, Types.VARCHAR);
44+
}
45+
46+
/**
47+
* Init MapBinder and determine type of data vector.
48+
*
49+
* @param vector corresponding data vector from arrow buffer for binding
50+
* @param jdbcType parameter jdbc type
51+
*/
52+
public MapBinder(MapVector vector, int jdbcType) {
53+
super(vector, jdbcType);
54+
reader = vector.getReader();
55+
List<Field> structField = Objects.requireNonNull(vector.getField()).getChildren();
56+
if (structField.size() != 1) {
57+
throw new IllegalArgumentException("Expected Struct field metadata inside Map field");
58+
}
59+
List<Field> keyValueFields = Objects.requireNonNull(structField.get(0)).getChildren();
60+
if (keyValueFields.size() != 2) {
61+
throw new IllegalArgumentException("Expected two children fields " +
62+
"inside nested Struct field in Map");
63+
}
64+
ArrowType keyType = Objects.requireNonNull(keyValueFields.get(0)).getType();
65+
ArrowType valueType = Objects.requireNonNull(keyValueFields.get(1)).getType();
66+
isTextKey = ArrowType.Utf8.INSTANCE.equals(keyType);
67+
isTextValue = ArrowType.Utf8.INSTANCE.equals(valueType);
68+
}
69+
70+
@Override
71+
public void bind(PreparedStatement statement,
72+
int parameterIndex, int rowIndex) throws SQLException {
73+
reader.setPosition(rowIndex);
74+
LinkedHashMap<Object, Object> tags = new JsonStringHashMap<>();
75+
while (reader.next()) {
76+
Object key = reader.key().readObject();
77+
Object value = reader.value().readObject();
78+
tags.put(isTextKey && key != null ? key.toString() : key,
79+
isTextValue && value != null ? value.toString() : value);
80+
}
81+
switch (jdbcType) {
82+
case Types.VARCHAR:
83+
statement.setString(parameterIndex, tags.toString());
84+
break;
85+
case Types.OTHER:
86+
default:
87+
statement.setObject(parameterIndex, tags);
88+
}
89+
}
90+
}

‎java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import java.util.Arrays;
3131
import java.util.Collections;
3232
import java.util.List;
33+
import java.util.Map;
3334
import java.util.function.BiConsumer;
3435

3536
import org.apache.arrow.adapter.jdbc.binder.ColumnBinder;
@@ -69,13 +70,15 @@
6970
import org.apache.arrow.vector.VarCharVector;
7071
import org.apache.arrow.vector.VectorSchemaRoot;
7172
import org.apache.arrow.vector.complex.ListVector;
73+
import org.apache.arrow.vector.complex.MapVector;
7274
import org.apache.arrow.vector.types.DateUnit;
7375
import org.apache.arrow.vector.types.FloatingPointPrecision;
7476
import org.apache.arrow.vector.types.TimeUnit;
7577
import org.apache.arrow.vector.types.pojo.ArrowType;
7678
import org.apache.arrow.vector.types.pojo.Field;
7779
import org.apache.arrow.vector.types.pojo.FieldType;
7880
import org.apache.arrow.vector.types.pojo.Schema;
81+
import org.apache.arrow.vector.util.JsonStringHashMap;
7982
import org.junit.jupiter.api.AfterEach;
8083
import org.junit.jupiter.api.BeforeEach;
8184
import org.junit.jupiter.api.Test;
@@ -473,6 +476,81 @@ void listOfString() throws SQLException {
473476
testListType((ArrowType) new ArrowType.Utf8(), setValue, ListVector::setNull, values);
474477
}
475478

479+
@Test
480+
void mapOfString() throws SQLException {
481+
TriConsumer<MapVector, Integer, Map<String, String>> setValue = (mapVector, index, values) -> {
482+
org.apache.arrow.vector.complex.impl.UnionMapWriter mapWriter = mapVector.getWriter();
483+
mapWriter.setPosition(index);
484+
mapWriter.startMap();
485+
values.entrySet().forEach(mapValue -> {
486+
if (mapValue != null) {
487+
byte[] keyBytes = mapValue.getKey().getBytes(StandardCharsets.UTF_8);
488+
byte[] valueBytes = mapValue.getValue().getBytes(StandardCharsets.UTF_8);
489+
try (
490+
ArrowBuf keyBuf = allocator.buffer(keyBytes.length);
491+
ArrowBuf valueBuf = allocator.buffer(valueBytes.length);
492+
) {
493+
mapWriter.startEntry();
494+
keyBuf.writeBytes(keyBytes);
495+
valueBuf.writeBytes(valueBytes);
496+
mapWriter.key().varChar().writeVarChar(0, keyBytes.length, keyBuf);
497+
mapWriter.value().varChar().writeVarChar(0, valueBytes.length, valueBuf);
498+
mapWriter.endEntry();
499+
}
500+
} else {
501+
mapWriter.writeNull();
502+
}
503+
});
504+
mapWriter.endMap();
505+
};
506+
507+
JsonStringHashMap<String, String> value1 = new JsonStringHashMap<String, String>();
508+
value1.put("a", "b");
509+
value1.put("c", "d");
510+
JsonStringHashMap<String, String> value2 = new JsonStringHashMap<String, String>();
511+
value2.put("d", "e");
512+
value2.put("f", "g");
513+
value2.put("k", "l");
514+
JsonStringHashMap<String, String> value3 = new JsonStringHashMap<String, String>();
515+
value3.put("y", "z");
516+
value3.put("arrow", "cool");
517+
List<Map<String, String>> values = Arrays.asList(value1, value2, value3, Collections.emptyMap());
518+
testMapType(new ArrowType.Map(true), setValue, MapVector::setNull, values, new ArrowType.Utf8());
519+
}
520+
521+
@Test
522+
void mapOfInteger() throws SQLException {
523+
TriConsumer<MapVector, Integer, Map<Integer, Integer>> setValue = (mapVector, index, values) -> {
524+
org.apache.arrow.vector.complex.impl.UnionMapWriter mapWriter = mapVector.getWriter();
525+
mapWriter.setPosition(index);
526+
mapWriter.startMap();
527+
values.entrySet().forEach(mapValue -> {
528+
if (mapValue != null) {
529+
mapWriter.startEntry();
530+
mapWriter.key().integer().writeInt(mapValue.getKey());
531+
mapWriter.value().integer().writeInt(mapValue.getValue());
532+
mapWriter.endEntry();
533+
} else {
534+
mapWriter.writeNull();
535+
}
536+
});
537+
mapWriter.endMap();
538+
};
539+
540+
JsonStringHashMap<Integer, Integer> value1 = new JsonStringHashMap<Integer, Integer>();
541+
value1.put(1, 2);
542+
value1.put(3, 4);
543+
JsonStringHashMap<Integer, Integer> value2 = new JsonStringHashMap<Integer, Integer>();
544+
value2.put(5, 6);
545+
value2.put(7, 8);
546+
value2.put(9, 1024);
547+
JsonStringHashMap<Integer, Integer> value3 = new JsonStringHashMap<Integer, Integer>();
548+
value3.put(Integer.MIN_VALUE, Integer.MAX_VALUE);
549+
value3.put(0, 4096);
550+
List<Map<Integer, Integer>> values = Arrays.asList(value1, value2, value3, Collections.emptyMap());
551+
testMapType(new ArrowType.Map(true), setValue, MapVector::setNull, values, new ArrowType.Int(32, true));
552+
}
553+
476554
@FunctionalInterface
477555
interface TriConsumer<T, U, V> {
478556
void accept(T value1, U value2, V value3);
@@ -672,4 +750,110 @@ <T, V extends FieldVector> void testListType(ArrowType arrowType, TriConsumer<V,
672750
assertThat(binder.next()).isFalse();
673751
}
674752
}
753+
754+
<T, V extends FieldVector> void testMapType(ArrowType arrowType, TriConsumer<V, Integer, T> setValue,
755+
BiConsumer<V, Integer> setNull, List<T> values,
756+
ArrowType elementType) throws SQLException {
757+
int jdbcType = Types.VARCHAR;
758+
FieldType keyType = new FieldType(false, elementType, null, null);
759+
FieldType mapType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null);
760+
Schema schema = new Schema(Collections.singletonList(new Field("field", FieldType.nullable(arrowType),
761+
Collections.singletonList(new Field(MapVector.KEY_NAME, mapType,
762+
Arrays.asList(new Field(MapVector.KEY_NAME, keyType, null),
763+
new Field(MapVector.VALUE_NAME, keyType, null)))))));
764+
try (final MockPreparedStatement statement = new MockPreparedStatement();
765+
final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
766+
final JdbcParameterBinder binder =
767+
JdbcParameterBinder.builder(statement, root).bindAll().build();
768+
assertThat(binder.next()).isFalse();
769+
770+
@SuppressWarnings("unchecked")
771+
final V vector = (V) root.getVector(0);
772+
final ColumnBinder columnBinder = ColumnBinder.forVector(vector);
773+
assertThat(columnBinder.getJdbcType()).isEqualTo(jdbcType);
774+
775+
setValue.accept(vector, 0, values.get(0));
776+
setValue.accept(vector, 1, values.get(1));
777+
setNull.accept(vector, 2);
778+
root.setRowCount(3);
779+
780+
assertThat(binder.next()).isTrue();
781+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(0).toString());
782+
assertThat(binder.next()).isTrue();
783+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(1).toString());
784+
assertThat(binder.next()).isTrue();
785+
assertThat(statement.getParamValue(1)).isNull();
786+
assertThat(statement.getParamType(1)).isEqualTo(jdbcType);
787+
assertThat(binder.next()).isFalse();
788+
789+
binder.reset();
790+
791+
setNull.accept(vector, 0);
792+
setValue.accept(vector, 1, values.get(3));
793+
setValue.accept(vector, 2, values.get(0));
794+
setValue.accept(vector, 3, values.get(2));
795+
setValue.accept(vector, 4, values.get(1));
796+
root.setRowCount(5);
797+
798+
assertThat(binder.next()).isTrue();
799+
assertThat(statement.getParamValue(1)).isNull();
800+
assertThat(statement.getParamType(1)).isEqualTo(jdbcType);
801+
assertThat(binder.next()).isTrue();
802+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(3).toString());
803+
assertThat(binder.next()).isTrue();
804+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(0).toString());
805+
assertThat(binder.next()).isTrue();
806+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(2).toString());
807+
assertThat(binder.next()).isTrue();
808+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(1).toString());
809+
assertThat(binder.next()).isFalse();
810+
}
811+
812+
// Non-nullable (since some types have a specialized binder)
813+
schema = new Schema(Collections.singletonList(new Field("field", FieldType.notNullable(arrowType),
814+
Collections.singletonList(new Field(MapVector.KEY_NAME, mapType,
815+
Arrays.asList(new Field(MapVector.KEY_NAME, keyType, null),
816+
new Field(MapVector.VALUE_NAME, keyType, null)))))));
817+
try (final MockPreparedStatement statement = new MockPreparedStatement();
818+
final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
819+
@SuppressWarnings("unchecked")
820+
final V vector = (V) root.getVector(0);
821+
822+
final JdbcParameterBinder binder =
823+
JdbcParameterBinder.builder(statement, root).bind(1,
824+
new org.apache.arrow.adapter.jdbc.binder.MapBinder((MapVector) vector, Types.OTHER)).build();
825+
assertThat(binder.next()).isFalse();
826+
827+
setValue.accept(vector, 0, values.get(0));
828+
setValue.accept(vector, 1, values.get(1));
829+
root.setRowCount(2);
830+
831+
assertThat(binder.next()).isTrue();
832+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(0));
833+
assertThat(binder.next()).isTrue();
834+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(1));
835+
assertThat(binder.next()).isFalse();
836+
837+
binder.reset();
838+
839+
setValue.accept(vector, 0, values.get(0));
840+
setValue.accept(vector, 1, values.get(2));
841+
setValue.accept(vector, 2, values.get(0));
842+
setValue.accept(vector, 3, values.get(2));
843+
setValue.accept(vector, 4, values.get(1));
844+
root.setRowCount(5);
845+
846+
assertThat(binder.next()).isTrue();
847+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(0));
848+
assertThat(binder.next()).isTrue();
849+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(2));
850+
assertThat(binder.next()).isTrue();
851+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(0));
852+
assertThat(binder.next()).isTrue();
853+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(2));
854+
assertThat(binder.next()).isTrue();
855+
assertThat(statement.getParamValue(1)).isEqualTo(values.get(1));
856+
assertThat(binder.next()).isFalse();
857+
}
858+
}
675859
}

0 commit comments

Comments
 (0)