Skip to content

Commit b4e55ea

Browse files
author
Jordan Epstein
committed
[common] Fix SliceComparator on BINARY and VARBINARY fields (apache#8873)
TypeCheckUtils.isComparable admits BINARY and VARBINARY, and primary keys may contain them, but SliceComparator cast every field value to Comparable — byte[] is not, so any sorted-lookup-store comparison over a binary key field (lookup compaction, deletion-vector maintenance, lookup changelog producer) threw ClassCastException. Compare binary fields with SortUtil.compareBinary, the same unsigned lexicographic order BinaryRow uses.
1 parent a998338 commit b4e55ea

2 files changed

Lines changed: 53 additions & 2 deletions

File tree

‎paimon-common/src/main/java/org/apache/paimon/data/serializer/RowCompactedSerializer.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.paimon.types.RowKind;
4141
import org.apache.paimon.types.RowType;
4242
import org.apache.paimon.types.VectorType;
43+
import org.apache.paimon.utils.SortUtil;
4344
import org.apache.paimon.utils.TypeCheckUtils;
4445
import org.apache.paimon.utils.VarLengthIntUtils;
4546

@@ -760,8 +761,16 @@ public int compare(MemorySlice slice1, MemorySlice slice2) {
760761
FieldReader fieldReader = fieldReaders[i];
761762
Object o1 = fieldReader.readField(reader1, i);
762763
Object o2 = fieldReader.readField(reader2, i);
763-
@SuppressWarnings({"unchecked", "rawtypes"})
764-
int comp = ((Comparable) o1).compareTo(o2);
764+
int comp;
765+
if (o1 instanceof byte[]) {
766+
// BINARY / VARBINARY fields read back as byte[], which does not
767+
// implement Comparable; order them like BinaryRow does.
768+
comp = SortUtil.compareBinary((byte[]) o1, (byte[]) o2);
769+
} else {
770+
@SuppressWarnings({"unchecked", "rawtypes"})
771+
int comparableComp = ((Comparable) o1).compareTo(o2);
772+
comp = comparableComp;
773+
}
765774
if (comp != 0) {
766775
return comp;
767776
}

‎paimon-common/src/test/java/org/apache/paimon/data/serializer/RowCompactedSerializerTest.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,21 @@
2323
import org.apache.paimon.data.GenericRow;
2424
import org.apache.paimon.data.InternalRow;
2525
import org.apache.paimon.data.variant.GenericVariant;
26+
import org.apache.paimon.memory.MemorySlice;
2627
import org.apache.paimon.types.DataType;
2728
import org.apache.paimon.types.DataTypes;
2829
import org.apache.paimon.types.RowType;
2930

3031
import org.junit.jupiter.api.Test;
3132

33+
import java.util.Comparator;
34+
3235
import static org.apache.paimon.data.BinaryString.fromString;
3336
import static org.apache.paimon.data.serializer.InternalRowSerializerTest.createArray;
3437
import static org.apache.paimon.data.serializer.InternalRowSerializerTest.createMap;
3538
import static org.apache.paimon.data.serializer.InternalRowSerializerTest.createRow;
3639
import static org.apache.paimon.data.serializer.InternalRowSerializerTest.deepEqualsInternalRow;
40+
import static org.assertj.core.api.Assertions.assertThat;
3741
import static org.assertj.core.api.Assertions.assertThatThrownBy;
3842

3943
/** Test for {@link RowCompactedSerializer}. */
@@ -202,6 +206,44 @@ private static RowCompactedSerializer getRowSerializer() {
202206
}
203207
}
204208

209+
static final class BinaryFieldTest extends RowCompactedSerializerTest {
210+
public BinaryFieldTest() {
211+
super(getRowSerializer(), getData());
212+
}
213+
214+
private static InternalRow[] getData() {
215+
return new GenericRow[] {
216+
GenericRow.of(1, new byte[] {1, 2, 3}),
217+
GenericRow.of(1, new byte[] {1, 2, 4}),
218+
GenericRow.of(2, new byte[] {(byte) 0xFF})
219+
};
220+
}
221+
222+
private static RowCompactedSerializer getRowSerializer() {
223+
return new RowCompactedSerializer(RowType.of(DataTypes.INT(), DataTypes.BYTES()));
224+
}
225+
226+
@Test
227+
public void testSliceComparatorOnBinaryFields() {
228+
RowCompactedSerializer serializer = getRowSerializer();
229+
Comparator<MemorySlice> comparator = serializer.createSliceComparator();
230+
MemorySlice small =
231+
MemorySlice.wrap(
232+
serializer.serializeToBytes(GenericRow.of(1, new byte[] {1, 2, 3})));
233+
MemorySlice large =
234+
MemorySlice.wrap(
235+
serializer.serializeToBytes(GenericRow.of(1, new byte[] {1, 2, 4})));
236+
MemorySlice unsigned =
237+
MemorySlice.wrap(
238+
serializer.serializeToBytes(
239+
GenericRow.of(1, new byte[] {(byte) 0xFF})));
240+
assertThat(comparator.compare(small, large)).isLessThan(0);
241+
assertThat(comparator.compare(large, small)).isGreaterThan(0);
242+
assertThat(comparator.compare(small, small)).isEqualTo(0);
243+
assertThat(comparator.compare(unsigned, small)).isGreaterThan(0);
244+
}
245+
}
246+
205247
static final class VectorTypesTest extends RowCompactedSerializerTest {
206248
public VectorTypesTest() {
207249
super(getRowSerializer(), getData());

0 commit comments

Comments
 (0)