Skip to content

Commit 2ab66d7

Browse files
authored
feat: reduce allocations in Spark Decimal access (#9842)
## Summary Closes #9837. See the issue for the motivation, design, benchmark results, and decimal-specific validation. ## Changes - Add `SmallDecimalAccessor` for source precision 1–18, preserving full-width fallback and Spark's decimal conversion semantics. - Add regression tests for decimal decoding, conversions, and returned-value independence. AI assistance: AI assistance was used for implementation, tests, validation tooling, and this description, including translation from Chinese. --------- Signed-off-by: Peifeng Li <[email protected]>
1 parent da48008 commit 2ab66d7

2 files changed

Lines changed: 355 additions & 1 deletion

File tree

‎java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexArrowColumnVector.java‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
import dev.vortex.relocated.org.apache.arrow.vector.types.TimeUnit;
1616
import dev.vortex.relocated.org.apache.arrow.vector.types.pojo.ArrowType;
1717
import dev.vortex.spark.ArrowUtils;
18+
import java.math.BigDecimal;
19+
import java.nio.ByteOrder;
1820
import org.apache.spark.sql.types.DataType;
1921
import org.apache.spark.sql.types.DataTypes;
2022
import org.apache.spark.sql.types.Decimal;
@@ -309,7 +311,12 @@ void initAccessor(ValueVector vector) {
309311
} else if (vector instanceof Float8Vector float8Vector) {
310312
accessor = new VortexArrowColumnVector.DoubleAccessor(float8Vector);
311313
} else if (vector instanceof DecimalVector decimalVector) {
312-
accessor = new VortexArrowColumnVector.DecimalAccessor(decimalVector);
314+
int precision = decimalVector.getPrecision();
315+
if (precision > 0 && precision <= Decimal.MAX_LONG_DIGITS()) {
316+
accessor = new VortexArrowColumnVector.SmallDecimalAccessor(decimalVector, ByteOrder.nativeOrder());
317+
} else {
318+
accessor = new VortexArrowColumnVector.DecimalAccessor(decimalVector);
319+
}
313320
} else if (vector instanceof VarCharVector varCharVector) {
314321
accessor = new VortexArrowColumnVector.StringAccessor(varCharVector);
315322
} else if (vector instanceof LargeVarCharVector largeVarCharVector) {
@@ -556,6 +563,38 @@ final Decimal getDecimal(int rowId, int precision, int scale) {
556563
}
557564
}
558565

566+
static final class SmallDecimalAccessor extends VortexArrowColumnVector.ArrowVectorAccessor {
567+
568+
private final DecimalVector accessor;
569+
private final int lowWordOffset;
570+
private final int highWordOffset;
571+
572+
SmallDecimalAccessor(DecimalVector vector, ByteOrder byteOrder) {
573+
super(vector);
574+
this.accessor = vector;
575+
// Arrow Java stores Decimal128 in native byte order, including the order of its two words.
576+
this.lowWordOffset = byteOrder == ByteOrder.LITTLE_ENDIAN ? 0 : Long.BYTES;
577+
this.highWordOffset = byteOrder == ByteOrder.LITTLE_ENDIAN ? Long.BYTES : 0;
578+
}
579+
580+
@Override
581+
final Decimal getDecimal(int rowId, int precision, int scale) {
582+
if (isNullAt(rowId)) {
583+
return null;
584+
}
585+
long offset = (long) rowId * DecimalVector.TYPE_WIDTH;
586+
long unscaled = accessor.getDataBuffer().getLong(offset + lowWordOffset);
587+
long high = accessor.getDataBuffer().getLong(offset + highWordOffset);
588+
if (high == (unscaled >> 63)) {
589+
// Decode with the source scale; Spark performs the requested rescaling and overflow checks.
590+
// Keep the expanded Decimal representation for Spark's checked integer casts.
591+
return Decimal.apply(BigDecimal.valueOf(unscaled, accessor.getScale()), precision, scale);
592+
}
593+
// Preserve all 128 bits when the stored integer does not fit in a long.
594+
return Decimal.apply(accessor.getObject(rowId), precision, scale);
595+
}
596+
}
597+
559598
@Open
560599
static class StringAccessor extends VortexArrowColumnVector.ArrowVectorAccessor {
561600

Lines changed: 315 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,315 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
package dev.vortex.spark.read;
5+
6+
import static org.junit.jupiter.api.Assertions.assertEquals;
7+
import static org.junit.jupiter.api.Assertions.assertNotSame;
8+
import static org.junit.jupiter.api.Assertions.assertNull;
9+
import static org.junit.jupiter.api.Assertions.assertThrows;
10+
11+
import dev.vortex.relocated.org.apache.arrow.memory.RootAllocator;
12+
import dev.vortex.relocated.org.apache.arrow.vector.DecimalVector;
13+
import java.math.BigDecimal;
14+
import java.math.BigInteger;
15+
import java.nio.ByteOrder;
16+
import java.util.ArrayList;
17+
import java.util.List;
18+
import java.util.Random;
19+
import java.util.function.Supplier;
20+
import org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter;
21+
import org.apache.spark.sql.internal.SQLConf;
22+
import org.apache.spark.sql.types.Decimal;
23+
import org.junit.jupiter.api.Test;
24+
25+
final class VortexDecimalAccessorTest {
26+
@Test
27+
void compactPrecisionsAndScalesMatchArrowValues() {
28+
Random random = new Random(1024);
29+
try (RootAllocator allocator = new RootAllocator()) {
30+
long limit = 1;
31+
for (int precision = 1; precision <= Decimal.MAX_LONG_DIGITS(); precision++) {
32+
limit *= 10;
33+
List<Long> values = new ArrayList<>(List.of(0L, 1L, -1L, limit - 1, 1 - limit));
34+
for (long power = 10; power < limit; power *= 10) {
35+
values.add(power - 1);
36+
values.add(power);
37+
values.add(power + 1);
38+
values.add(1 - power);
39+
values.add(-power);
40+
values.add(-power - 1);
41+
}
42+
for (int i = 0; i < 32; i++) {
43+
values.add(random.nextLong() % limit);
44+
}
45+
for (int scale = 0; scale <= precision; scale++) {
46+
try (DecimalVector vector = new DecimalVector("decimal", allocator, precision, scale)) {
47+
vector.allocateNew(values.size() + 1);
48+
for (int row = 0; row < values.size(); row++) {
49+
vector.set(row, BigDecimal.valueOf(values.get(row), scale));
50+
}
51+
vector.setNull(values.size());
52+
vector.setValueCount(values.size() + 1);
53+
VortexArrowColumnVector column = new VortexArrowColumnVector(vector);
54+
for (int row = values.size() - 1; row >= 0; row--) {
55+
BigDecimal expected = BigDecimal.valueOf(values.get(row), scale);
56+
Decimal actual = column.getDecimal(row, precision, scale);
57+
assertEquivalent(Decimal.apply(expected, precision, scale), actual, precision, scale);
58+
}
59+
assertNull(column.getDecimal(values.size(), precision, scale));
60+
}
61+
}
62+
}
63+
}
64+
}
65+
66+
@Test
67+
void smallDecimalReadsNativeWordsInEitherOrder() {
68+
String[] inputs = {"0.00", "123.45", "-123.45", "9999999999999999.99", "-9999999999999999.99"};
69+
try (RootAllocator allocator = new RootAllocator()) {
70+
for (ByteOrder order : new ByteOrder[] {ByteOrder.LITTLE_ENDIAN, ByteOrder.BIG_ENDIAN}) {
71+
try (DecimalVector vector = new DecimalVector("decimal", allocator, 18, 2)) {
72+
vector.allocateNew(inputs.length + 1);
73+
for (int row = 0; row < inputs.length; row++) {
74+
BigDecimal value = new BigDecimal(inputs[row]);
75+
vector.set(row, value);
76+
long offset = (long) row * DecimalVector.TYPE_WIDTH;
77+
long low = value.unscaledValue().longValueExact();
78+
long high = value.signum() < 0 ? -1L : 0L;
79+
// Model the two native-word positions; ArrowBuf supplies native-endian long reads.
80+
vector.getDataBuffer().setLong(offset, order == ByteOrder.LITTLE_ENDIAN ? low : high);
81+
vector.getDataBuffer()
82+
.setLong(offset + Long.BYTES, order == ByteOrder.LITTLE_ENDIAN ? high : low);
83+
}
84+
vector.setNull(inputs.length);
85+
vector.setValueCount(inputs.length + 1);
86+
var accessor = new VortexArrowColumnVector.SmallDecimalAccessor(vector, order);
87+
for (int row = 0; row < inputs.length; row++) {
88+
BigDecimal expected = new BigDecimal(inputs[row]);
89+
assertEquivalent(Decimal.apply(expected, 18, 2), accessor.getDecimal(row, 18, 2), 18, 2);
90+
}
91+
assertNull(accessor.getDecimal(inputs.length, 18, 2));
92+
}
93+
}
94+
}
95+
}
96+
97+
@Test
98+
void requestedScaleAndPrecisionRetainRoundingAndOverflow() {
99+
try (RootAllocator allocator = new RootAllocator();
100+
DecimalVector vector = new DecimalVector("decimal", allocator, 5, 2)) {
101+
vector.allocateNew(4);
102+
vector.set(0, new BigDecimal("123.45"));
103+
vector.set(1, new BigDecimal("-123.45"));
104+
vector.set(2, new BigDecimal("999.95"));
105+
vector.setNull(3);
106+
vector.setValueCount(4);
107+
VortexArrowColumnVector column = new VortexArrowColumnVector(vector);
108+
assertEquals(new BigDecimal("123.5"), column.getDecimal(0, 5, 1).toJavaBigDecimal());
109+
assertEquals(new BigDecimal("-123.5"), column.getDecimal(1, 5, 1).toJavaBigDecimal());
110+
assertEquals(
111+
new BigDecimal("123.450000000000000000"),
112+
column.getDecimal(0, 38, 18).toJavaBigDecimal());
113+
for (int row = 0; row < 3; row++) {
114+
for (int precision : new int[] {2, 4, 5, 8, 19}) {
115+
for (int scale : new int[] {0, 1, 2, 3}) {
116+
assertMatchesReference(vector, column, row, precision, scale);
117+
}
118+
}
119+
}
120+
assertNull(column.getDecimal(3, 2, 0));
121+
}
122+
}
123+
124+
@Test
125+
void wideVectorsRetainAllBitsEvenWhenRequestedPrecisionIsSmall() {
126+
try (RootAllocator allocator = new RootAllocator()) {
127+
for (int precision : new int[] {19, 20, 38}) {
128+
BigInteger largest = BigInteger.TEN.pow(precision).subtract(BigInteger.ONE);
129+
try (DecimalVector vector = new DecimalVector("decimal", allocator, precision, 2)) {
130+
vector.allocateNew(4);
131+
vector.set(0, new BigDecimal(largest, 2));
132+
vector.set(1, new BigDecimal(largest.negate(), 2));
133+
vector.set(2, new BigDecimal("1.23"));
134+
vector.setNull(3);
135+
vector.setValueCount(4);
136+
VortexArrowColumnVector column = new VortexArrowColumnVector(vector);
137+
for (int row = 0; row < 3; row++) {
138+
assertMatchesReference(vector, column, row, precision, 2);
139+
assertMatchesReference(vector, column, row, 18, 2);
140+
}
141+
assertNull(column.getDecimal(3, precision, 2));
142+
}
143+
}
144+
}
145+
}
146+
147+
@Test
148+
void outOfPrecisionBufferValuesRetainReferenceResultsAndErrors() {
149+
try (RootAllocator allocator = new RootAllocator();
150+
DecimalVector vector = new DecimalVector("decimal", allocator, 18, 0)) {
151+
// The byte setter permits values inconsistent with the field's precision.
152+
BigInteger[] invalid = {
153+
BigInteger.ONE.shiftLeft(64).add(BigInteger.valueOf(7)),
154+
BigInteger.ONE.shiftLeft(64).negate().add(BigInteger.valueOf(7)),
155+
BigInteger.TEN.pow(18),
156+
BigInteger.TEN.pow(18).negate(),
157+
BigInteger.valueOf(Long.MAX_VALUE),
158+
BigInteger.valueOf(Long.MIN_VALUE),
159+
BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE),
160+
BigInteger.valueOf(Long.MIN_VALUE).subtract(BigInteger.ONE)
161+
};
162+
vector.allocateNew(invalid.length);
163+
for (int row = 0; row < invalid.length; row++) {
164+
vector.setBigEndian(row, invalid[row].toByteArray());
165+
}
166+
vector.setValueCount(invalid.length);
167+
VortexArrowColumnVector column = new VortexArrowColumnVector(vector);
168+
for (int row = 0; row < invalid.length; row++) {
169+
assertMatchesReference(vector, column, row, 18, 0);
170+
assertMatchesReference(vector, column, row, 38, 0);
171+
}
172+
vector.setNull(0);
173+
assertNull(column.getDecimal(0, 18, 0));
174+
}
175+
}
176+
177+
@Test
178+
void slicedVectorsAndReturnedValuesRemainIndependent() {
179+
try (RootAllocator allocator = new RootAllocator();
180+
DecimalVector vector = new DecimalVector("decimal", allocator, 10, 2)) {
181+
vector.allocateNew(4);
182+
vector.set(0, new BigDecimal("999.99"));
183+
vector.set(1, new BigDecimal("-12.34"));
184+
vector.setNull(2);
185+
vector.set(3, new BigDecimal("56.78"));
186+
vector.setValueCount(4);
187+
var transfer = vector.getTransferPair(allocator);
188+
transfer.splitAndTransfer(1, 3);
189+
try (DecimalVector slice = (DecimalVector) transfer.getTo()) {
190+
VortexArrowColumnVector column = new VortexArrowColumnVector(slice);
191+
Decimal first = column.getDecimal(0, 10, 2);
192+
Decimal repeated = column.getDecimal(0, 10, 2);
193+
Decimal last = column.getDecimal(2, 10, 2);
194+
assertNotSame(first, repeated);
195+
assertEquals(new BigDecimal("-12.34"), first.toJavaBigDecimal());
196+
assertEquals(new BigDecimal("56.78"), last.toJavaBigDecimal());
197+
first.set(0L);
198+
assertEquals(new BigDecimal("-12.34"), repeated.toJavaBigDecimal());
199+
assertEquals(new BigDecimal("56.78"), last.toJavaBigDecimal());
200+
assertNull(column.getDecimal(1, 10, 2));
201+
}
202+
}
203+
}
204+
205+
@Test
206+
void negativeScaleRetainsReferenceConversionsAndValidation() {
207+
SQLConf conf = SQLConf.get();
208+
String key = "spark.sql.legacy.allowNegativeScaleOfDecimal";
209+
String previous = conf.getConfString(key, "false");
210+
boolean wasSet = conf.getAllConfs().contains(key);
211+
conf.setConfString(key, "true");
212+
try (RootAllocator allocator = new RootAllocator();
213+
DecimalVector vector = new DecimalVector("decimal", allocator, 5, -2)) {
214+
vector.allocateNew(1);
215+
vector.set(0, new BigDecimal("1.23E+4"));
216+
vector.setValueCount(1);
217+
VortexArrowColumnVector column = new VortexArrowColumnVector(vector);
218+
assertMatchesReference(vector, column, 0, 5, -2);
219+
assertMatchesReference(vector, column, 0, 8, 0);
220+
assertMatchesReference(vector, column, 0, 5, -3);
221+
assertEquals(12300L, column.getDecimal(0, 5, -2).toLong());
222+
conf.setConfString(key, "false");
223+
assertMatchesReference(vector, column, 0, 5, -2);
224+
} finally {
225+
if (wasSet) {
226+
conf.setConfString(key, previous);
227+
} else {
228+
conf.unsetConf(key);
229+
}
230+
}
231+
}
232+
233+
@Test
234+
void checkedIntegralConversionsRetainResultsAndOverflow() {
235+
String[] values = {
236+
"127.999999999999999",
237+
"-128.999999999999999",
238+
"32767.9999999999999",
239+
"-32768.9999999999999",
240+
"2147483647.99999999",
241+
"-2147483648.99999999",
242+
"123456789.999999999"
243+
};
244+
try (RootAllocator allocator = new RootAllocator()) {
245+
for (String value : values) {
246+
BigDecimal input = new BigDecimal(value);
247+
int precision = input.precision();
248+
int scale = input.scale();
249+
try (DecimalVector vector = new DecimalVector("decimal", allocator, precision, scale)) {
250+
vector.allocateNew(1);
251+
vector.set(0, input);
252+
vector.setValueCount(1);
253+
VortexArrowColumnVector column = new VortexArrowColumnVector(vector);
254+
Decimal expected = Decimal.apply(vector.getObject(0), precision, scale);
255+
Decimal actual = column.getDecimal(0, precision, scale);
256+
assertConversion(expected::roundToByte, actual::roundToByte);
257+
assertConversion(expected::roundToShort, actual::roundToShort);
258+
assertConversion(expected::roundToInt, actual::roundToInt);
259+
assertConversion(expected::roundToLong, actual::roundToLong);
260+
}
261+
}
262+
}
263+
}
264+
265+
private static void assertConversion(Supplier<Object> expectedConversion, Supplier<Object> actualConversion) {
266+
Object expected;
267+
try {
268+
expected = expectedConversion.get();
269+
} catch (Exception expectedError) {
270+
Exception actualError = assertThrows(expectedError.getClass(), actualConversion::get);
271+
assertEquals(expectedError.getClass(), actualError.getClass());
272+
assertEquals(expectedError.getMessage(), actualError.getMessage());
273+
return;
274+
}
275+
assertEquals(expected, actualConversion.get());
276+
}
277+
278+
private static void assertMatchesReference(
279+
DecimalVector vector, VortexArrowColumnVector column, int row, int precision, int scale) {
280+
Decimal expected;
281+
try {
282+
expected = Decimal.apply(vector.getObject(row), precision, scale);
283+
} catch (Exception expectedError) {
284+
Exception actualError =
285+
assertThrows(expectedError.getClass(), () -> column.getDecimal(row, precision, scale));
286+
assertEquals(expectedError.getClass(), actualError.getClass());
287+
assertEquals(expectedError.getMessage(), actualError.getMessage());
288+
return;
289+
}
290+
assertEquivalent(expected, column.getDecimal(row, precision, scale), precision, scale);
291+
}
292+
293+
private static void assertEquivalent(Decimal expected, Decimal actual, int precision, int scale) {
294+
assertEquals(expected.toJavaBigDecimal(), actual.toJavaBigDecimal());
295+
assertEquals(expected.precision(), actual.precision());
296+
assertEquals(expected.scale(), actual.scale());
297+
assertEquals(expected.hashCode(), actual.hashCode());
298+
assertEquals(0, expected.compare(actual));
299+
assertEquals(expected.toLong(), actual.toLong());
300+
assertEquals(expected, actual.clone());
301+
Decimal expectedCopy = expected.clone();
302+
Decimal actualCopy = actual.clone();
303+
assertEquals(expectedCopy.changePrecision(precision, 0), actualCopy.changePrecision(precision, 0));
304+
assertEquals(expectedCopy.toJavaBigDecimal(), actualCopy.toJavaBigDecimal());
305+
UnsafeRowWriter expectedWriter = new UnsafeRowWriter(1);
306+
UnsafeRowWriter actualWriter = new UnsafeRowWriter(1);
307+
expectedWriter.resetRowWriter();
308+
actualWriter.resetRowWriter();
309+
expectedWriter.write(0, expected, precision, scale);
310+
actualWriter.write(0, actual, precision, scale);
311+
assertEquals(expected, expectedWriter.getRow().getDecimal(0, precision, scale));
312+
assertEquals(actual, actualWriter.getRow().getDecimal(0, precision, scale));
313+
assertEquals(expectedWriter.getRow(), actualWriter.getRow());
314+
}
315+
}

0 commit comments

Comments
 (0)