Skip to content

Commit 1839d48

Browse files
jerrinotjerrinot
authored
feat(ilp): add reshape() and clear() methods for array reusability (#5996)
1 parent b150294 commit 1839d48

5 files changed

Lines changed: 886 additions & 62 deletions

File tree

‎core/src/main/java/io/questdb/cutlass/line/array/AbstractArray.java‎

Lines changed: 198 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -25,43 +25,63 @@
2525
package io.questdb.cutlass.line.array;
2626

2727
import io.questdb.cairo.ColumnType;
28+
import io.questdb.cairo.arr.ArrayView;
2829
import io.questdb.cairo.arr.BorrowedFlatArrayView;
2930
import io.questdb.cairo.arr.DirectArray;
3031
import io.questdb.cairo.vm.api.MemoryA;
32+
import io.questdb.client.Sender;
3133
import io.questdb.cutlass.line.LineSenderException;
3234
import io.questdb.std.QuietCloseable;
3335

3436
/**
35-
* `AbstractArray` provides an interface for Java client users to create multi-dimensional arrays,
36-
* supporting up to 32 dimensions.
37-
* <p>It manages a contiguous block of memory to store the actual array data.
38-
* To prevent memory leaks, please ensure to invoke the {@link #close()} method after usage.
39-
* <p>Example of usage:
40-
* <pre>{@code
41-
* // Creates a 2x3x2 matrix (of rank 3)
42-
* try (
43-
* DoubleArray matrix3d = DoubleArray.create(2, 3, 2)) {
44-
* matrix3d.set(DoubleArray.create(new double[]{1.0, 2.0}), true, 0, 0)
45-
* .set(DoubleArray.create(new double[]{3.0, 4.0}), true, 0, 1)
46-
* .set(DoubleArray.create(new double[]{5.0, 6.0}), true, 0, 2)
47-
* .set(DoubleArray.create(new double[]{7.0, 8.0}), true, 1, 0)
48-
* .set(DoubleArray.create(new double[]{9.0, 10.0}), true, 1, 1)
49-
* .set(DoubleArray.create(new double[]{11.0, 12.0}), true, 1, 2);
37+
* Use this class to prepare data for an N-dimensional array column in QuestDB.
38+
* It manages a contiguous block of native memory to store the array data.
39+
* To avoid leaking this memory, make sure you use it in a try-with-resources block,
40+
* or call {@link #close()} explicitly.
41+
* <p>
42+
* Example of usage:
43+
* <pre>
44+
* // Create a 2x3 array:
45+
* try (DoubleArray matrix = new DoubleArray(2, 3)) {
5046
*
51-
* // send matrix3d to line
52-
* sender.table(tableName).doubleArray(columnName, matrix3d);
53-
* }
47+
* // Append data in row-major order:
48+
* matrix.append(1.0).append(2.0).append(3.0) // first row
49+
* .append(4.0).append(5.0).append(6.0); // second row
5450
*
51+
* // Or, set a value at specific coordinates:
52+
* matrix.set(1.5, 0, 1); // set element at row 0, column 1
53+
*
54+
* // Send to QuestDB
55+
* sender.table("my_table").doubleArray("matrix_column", matrix).atNow();
56+
* }
5557
* }</pre>
5658
*/
5759
public abstract class AbstractArray implements QuietCloseable {
5860

5961
protected final DirectArray array = new DirectArray();
60-
protected final int flatLength;
6162
protected boolean closed = false;
63+
protected int flatLength;
6264
protected MemoryA memA = array.startMemoryA();
6365

6466
protected AbstractArray(int[] shape, short columnType) {
67+
if (shape.length == 0) {
68+
throw new LineSenderException("Shape must have at least one dimension");
69+
}
70+
if (shape.length > ColumnType.ARRAY_NDIMS_LIMIT) {
71+
throw new LineSenderException("Maximum supported dimensionality is " +
72+
ColumnType.ARRAY_NDIMS_LIMIT + "D, but got " + shape.length + "D");
73+
}
74+
for (int dim = 0; dim < shape.length; dim++) {
75+
if (shape[dim] < 0) {
76+
throw new LineSenderException("dimension length must not be negative [dim=" + dim +
77+
", dimLen=" + shape[dim] + "]");
78+
}
79+
if (shape[dim] > ArrayView.DIM_MAX_LEN) {
80+
throw new LineSenderException("dimension length out of range [dim=" + dim +
81+
", dimLen=" + shape[dim] + ", maxLen=" + ArrayView.DIM_MAX_LEN + "]");
82+
}
83+
}
84+
6585
array.setType(ColumnType.encodeArrayType(columnType, shape.length));
6686
for (int dim = 0, size = shape.length; dim < size; dim++) {
6787
array.setDimLen(dim, shape[dim]);
@@ -86,6 +106,42 @@ public void appendToBufPtr(ArrayBufferAppender mem) {
86106
}
87107
}
88108

109+
/**
110+
* Resets the append position to the beginning of the array without modifying any data.
111+
* <p>
112+
* This method only resets the append position marker, and the array data remains unchanged.
113+
* Subsequent {@code append()} calls will start overwriting from the first element.
114+
* <p>
115+
* <strong>Use cases:</strong>
116+
* <ul>
117+
* <li>Reset the array after getting an error and/or calling {@link Sender#cancelRow()}</li>
118+
* <li>Start fresh without relying on auto-wrapping behavior</li>
119+
* </ul>
120+
* <strong>Note:</strong> to change the array dimensions, use {@code reshape()}.
121+
* This method only resets the position while maintaining the array shape.
122+
*
123+
* @see #reshape(int...)
124+
*/
125+
public void clear() {
126+
assert !closed;
127+
memA = array.startMemoryA();
128+
}
129+
130+
/**
131+
* Closes this array and releases all associated native memory resources.
132+
* <p>
133+
* <strong>Important:</strong> after calling this method, the array becomes unusable.
134+
* Any subsequent operations (append, set, reshape, etc.) will result in undefined
135+
* behavior or exceptions.
136+
* <p>
137+
* This method is idempotent &mdash; calling it multiple times has no additional effect.
138+
* <p>
139+
* <strong>Memory Management:</strong> since the class uses native memory, failing to call
140+
* this method will result in a native memory leak. Use it inside try-with-resources, or
141+
* call explicitly in a finally block.
142+
*
143+
* @see java.lang.AutoCloseable#close()
144+
*/
89145
@Override
90146
public void close() {
91147
if (!closed) {
@@ -94,6 +150,129 @@ public void close() {
94150
closed = true;
95151
}
96152

153+
/**
154+
* Reshapes the array to the specified dimensions, and resets the append
155+
* position to the start of the array.
156+
*
157+
* @param shape the new dimensions for the array
158+
* @throws LineSenderException if the array is already closed or shape has invalid dimensions
159+
* @see #reshape(int)
160+
* @see #reshape(int, int)
161+
* @see #reshape(int, int, int)
162+
*/
163+
public void reshape(int... shape) {
164+
if (closed) {
165+
throw new LineSenderException("Cannot reshape a closed array");
166+
}
167+
int nDim = shape.length;
168+
if (nDim > ColumnType.ARRAY_NDIMS_LIMIT) {
169+
throw new LineSenderException("Maximum supported dimensionality is " +
170+
ColumnType.ARRAY_NDIMS_LIMIT + "D, but got " + nDim + "D");
171+
}
172+
if (nDim == 0) {
173+
throw new LineSenderException("Shape must have at least one dimension");
174+
}
175+
for (int dim = 0; dim < nDim; dim++) {
176+
if (shape[dim] < 0) {
177+
throw new LineSenderException("dimension length must not be negative [dim=" + dim +
178+
", dimLen=" + shape[dim] + "]");
179+
}
180+
if (shape[dim] > ArrayView.DIM_MAX_LEN) {
181+
throw new LineSenderException("dimension length out of range [dim=" + dim +
182+
", dimLen=" + shape[dim] + ", maxLen=" + ArrayView.DIM_MAX_LEN + "]");
183+
}
184+
}
185+
array.setType(ColumnType.encodeArrayType(array.getElemType(), nDim));
186+
for (int dim = 0; dim < nDim; dim++) {
187+
array.setDimLen(dim, shape[dim]);
188+
}
189+
array.applyShape();
190+
flatLength = array.getFlatViewLength();
191+
memA = array.startMemoryA();
192+
}
193+
194+
/**
195+
* Reshapes the array to a single dimension with the specified length, and resets
196+
* the append position to the start of the array.
197+
*
198+
* @param dimLen the length of the single dimension
199+
* @throws LineSenderException if the array is already closed or dimLen is negative
200+
*/
201+
public void reshape(int dimLen) {
202+
if (closed) {
203+
throw new LineSenderException("Cannot reshape a closed array");
204+
}
205+
if (dimLen < 0) {
206+
throw new LineSenderException("Array size must not be negative, but got " + dimLen);
207+
}
208+
if (dimLen > ArrayView.DIM_MAX_LEN) {
209+
throw new LineSenderException("Array size out of range [dimLen=" + dimLen +
210+
", maxLen=" + ArrayView.DIM_MAX_LEN + "]");
211+
}
212+
array.setType(ColumnType.encodeArrayType(array.getElemType(), 1));
213+
array.setDimLen(0, dimLen);
214+
array.applyShape();
215+
flatLength = array.getFlatViewLength();
216+
memA = array.startMemoryA();
217+
}
218+
219+
/**
220+
* Reshapes the array to two dimensions with the specified lengths, and resets
221+
* the append position to the start of the array.
222+
*
223+
* @param dim1 the length of the first dimension (rows)
224+
* @param dim2 the length of the second dimension (columns)
225+
* @throws LineSenderException if the array is already closed or any dimension is negative
226+
*/
227+
public void reshape(int dim1, int dim2) {
228+
if (closed) {
229+
throw new LineSenderException("Cannot reshape a closed array");
230+
}
231+
if (dim1 < 0 || dim2 < 0) {
232+
throw new LineSenderException("Array dimensions must not be negative, but got [" + dim1 + ", " + dim2 + "]");
233+
}
234+
if (dim1 > ArrayView.DIM_MAX_LEN || dim2 > ArrayView.DIM_MAX_LEN) {
235+
throw new LineSenderException("Array dimensions out of range [dim1=" + dim1 +
236+
", dim2=" + dim2 + ", maxLen=" + ArrayView.DIM_MAX_LEN + "]");
237+
}
238+
array.setType(ColumnType.encodeArrayType(array.getElemType(), 2));
239+
array.setDimLen(0, dim1);
240+
array.setDimLen(1, dim2);
241+
array.applyShape();
242+
flatLength = array.getFlatViewLength();
243+
memA = array.startMemoryA();
244+
}
245+
246+
/**
247+
* Reshapes the array to three dimensions with the specified lengths, and resets
248+
* the append position to the start of the array.
249+
*
250+
* @param dim1 the length of the first dimension
251+
* @param dim2 the length of the second dimension
252+
* @param dim3 the length of the third dimension
253+
* @throws LineSenderException if the array is already closed or any dimension is negative
254+
*/
255+
public void reshape(int dim1, int dim2, int dim3) {
256+
if (closed) {
257+
throw new LineSenderException("Cannot reshape a closed array");
258+
}
259+
if (dim1 < 0 || dim2 < 0 || dim3 < 0) {
260+
throw new LineSenderException("Array dimensions must not be negative, but got [" +
261+
dim1 + ", " + dim2 + ", " + dim3 + "]");
262+
}
263+
if (dim1 > ArrayView.DIM_MAX_LEN || dim2 > ArrayView.DIM_MAX_LEN || dim3 > ArrayView.DIM_MAX_LEN) {
264+
throw new LineSenderException("Array dimensions out of range [dim1=" + dim1 +
265+
", dim2=" + dim2 + ", dim3=" + dim3 + ", maxLen=" + ArrayView.DIM_MAX_LEN + "]");
266+
}
267+
array.setType(ColumnType.encodeArrayType(array.getElemType(), 3));
268+
array.setDimLen(0, dim1);
269+
array.setDimLen(1, dim2);
270+
array.setDimLen(2, dim3);
271+
array.applyShape();
272+
flatLength = array.getFlatViewLength();
273+
memA = array.startMemoryA();
274+
}
275+
97276
protected void ensureLegalAppendPosition() {
98277
long elementSize = ColumnType.sizeOf(array.getElemType());
99278
if (memA.getAppendOffset() == flatLength * elementSize) {

‎core/src/main/java/io/questdb/cutlass/line/array/DoubleArray.java‎

Lines changed: 58 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -27,25 +27,46 @@
2727
import io.questdb.cairo.ColumnType;
2828
import io.questdb.std.Unsafe;
2929

30-
/**
31-
* Used to accumulate the data of an N-dimensional array of {@code double} values.
32-
* You must close the array when done with it because it uses native memory.
33-
*/
3430
public class DoubleArray extends AbstractArray {
3531

32+
/**
33+
* Creates a new DoubleArray with the specified shape and the append position
34+
* pointing to the first element.
35+
* <p>
36+
* The shape defines the dimensions of the N-dimensional array. For example:
37+
* <ul>
38+
* <li>{@code new DoubleArray(10)} creates a 1D array with 10 elements</li>
39+
* <li>{@code new DoubleArray(3, 4)} creates a 2D array (3x4 matrix)</li>
40+
* <li>{@code new DoubleArray(2, 3, 4)} creates a 3D array (2x3x4)</li>
41+
* </ul>
42+
* <p>
43+
* You can change the array's shape at any time using {@link #reshape}.
44+
*
45+
* @param shape the dimensions of the array (must have at least one dimension)
46+
* @throws io.questdb.cutlass.line.LineSenderException if shape is empty or contains negative values
47+
* @see #reshape(int...)
48+
*/
3649
public DoubleArray(int... shape) {
3750
super(shape, ColumnType.DOUBLE);
3851
}
3952

4053
/**
41-
* Appends the value at the current append positions, and then advances it.
42-
* The append position advances in the row-major order across the entire array.
43-
* If the append position is currently beyond the last element, it first resets
44-
* the position to zero and then appends the new value.
54+
* Appends the value at the current append position, and then advances it.
55+
* The append position advances in row-major order across the entire array.
56+
* If it is currently at the last element, it will automatically wrap around to the first
57+
* element after this call.
4558
* <p>
46-
* The intention is for this array object to be reused for all the rows you are
47-
* inserting, and this auto-wrapping behavior allows you to repeatedly fill the
48-
* array without the need for other lifecycle calls like {@code clear()}.
59+
* <strong>Auto-wrapping behavior:</strong> this array is designed to be reused across
60+
* multiple rows. As soon as you are done filling it up, append position wraps back to
61+
* the first element. After you have sent it to QuestDB, just start appending more data
62+
* for the next row.
63+
* <p>
64+
* <strong>Error recovery:</strong> If you need to abandon a partially-filled array
65+
* (e.g., after {@code sender.cancelRow()}), use {@code clear()} to reset the append
66+
* position.
67+
*
68+
* @param value the double value to append
69+
* @return this array instance for method chaining
4970
*/
5071
public DoubleArray append(double value) {
5172
ensureLegalAppendPosition();
@@ -54,7 +75,15 @@ public DoubleArray append(double value) {
5475
}
5576

5677
/**
57-
* Sets a value at the supplied coordinates.
78+
* Sets a value at the specified coordinates without affecting the append position.
79+
* <p>
80+
* This method allows direct access to an array element by its coordinates. Unlike {@code
81+
* append()}, this does not modify the current append position.
82+
*
83+
* @param value the double value to set
84+
* @param coords the coordinates specifying the position (must match array dimensionality)
85+
* @return this array instance for method chaining
86+
* @throws io.questdb.cutlass.line.LineSenderException if coordinates don't match the array shape
5887
*/
5988
public DoubleArray set(double value, int... coords) {
6089
assert !closed;
@@ -63,7 +92,23 @@ public DoubleArray set(double value, int... coords) {
6392
}
6493

6594
/**
66-
* Sets all data points in the array to the supplied value.
95+
* Sets all data points in the array to the supplied value, without changing
96+
* the append position.
97+
* <p>
98+
* <strong>Append position behavior:</strong> this method does NOT change the current
99+
* append position. If you were in the middle of appending data, subsequent {@code
100+
* append()} calls will continue from where they left off, potentially overwriting the
101+
* values set by this method.
102+
* <p>
103+
* <strong>Use cases:</strong>
104+
* <ul>
105+
* <li>Initialize the array with default values before using {@code set()} for selective
106+
* updates</li>
107+
* <li>Set the whole array to a uniform value</li>
108+
* </ul>
109+
*
110+
* @param value the double value to set for all array elements
111+
* @return this array instance for method chaining
67112
*/
68113
public DoubleArray setAll(double value) {
69114
long ptr = array.ptr();

0 commit comments

Comments
 (0)