Skip to content

Commit b129fdc

Browse files
Merge remote-tracking branch 'upstream/main' into apachegh-41664-pretty-print-non-cpu
2 parents 555f8b6 + a42ec1d commit b129fdc

14 files changed

Lines changed: 280 additions & 37 deletions

File tree

‎.pre-commit-config.yaml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,13 @@ repos:
3434
hooks:
3535
- id: hadolint-docker
3636
name: Docker Format
37-
exclude: ^dev/.*$
37+
# We can enable this after we fix all existing lint failures.
38+
# files: (/Dockerfile|\.dockerfile)$
39+
files: >-
40+
(
41+
?^ci/docker/python-wheel-windows-test-vs2019\.dockerfile$|
42+
)
43+
types: []
3844
- repo: https://github.com/pycqa/flake8
3945
rev: 6.1.0
4046
hooks:

‎ci/docker/python-wheel-windows-test-vs2019.dockerfile‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
# contains choco and vs2019 preinstalled
2323
FROM abrarov/msvc-2019:2.11.0
2424

25+
# hadolint shell=cmd.exe
26+
2527
# Add unix tools to path
2628
RUN setx path "%path%;C:\Program Files\Git\usr\bin"
2729

@@ -40,8 +42,8 @@ RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\P
4042
(if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \
4143
(if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \
4244
(if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts")
43-
RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
44-
RUN python -m pip install -U pip setuptools
4545

4646
# Install archiver to extract xz archives
47-
RUN choco install --no-progress -r -y archiver
47+
RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% & \
48+
python -m pip install --no-cache-dir -U pip setuptools & \
49+
choco install --no-progress -r -y archiver

‎go/arrow/array/builder.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,7 @@ func NewBuilder(mem memory.Allocator, dtype arrow.DataType) Builder {
357357
return bldr
358358
case arrow.FIXED_SIZE_LIST:
359359
typ := dtype.(*arrow.FixedSizeListType)
360-
return NewFixedSizeListBuilder(mem, typ.Len(), typ.Elem())
360+
return NewFixedSizeListBuilderWithField(mem, typ.Len(), typ.ElemField())
361361
case arrow.DURATION:
362362
typ := dtype.(*arrow.DurationType)
363363
return NewDurationBuilder(mem, typ)

‎go/arrow/array/fixed_size_list.go‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -162,25 +162,38 @@ func (a *FixedSizeList) MarshalJSON() ([]byte, error) {
162162
}
163163

164164
type FixedSizeListBuilder struct {
165-
builder
166-
167-
etype arrow.DataType // data type of the list's elements.
168-
n int32 // number of elements in the fixed-size list.
169-
values Builder // value builder for the list's elements.
165+
baseListBuilder
166+
n int32 // number of elements in the fixed-size list.
170167
}
171168

172169
// NewFixedSizeListBuilder returns a builder, using the provided memory allocator.
173170
// The created list builder will create a list whose elements will be of type etype.
174171
func NewFixedSizeListBuilder(mem memory.Allocator, n int32, etype arrow.DataType) *FixedSizeListBuilder {
175172
return &FixedSizeListBuilder{
176-
builder: builder{refCount: 1, mem: mem},
177-
etype: etype,
178-
n: n,
179-
values: NewBuilder(mem, etype),
173+
baseListBuilder{
174+
builder: builder{refCount: 1, mem: mem},
175+
values: NewBuilder(mem, etype),
176+
dt: arrow.FixedSizeListOf(n, etype),
177+
},
178+
n,
179+
}
180+
}
181+
182+
// NewFixedSizeListBuilderWithField returns a builder similarly to
183+
// NewFixedSizeListBuilder, but it accepts a child rather than just a datatype
184+
// to ensure nullability context is preserved.
185+
func NewFixedSizeListBuilderWithField(mem memory.Allocator, n int32, field arrow.Field) *FixedSizeListBuilder {
186+
return &FixedSizeListBuilder{
187+
baseListBuilder{
188+
builder: builder{refCount: 1, mem: mem},
189+
values: NewBuilder(mem, field.Type),
190+
dt: arrow.FixedSizeListOfField(n, field),
191+
},
192+
n,
180193
}
181194
}
182195

183-
func (b *FixedSizeListBuilder) Type() arrow.DataType { return arrow.FixedSizeListOf(b.n, b.etype) }
196+
func (b *FixedSizeListBuilder) Type() arrow.DataType { return b.dt }
184197

185198
// Release decreases the reference count by 1.
186199
// When the reference count goes to zero, the memory is freed.
@@ -296,7 +309,7 @@ func (b *FixedSizeListBuilder) newData() (data *Data) {
296309
defer values.Release()
297310

298311
data = NewData(
299-
arrow.FixedSizeListOf(b.n, b.etype), b.length,
312+
b.dt, b.length,
300313
[]*memory.Buffer{b.nullBitmap},
301314
[]arrow.ArrayData{values.Data()},
302315
b.nulls,
@@ -336,7 +349,7 @@ func (b *FixedSizeListBuilder) UnmarshalOne(dec *json.Decoder) error {
336349
default:
337350
return &json.UnmarshalTypeError{
338351
Value: fmt.Sprint(t),
339-
Struct: arrow.FixedSizeListOf(b.n, b.etype).String(),
352+
Struct: b.dt.String(),
340353
}
341354
}
342355

‎go/arrow/array/record_test.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,45 @@ func TestRecordReader(t *testing.T) {
372372
}
373373
}
374374

375+
func TestRecordBuilderRespectsFixedSizeArrayNullability(t *testing.T) {
376+
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
377+
defer mem.AssertSize(t, 0)
378+
379+
cases := []struct {
380+
assertion string
381+
fields []arrow.Field
382+
}{
383+
{
384+
"nullable",
385+
[]arrow.Field{{Name: "data", Type: arrow.FixedSizeListOf(1, arrow.PrimitiveTypes.Int32)}},
386+
},
387+
{
388+
"not nullable",
389+
[]arrow.Field{{Name: "data", Type: arrow.FixedSizeListOfNonNullable(1, arrow.PrimitiveTypes.Int32)}},
390+
},
391+
}
392+
for _, c := range cases {
393+
t.Run(c.assertion, func(t *testing.T) {
394+
schema := arrow.NewSchema(c.fields, nil)
395+
b := array.NewRecordBuilder(mem, schema)
396+
defer b.Release()
397+
398+
lb := b.Field(0).(*array.FixedSizeListBuilder)
399+
lb.Append(true)
400+
401+
vb := lb.ValueBuilder().(*array.Int32Builder)
402+
vb.Append(10)
403+
404+
rec := b.NewRecord()
405+
defer rec.Release()
406+
407+
if got, want := rec.Column(0).String(), "[[10]]"; got != want {
408+
t.Fatalf("invalid record: got=%q, want=%q", got, want)
409+
}
410+
})
411+
}
412+
}
413+
375414
func TestRecordBuilder(t *testing.T) {
376415
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
377416
defer mem.AssertSize(t, 0)

‎go/arrow/ipc/file_writer.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,7 @@ type FileWriter struct {
278278
mapper dictutils.Mapper
279279
codec flatbuf.CompressionType
280280
compressNP int
281+
compressors []compressor
281282
minSpaceSavings *float64
282283

283284
// map of the last written dictionaries by id
@@ -302,6 +303,7 @@ func NewFileWriter(w io.WriteSeeker, opts ...Option) (*FileWriter, error) {
302303
codec: cfg.codec,
303304
compressNP: cfg.compressNP,
304305
minSpaceSavings: cfg.minSpaceSavings,
306+
compressors: make([]compressor, cfg.compressNP),
305307
}
306308

307309
pos, err := f.w.Seek(0, io.SeekCurrent)
@@ -345,7 +347,9 @@ func (f *FileWriter) Write(rec arrow.Record) error {
345347
const allow64b = true
346348
var (
347349
data = Payload{msg: MessageRecordBatch}
348-
enc = newRecordEncoder(f.mem, 0, kMaxNestingDepth, allow64b, f.codec, f.compressNP, f.minSpaceSavings)
350+
enc = newRecordEncoder(
351+
f.mem, 0, kMaxNestingDepth, allow64b, f.codec, f.compressNP, f.minSpaceSavings, f.compressors,
352+
)
349353
)
350354
defer data.Release()
351355

‎go/arrow/ipc/ipc.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ func newConfig(opts ...Option) *config {
7979
alloc: memory.NewGoAllocator(),
8080
codec: -1, // uncompressed
8181
ensureNativeEndian: true,
82+
compressNP: 1,
8283
}
8384

8485
for _, opt := range opts {
@@ -132,9 +133,12 @@ func WithZstd() Option {
132133
// WithCompressConcurrency specifies a number of goroutines to spin up for
133134
// concurrent compression of the body buffers when writing compress IPC records.
134135
// If n <= 1 then compression will be done serially without goroutine
135-
// parallelization. Default is 0.
136+
// parallelization. Default is 1.
136137
func WithCompressConcurrency(n int) Option {
137138
return func(cfg *config) {
139+
if n <= 0 {
140+
n = 1
141+
}
138142
cfg.compressNP = n
139143
}
140144
}

‎go/arrow/ipc/writer.go‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ type Writer struct {
8686
mapper dictutils.Mapper
8787
codec flatbuf.CompressionType
8888
compressNP int
89+
compressors []compressor
8990
minSpaceSavings *float64
9091

9192
// map of the last written dictionaries by id
@@ -107,6 +108,7 @@ func NewWriterWithPayloadWriter(pw PayloadWriter, opts ...Option) *Writer {
107108
compressNP: cfg.compressNP,
108109
minSpaceSavings: cfg.minSpaceSavings,
109110
emitDictDeltas: cfg.emitDictDeltas,
111+
compressors: make([]compressor, cfg.compressNP),
110112
}
111113
}
112114

@@ -120,6 +122,8 @@ func NewWriter(w io.Writer, opts ...Option) *Writer {
120122
schema: cfg.schema,
121123
codec: cfg.codec,
122124
emitDictDeltas: cfg.emitDictDeltas,
125+
compressNP: cfg.compressNP,
126+
compressors: make([]compressor, cfg.compressNP),
123127
}
124128
}
125129

@@ -170,7 +174,16 @@ func (w *Writer) Write(rec arrow.Record) (err error) {
170174
const allow64b = true
171175
var (
172176
data = Payload{msg: MessageRecordBatch}
173-
enc = newRecordEncoder(w.mem, 0, kMaxNestingDepth, allow64b, w.codec, w.compressNP, w.minSpaceSavings)
177+
enc = newRecordEncoder(
178+
w.mem,
179+
0,
180+
kMaxNestingDepth,
181+
allow64b,
182+
w.codec,
183+
w.compressNP,
184+
w.minSpaceSavings,
185+
w.compressors,
186+
)
174187
)
175188
defer data.Release()
176189

@@ -310,17 +323,28 @@ type recordEncoder struct {
310323
allow64b bool
311324
codec flatbuf.CompressionType
312325
compressNP int
326+
compressors []compressor
313327
minSpaceSavings *float64
314328
}
315329

316-
func newRecordEncoder(mem memory.Allocator, startOffset, maxDepth int64, allow64b bool, codec flatbuf.CompressionType, compressNP int, minSpaceSavings *float64) *recordEncoder {
330+
func newRecordEncoder(
331+
mem memory.Allocator,
332+
startOffset,
333+
maxDepth int64,
334+
allow64b bool,
335+
codec flatbuf.CompressionType,
336+
compressNP int,
337+
minSpaceSavings *float64,
338+
compressors []compressor,
339+
) *recordEncoder {
317340
return &recordEncoder{
318341
mem: mem,
319342
start: startOffset,
320343
depth: maxDepth,
321344
allow64b: allow64b,
322345
codec: codec,
323346
compressNP: compressNP,
347+
compressors: compressors,
324348
minSpaceSavings: minSpaceSavings,
325349
}
326350
}
@@ -340,6 +364,13 @@ func (w *recordEncoder) reset() {
340364
w.fields = make([]fieldMetadata, 0)
341365
}
342366

367+
func (w *recordEncoder) getCompressor(id int) compressor {
368+
if w.compressors[id] == nil {
369+
w.compressors[id] = getCompressor(w.codec)
370+
}
371+
return w.compressors[id]
372+
}
373+
343374
func (w *recordEncoder) compressBodyBuffers(p *Payload) error {
344375
compress := func(idx int, codec compressor) error {
345376
if p.body[idx] == nil || p.body[idx].Len() == 0 {
@@ -378,7 +409,7 @@ func (w *recordEncoder) compressBodyBuffers(p *Payload) error {
378409
}
379410

380411
if w.compressNP <= 1 {
381-
codec := getCompressor(w.codec)
412+
codec := w.getCompressor(0)
382413
for idx := range p.body {
383414
if err := compress(idx, codec); err != nil {
384415
return err
@@ -395,11 +426,11 @@ func (w *recordEncoder) compressBodyBuffers(p *Payload) error {
395426
)
396427
defer cancel()
397428

398-
for i := 0; i < w.compressNP; i++ {
429+
for workerID := 0; workerID < w.compressNP; workerID++ {
399430
wg.Add(1)
400-
go func() {
431+
go func(id int) {
401432
defer wg.Done()
402-
codec := getCompressor(w.codec)
433+
codec := w.getCompressor(id)
403434
for {
404435
select {
405436
case idx, ok := <-ch:
@@ -418,7 +449,7 @@ func (w *recordEncoder) compressBodyBuffers(p *Payload) error {
418449
return
419450
}
420451
}
421-
}()
452+
}(workerID)
422453
}
423454

424455
for idx := range p.body {

‎go/arrow/ipc/writer_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,8 @@ func TestWriteWithCompressionAndMinSavings(t *testing.T) {
193193
}
194194

195195
for _, codec := range []flatbuf.CompressionType{flatbuf.CompressionTypeLZ4_FRAME, flatbuf.CompressionTypeZSTD} {
196-
enc := newRecordEncoder(mem, 0, 5, true, codec, 1, nil)
196+
compressors := []compressor{getCompressor(codec)}
197+
enc := newRecordEncoder(mem, 0, 5, true, codec, 1, nil, compressors)
197198
var payload Payload
198199
require.NoError(t, enc.encode(&payload, batch))
199200
assert.Len(t, payload.body, 2)
@@ -205,7 +206,7 @@ func TestWriteWithCompressionAndMinSavings(t *testing.T) {
205206
assert.Greater(t, compressedSize, int64(0))
206207
expectedSavings := 1.0 - float64(compressedSize)/float64(uncompressedSize)
207208

208-
compressEncoder := newRecordEncoder(mem, 0, 5, true, codec, 1, &expectedSavings)
209+
compressEncoder := newRecordEncoder(mem, 0, 5, true, codec, 1, &expectedSavings, compressors)
209210
payload.Release()
210211
payload.body = payload.body[:0]
211212
require.NoError(t, compressEncoder.encode(&payload, batch))

‎java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -954,7 +954,7 @@ public void setBytes(long index, ArrowBuf src, long srcIndex, long length) {
954954
if (isOutOfBounds(srcIndex, length, src.capacity())) {
955955
throw new IndexOutOfBoundsException(
956956
String.format(
957-
"index: %d, length: %d (expected: range(0, %d))", index, length, src.capacity()));
957+
"index: %d, length: %d (expected: range(0, %d))", srcIndex, length, src.capacity()));
958958
}
959959
if (length != 0) {
960960
// copy length bytes of data from src ArrowBuf starting at

0 commit comments

Comments
 (0)