Skip to content

Commit 62461b0

Browse files
authored
ARROW-17499: [Go] Shift MakeArrayOfNull to array Package (#13944)
Authored-by: Matt Topol <[email protected]> Signed-off-by: Matt Topol <[email protected]>
1 parent 510eb6f commit 62461b0

3 files changed

Lines changed: 159 additions & 4 deletions

File tree

‎go/arrow/array/util.go‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,3 +299,156 @@ func DictArrayFromJSON(mem memory.Allocator, dt *arrow.DictionaryType, indicesJS
299299

300300
return NewDictionaryArray(dt, indices, dict), nil
301301
}
302+
303+
func getMaxBufferLen(dt arrow.DataType, length int) int {
304+
bufferLen := int(bitutil.BytesForBits(int64(length)))
305+
306+
maxOf := func(bl int) int {
307+
if bl > bufferLen {
308+
return bl
309+
}
310+
return bufferLen
311+
}
312+
313+
switch dt := dt.(type) {
314+
case *arrow.DictionaryType:
315+
bufferLen = maxOf(getMaxBufferLen(dt.ValueType, length))
316+
return maxOf(getMaxBufferLen(dt.IndexType, length))
317+
case *arrow.FixedSizeBinaryType:
318+
return maxOf(dt.ByteWidth * length)
319+
case arrow.FixedWidthDataType:
320+
return maxOf(int(bitutil.BytesForBits(int64(dt.BitWidth()))) * length)
321+
case *arrow.StructType:
322+
for _, f := range dt.Fields() {
323+
bufferLen = maxOf(getMaxBufferLen(f.Type, length))
324+
}
325+
return bufferLen
326+
case *arrow.SparseUnionType:
327+
// type codes
328+
bufferLen = maxOf(length)
329+
// creates children of the same length of the union
330+
for _, f := range dt.Fields() {
331+
bufferLen = maxOf(getMaxBufferLen(f.Type, length))
332+
}
333+
return bufferLen
334+
case *arrow.DenseUnionType:
335+
// type codes
336+
bufferLen = maxOf(length)
337+
// offsets
338+
bufferLen = maxOf(arrow.Int32SizeBytes * length)
339+
// create children of length 1
340+
for _, f := range dt.Fields() {
341+
bufferLen = maxOf(getMaxBufferLen(f.Type, 1))
342+
}
343+
return bufferLen
344+
case arrow.OffsetsDataType:
345+
return maxOf(dt.OffsetTypeTraits().BytesRequired(length + 1))
346+
case *arrow.FixedSizeListType:
347+
return maxOf(getMaxBufferLen(dt.Elem(), int(dt.Len())*length))
348+
case arrow.ExtensionType:
349+
return maxOf(getMaxBufferLen(dt.StorageType(), length))
350+
default:
351+
panic(fmt.Errorf("arrow/array: arrayofnull not implemented for type %s", dt))
352+
}
353+
}
354+
355+
type nullArrayFactory struct {
356+
mem memory.Allocator
357+
dt arrow.DataType
358+
len int
359+
buf *memory.Buffer
360+
}
361+
362+
func (n *nullArrayFactory) create() *Data {
363+
if n.buf == nil {
364+
bufLen := getMaxBufferLen(n.dt, n.len)
365+
n.buf = memory.NewResizableBuffer(n.mem)
366+
n.buf.Resize(bufLen)
367+
defer n.buf.Release()
368+
}
369+
370+
var (
371+
dt = n.dt
372+
bufs = []*memory.Buffer{memory.SliceBuffer(n.buf, 0, int(bitutil.BytesForBits(int64(n.len))))}
373+
childData []arrow.ArrayData
374+
dictData arrow.ArrayData
375+
)
376+
defer bufs[0].Release()
377+
378+
if ex, ok := dt.(arrow.ExtensionType); ok {
379+
dt = ex.StorageType()
380+
}
381+
382+
if nf, ok := dt.(arrow.NestedType); ok {
383+
childData = make([]arrow.ArrayData, len(nf.Fields()))
384+
}
385+
386+
switch dt := dt.(type) {
387+
case *arrow.NullType:
388+
case *arrow.DictionaryType:
389+
bufs = append(bufs, n.buf)
390+
arr := MakeArrayOfNull(n.mem, dt.ValueType, 0)
391+
defer arr.Release()
392+
dictData = arr.Data()
393+
case arrow.FixedWidthDataType:
394+
bufs = append(bufs, n.buf)
395+
case arrow.BinaryDataType:
396+
bufs = append(bufs, n.buf, n.buf)
397+
case arrow.OffsetsDataType:
398+
bufs = append(bufs, n.buf)
399+
childData[0] = n.createChild(dt, 0, 0)
400+
defer childData[0].Release()
401+
case *arrow.FixedSizeListType:
402+
childData[0] = n.createChild(dt, 0, n.len*int(dt.Len()))
403+
defer childData[0].Release()
404+
case *arrow.StructType:
405+
for i := range dt.Fields() {
406+
childData[i] = n.createChild(dt, i, n.len)
407+
defer childData[i].Release()
408+
}
409+
case arrow.UnionType:
410+
bufs[0].Release()
411+
bufs[0] = nil
412+
bufs = append(bufs, n.buf)
413+
// buffer is zeroed, but 0 may not be a valid type code
414+
if dt.TypeCodes()[0] != 0 {
415+
bufs[1] = memory.NewResizableBuffer(n.mem)
416+
bufs[1].Resize(n.len)
417+
defer bufs[1].Release()
418+
memory.Set(bufs[1].Bytes(), byte(dt.TypeCodes()[0]))
419+
}
420+
421+
// for sparse unions we create children with the same length
422+
childLen := n.len
423+
if dt.Mode() == arrow.DenseMode {
424+
// for dense unions, offsets are all 0 and make children
425+
// with length 1
426+
bufs = append(bufs, n.buf)
427+
childLen = 1
428+
}
429+
for i := range dt.Fields() {
430+
childData[i] = n.createChild(dt, i, childLen)
431+
defer childData[i].Release()
432+
}
433+
}
434+
435+
out := NewData(n.dt, n.len, bufs, childData, n.len, 0)
436+
if dictData != nil {
437+
out.SetDictionary(dictData)
438+
}
439+
return out
440+
}
441+
442+
func (n *nullArrayFactory) createChild(dt arrow.DataType, i, length int) *Data {
443+
childFactory := &nullArrayFactory{
444+
mem: n.mem, dt: n.dt.(arrow.NestedType).Fields()[i].Type,
445+
len: length, buf: n.buf}
446+
return childFactory.create()
447+
}
448+
449+
// MakeArrayOfNull creates an array of size length which is all null of the given data type.
450+
func MakeArrayOfNull(mem memory.Allocator, dt arrow.DataType, length int) arrow.Array {
451+
data := (&nullArrayFactory{mem: mem, dt: dt, len: length}).create()
452+
defer data.Release()
453+
return MakeFromData(data)
454+
}

‎go/arrow/flight/flightsql/sqlite_server_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -156,9 +156,9 @@ func (s *FlightSqliteServerSuite) TestCommandGetTables() {
156156
s.NoError(err)
157157
defer rdr.Release()
158158

159-
catalogName := scalar.MakeArrayOfNull(arrow.BinaryTypes.String, 3, s.mem)
159+
catalogName := array.MakeArrayOfNull(s.mem, arrow.BinaryTypes.String, 3)
160160
defer catalogName.Release()
161-
schemaName := scalar.MakeArrayOfNull(arrow.BinaryTypes.String, 3, s.mem)
161+
schemaName := array.MakeArrayOfNull(s.mem, arrow.BinaryTypes.String, 3)
162162
defer schemaName.Release()
163163

164164
tableName := s.fromJSON(arrow.BinaryTypes.String, `["foreignTable", "intTable", "sqlite_sequence"]`)
@@ -243,9 +243,9 @@ func (s *FlightSqliteServerSuite) TestCommandGetTablesWithExistingTableTypeFilte
243243
s.NoError(err)
244244
defer rdr.Release()
245245

246-
catalogName := scalar.MakeArrayOfNull(arrow.BinaryTypes.String, 3, s.mem)
246+
catalogName := array.MakeArrayOfNull(s.mem, arrow.BinaryTypes.String, 3)
247247
defer catalogName.Release()
248-
schemaName := scalar.MakeArrayOfNull(arrow.BinaryTypes.String, 3, s.mem)
248+
schemaName := array.MakeArrayOfNull(s.mem, arrow.BinaryTypes.String, 3)
249249
defer schemaName.Release()
250250

251251
tableName := s.fromJSON(arrow.BinaryTypes.String, `["foreignTable", "intTable", "sqlite_sequence"]`)

‎go/arrow/scalar/scalar.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -705,6 +705,8 @@ func GetScalar(arr arrow.Array, idx int) (Scalar, error) {
705705
}
706706

707707
// MakeArrayOfNull creates an array of size length which is all null of the given data type.
708+
//
709+
// Deprecated: Use array.MakeArrayOfNull
708710
func MakeArrayOfNull(dt arrow.DataType, length int, mem memory.Allocator) arrow.Array {
709711
var (
710712
buffers = []*memory.Buffer{nil}

0 commit comments

Comments
 (0)