Skip to content

Commit db6c099

Browse files
authored
ARROW-17275: [Go][Integration] Handle Large offset types in IPC read/write (#13770)
Authored-by: Matt Topol <[email protected]> Signed-off-by: Matt Topol <[email protected]>
1 parent 3b987d9 commit db6c099

33 files changed

Lines changed: 1297 additions & 305 deletions

‎ci/scripts/go_test.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,16 +21,17 @@ set -ex
2121

2222
source_dir=${1}/go
2323

24+
# when we upgrade to at least go1.18, we can add the new -asan option here
2425
testargs="-race"
2526
case "$(uname)" in
2627
MINGW*)
27-
# -race doesn't work on windows currently
28+
# -asan and -race don't work on windows currently
2829
testargs=""
2930
;;
3031
esac
3132

3233
if [[ "$(go env GOHOSTARCH)" = "s390x" ]]; then
33-
testargs="" # -race not supported on s390x
34+
testargs="" # -race and -asan not supported on s390x
3435
fi
3536

3637
# Go static check (skipped in MinGW)

‎dev/archery/archery/integration/datagen.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1587,7 +1587,6 @@ def _temp_path():
15871587

15881588
generate_primitive_large_offsets_case([17, 20])
15891589
.skip_category('C#')
1590-
.skip_category('Go')
15911590
.skip_category('JS'),
15921591

15931592
generate_null_case([10, 0])
@@ -1634,7 +1633,6 @@ def _temp_path():
16341633

16351634
generate_nested_large_offsets_case()
16361635
.skip_category('C#')
1637-
.skip_category('Go')
16381636
.skip_category('JS'),
16391637

16401638
generate_unions_case()

‎docs/source/status.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ Data Types
7777
+-------------------+-------+-------+-------+------------+-------+-------+-------+
7878
| List | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
7979
+-------------------+-------+-------+-------+------------+-------+-------+-------+
80-
| Large List | ✓ | ✓ | | | | ✓ | ✓ |
80+
| Large List | ✓ | ✓ | ✓ | | | ✓ | ✓ |
8181
+-------------------+-------+-------+-------+------------+-------+-------+-------+
8282
| Struct | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
8383
+-------------------+-------+-------+-------+------------+-------+-------+-------+

‎go/arrow/array/array.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ func init() {
178178
arrow.DURATION: func(data arrow.ArrayData) arrow.Array { return NewDurationData(data) },
179179
arrow.LARGE_STRING: func(data arrow.ArrayData) arrow.Array { return NewLargeStringData(data) },
180180
arrow.LARGE_BINARY: func(data arrow.ArrayData) arrow.Array { return NewLargeBinaryData(data) },
181-
arrow.LARGE_LIST: unsupportedArrayType,
181+
arrow.LARGE_LIST: func(data arrow.ArrayData) arrow.Array { return NewLargeListData(data) },
182182
arrow.INTERVAL: func(data arrow.ArrayData) arrow.Array { return NewIntervalData(data) },
183183
arrow.INTERVAL_MONTH_DAY_NANO: func(data arrow.ArrayData) arrow.Array { return NewMonthDayNanoIntervalData(data) },
184184

‎go/arrow/array/array_test.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@ func TestMakeFromData(t *testing.T) {
8181
array.NewData(&testDataType{arrow.INT64}, 0 /* length */, make([]*memory.Buffer, 2 /*null bitmap, values*/), nil /* childData */, 0 /* nulls */, 0 /* offset */),
8282
}},
8383

84+
{name: "large list", d: &testDataType{arrow.LARGE_LIST}, child: []arrow.ArrayData{
85+
array.NewData(&testDataType{arrow.INT64}, 0 /* length */, make([]*memory.Buffer, 2 /*null bitmap, values*/), nil /* childData */, 0 /* nulls */, 0 /* offset */),
86+
array.NewData(&testDataType{arrow.INT64}, 0 /* length */, make([]*memory.Buffer, 2 /*null bitmap, values*/), nil /* childData */, 0 /* nulls */, 0 /* offset */),
87+
}},
88+
8489
{name: "struct", d: &testDataType{arrow.STRUCT}},
8590
{name: "struct", d: &testDataType{arrow.STRUCT}, child: []arrow.ArrayData{
8691
array.NewData(&testDataType{arrow.INT64}, 0 /* length */, make([]*memory.Buffer, 2 /*null bitmap, values*/), nil /* childData */, 0 /* nulls */, 0 /* offset */),
@@ -117,7 +122,6 @@ func TestMakeFromData(t *testing.T) {
117122
// unsupported types
118123
{name: "sparse union", d: &testDataType{arrow.SPARSE_UNION}, expPanic: true, expError: "unsupported data type: SPARSE_UNION"},
119124
{name: "dense union", d: &testDataType{arrow.DENSE_UNION}, expPanic: true, expError: "unsupported data type: DENSE_UNION"},
120-
{name: "large list", d: &testDataType{arrow.LARGE_LIST}, expPanic: true, expError: "unsupported data type: LARGE_LIST"},
121125
{name: "decimal256", d: &testDataType{arrow.DECIMAL256}, expPanic: true, expError: "unsupported data type: DECIMAL256"},
122126

123127
// invalid types

‎go/arrow/array/binary.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,12 @@ import (
2626
"github.com/goccy/go-json"
2727
)
2828

29+
type BinaryLike interface {
30+
arrow.Array
31+
ValueBytes() []byte
32+
ValueOffset64(int) int64
33+
}
34+
2935
// A type which represents an immutable sequence of variable-length binary strings.
3036
type Binary struct {
3137
array
@@ -64,6 +70,10 @@ func (a *Binary) ValueOffset(i int) int {
6470
return int(a.valueOffsets[a.array.data.offset+i])
6571
}
6672

73+
func (a *Binary) ValueOffset64(i int) int64 {
74+
return int64(a.ValueOffset(i))
75+
}
76+
6777
func (a *Binary) ValueLen(i int) int {
6878
if i < 0 || i >= a.array.data.length {
6979
panic("arrow/array: index out of range")
@@ -193,6 +203,10 @@ func (a *LargeBinary) ValueOffset(i int) int64 {
193203
return a.valueOffsets[a.array.data.offset+i]
194204
}
195205

206+
func (a *LargeBinary) ValueOffset64(i int) int64 {
207+
return a.ValueOffset(i)
208+
}
209+
196210
func (a *LargeBinary) ValueLen(i int) int {
197211
if i < 0 || i >= a.array.data.length {
198212
panic("arrow/array: index out of range")

‎go/arrow/array/builder.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,8 @@ func NewBuilder(mem memory.Allocator, dtype arrow.DataType) Builder {
303303
typ := dtype.(*arrow.DictionaryType)
304304
return NewDictionaryBuilder(mem, typ)
305305
case arrow.LARGE_LIST:
306+
typ := dtype.(*arrow.LargeListType)
307+
return NewLargeListBuilder(mem, typ.Elem())
306308
case arrow.MAP:
307309
typ := dtype.(*arrow.MapType)
308310
return NewMapBuilder(mem, typ.KeyType(), typ.ItemType(), typ.KeysSorted)

‎go/arrow/array/compare.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,9 @@ func Equal(left, right arrow.Array) bool {
294294
case *List:
295295
r := right.(*List)
296296
return arrayEqualList(l, r)
297+
case *LargeList:
298+
r := right.(*LargeList)
299+
return arrayEqualLargeList(l, r)
297300
case *FixedSizeList:
298301
r := right.(*FixedSizeList)
299302
return arrayEqualFixedSizeList(l, r)
@@ -535,6 +538,9 @@ func arrayApproxEqual(left, right arrow.Array, opt equalOption) bool {
535538
case *List:
536539
r := right.(*List)
537540
return arrayApproxEqualList(l, r, opt)
541+
case *LargeList:
542+
r := right.(*LargeList)
543+
return arrayApproxEqualLargeList(l, r, opt)
538544
case *FixedSizeList:
539545
r := right.(*FixedSizeList)
540546
return arrayApproxEqualFixedSizeList(l, r, opt)
@@ -650,6 +656,25 @@ func arrayApproxEqualList(left, right *List, opt equalOption) bool {
650656
return true
651657
}
652658

659+
func arrayApproxEqualLargeList(left, right *LargeList, opt equalOption) bool {
660+
for i := 0; i < left.Len(); i++ {
661+
if left.IsNull(i) {
662+
continue
663+
}
664+
o := func() bool {
665+
l := left.newListValue(i)
666+
defer l.Release()
667+
r := right.newListValue(i)
668+
defer r.Release()
669+
return arrayApproxEqual(l, r, opt)
670+
}()
671+
if !o {
672+
return false
673+
}
674+
}
675+
return true
676+
}
677+
653678
func arrayApproxEqualFixedSizeList(left, right *FixedSizeList, opt equalOption) bool {
654679
for i := 0; i < left.Len(); i++ {
655680
if left.IsNull(i) {

‎go/arrow/array/concat.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,23 @@ func concat(data []arrow.ArrayData, mem memory.Allocator) (arrow.ArrayData, erro
447447
defer c.Release()
448448
}
449449

450+
out.buffers[1] = offsetBuffer
451+
out.childData = make([]arrow.ArrayData, 1)
452+
out.childData[0], err = concat(childData, mem)
453+
if err != nil {
454+
return nil, err
455+
}
456+
case *arrow.LargeListType:
457+
offsetWidth := dt.Layout().Buffers[1].ByteWidth
458+
offsetBuffer, valueRanges, err := concatOffsets(gatherFixedBuffers(data, 1, offsetWidth), offsetWidth, mem)
459+
if err != nil {
460+
return nil, err
461+
}
462+
childData := gatherChildrenRanges(data, 0, valueRanges)
463+
for _, c := range childData {
464+
defer c.Release()
465+
}
466+
450467
out.buffers[1] = offsetBuffer
451468
out.childData = make([]arrow.ArrayData, 1)
452469
out.childData[0], err = concat(childData, mem)

‎go/arrow/array/concat_test.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ func TestConcatenate(t *testing.T) {
7676
{arrow.BinaryTypes.String},
7777
{arrow.BinaryTypes.LargeString},
7878
{arrow.ListOf(arrow.PrimitiveTypes.Int8)},
79+
{arrow.LargeListOf(arrow.PrimitiveTypes.Int8)},
7980
{arrow.FixedSizeListOf(3, arrow.PrimitiveTypes.Int8)},
8081
{arrow.StructOf()},
8182
{arrow.MapOf(arrow.PrimitiveTypes.Uint16, arrow.PrimitiveTypes.Int8)},
@@ -158,6 +159,32 @@ func (cts *ConcatTestSuite) generateArr(size int64, nullprob float64) arrow.Arra
158159
bldr := array.NewListBuilder(memory.DefaultAllocator, arrow.PrimitiveTypes.Int8)
159160
defer bldr.Release()
160161

162+
valid := make([]bool, len(offsetsVector)-1)
163+
for i := range valid {
164+
valid[i] = true
165+
}
166+
bldr.AppendValues(offsetsVector, valid)
167+
vb := bldr.ValueBuilder().(*array.Int8Builder)
168+
for i := 0; i < values.Len(); i++ {
169+
if values.IsValid(i) {
170+
vb.Append(values.Value(i))
171+
} else {
172+
vb.AppendNull()
173+
}
174+
}
175+
return bldr.NewArray()
176+
case arrow.LARGE_LIST:
177+
valuesSize := size * 8
178+
values := cts.rng.Int8(valuesSize, 0, 127, nullprob).(*array.Int8)
179+
defer values.Release()
180+
offsetsVector := cts.largeoffsets(int64(valuesSize), int32(size))
181+
// ensure the first and last offsets encompass the whole values
182+
offsetsVector[0] = 0
183+
offsetsVector[len(offsetsVector)-1] = int64(valuesSize)
184+
185+
bldr := array.NewLargeListBuilder(memory.DefaultAllocator, arrow.PrimitiveTypes.Int8)
186+
defer bldr.Release()
187+
161188
valid := make([]bool, len(offsetsVector)-1)
162189
for i := range valid {
163190
valid[i] = true
@@ -263,6 +290,16 @@ func (cts *ConcatTestSuite) offsets(length, slicecount int32) []int32 {
263290
return offsets
264291
}
265292

293+
func (cts *ConcatTestSuite) largeoffsets(length int64, slicecount int32) []int64 {
294+
offsets := make([]int64, slicecount+1)
295+
dist := rand.New(rand.NewSource(cts.seed))
296+
for i := range offsets {
297+
offsets[i] = dist.Int63n(length + 1)
298+
}
299+
sort.Slice(offsets, func(i, j int) bool { return offsets[i] < offsets[j] })
300+
return offsets
301+
}
302+
266303
func (cts *ConcatTestSuite) TestCheckConcat() {
267304
for _, sz := range cts.sizes {
268305
cts.Run(fmt.Sprintf("size %d", sz), func() {

0 commit comments

Comments
 (0)