Skip to content

Commit 9a13734

Browse files
authored
GH-32946: [Go] Implement REE Array and Compare (#14111)
* Closes: #32946 Authored-by: Matt Topol <[email protected]> Signed-off-by: Matt Topol <[email protected]>
1 parent 7579dbd commit 9a13734

14 files changed

Lines changed: 820 additions & 50 deletions

File tree

‎go/arrow/array/array.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,8 +175,8 @@ func init() {
175175
arrow.LARGE_STRING: func(data arrow.ArrayData) arrow.Array { return NewLargeStringData(data) },
176176
arrow.LARGE_BINARY: func(data arrow.ArrayData) arrow.Array { return NewLargeBinaryData(data) },
177177
arrow.LARGE_LIST: func(data arrow.ArrayData) arrow.Array { return NewLargeListData(data) },
178-
arrow.INTERVAL: func(data arrow.ArrayData) arrow.Array { return NewIntervalData(data) },
179178
arrow.INTERVAL_MONTH_DAY_NANO: func(data arrow.ArrayData) arrow.Array { return NewMonthDayNanoIntervalData(data) },
179+
arrow.RUN_END_ENCODED: func(data arrow.ArrayData) arrow.Array { return NewRunEndEncodedData(data) },
180180

181181
// invalid data types to fill out array to size 2^6 - 1
182182
63: invalidDataType,

‎go/arrow/array/builder.go‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -281,15 +281,6 @@ func NewBuilder(mem memory.Allocator, dtype arrow.DataType) Builder {
281281
case arrow.TIME64:
282282
typ := dtype.(*arrow.Time64Type)
283283
return NewTime64Builder(mem, typ)
284-
case arrow.INTERVAL:
285-
switch dtype.(type) {
286-
case *arrow.DayTimeIntervalType:
287-
return NewDayTimeIntervalBuilder(mem)
288-
case *arrow.MonthIntervalType:
289-
return NewMonthIntervalBuilder(mem)
290-
case *arrow.MonthDayNanoIntervalType:
291-
return NewMonthDayNanoIntervalBuilder(mem)
292-
}
293284
case arrow.INTERVAL_MONTHS:
294285
return NewMonthIntervalBuilder(mem)
295286
case arrow.INTERVAL_DAY_TIME:

‎go/arrow/array/compare.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,9 @@ func Equal(left, right arrow.Array) bool {
333333
case *DenseUnion:
334334
r := right.(*DenseUnion)
335335
return arrayDenseUnionEqual(l, r)
336+
case *RunEndEncoded:
337+
r := right.(*RunEndEncoded)
338+
return arrayRunEndEncodedEqual(l, r)
336339
default:
337340
panic(fmt.Errorf("arrow/array: unknown array type %T", l))
338341
}
@@ -591,6 +594,9 @@ func arrayApproxEqual(left, right arrow.Array, opt equalOption) bool {
591594
case *DenseUnion:
592595
r := right.(*DenseUnion)
593596
return arrayDenseUnionApproxEqual(l, r, opt)
597+
case *RunEndEncoded:
598+
r := right.(*RunEndEncoded)
599+
return arrayRunEndEncodedApproxEqual(l, r, opt)
594600
default:
595601
panic(fmt.Errorf("arrow/array: unknown array type %T", l))
596602
}

‎go/arrow/array/encoded.go‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing, software
12+
// distributed under the License is distributed on an "AS IS" BASIS,
13+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
// See the License for the specific language governing permissions and
15+
// limitations under the License.
16+
17+
package array
18+
19+
import (
20+
"bytes"
21+
"fmt"
22+
23+
"github.com/apache/arrow/go/v11/arrow"
24+
"github.com/apache/arrow/go/v11/arrow/encoded"
25+
"github.com/apache/arrow/go/v11/arrow/internal/debug"
26+
"github.com/apache/arrow/go/v11/arrow/memory"
27+
"github.com/goccy/go-json"
28+
)
29+
30+
// RunEndEncoded represents an array containing two children:
31+
// an array of int32 values defining the ends of each run of values
32+
// and an array of values
33+
type RunEndEncoded struct {
34+
array
35+
36+
ends arrow.Array
37+
values arrow.Array
38+
}
39+
40+
func NewRunEndEncodedArray(runEnds, values arrow.Array, logicalLength, offset int) *RunEndEncoded {
41+
data := NewData(arrow.RunEndEncodedOf(runEnds.DataType(), values.DataType()), logicalLength,
42+
[]*memory.Buffer{nil}, []arrow.ArrayData{runEnds.Data(), values.Data()}, 0, offset)
43+
defer data.Release()
44+
return NewRunEndEncodedData(data)
45+
}
46+
47+
func NewRunEndEncodedData(data arrow.ArrayData) *RunEndEncoded {
48+
r := &RunEndEncoded{}
49+
r.refCount = 1
50+
r.setData(data.(*Data))
51+
return r
52+
}
53+
54+
func (r *RunEndEncoded) Values() arrow.Array { return r.values }
55+
func (r *RunEndEncoded) RunEndsArr() arrow.Array { return r.ends }
56+
57+
func (r *RunEndEncoded) Retain() {
58+
r.array.Retain()
59+
r.values.Retain()
60+
r.ends.Retain()
61+
}
62+
63+
func (r *RunEndEncoded) Release() {
64+
r.array.Release()
65+
r.values.Release()
66+
r.ends.Release()
67+
}
68+
69+
func (r *RunEndEncoded) setData(data *Data) {
70+
if len(data.childData) != 2 {
71+
panic(fmt.Errorf("%w: arrow/array: RLE array must have exactly 2 children", arrow.ErrInvalid))
72+
}
73+
debug.Assert(data.dtype.ID() == arrow.RUN_END_ENCODED, "invalid type for RunLengthEncoded")
74+
if !data.dtype.(*arrow.RunEndEncodedType).ValidRunEndsType(data.childData[0].DataType()) {
75+
panic(fmt.Errorf("%w: arrow/array: run ends array must be int16, int32, or int64", arrow.ErrInvalid))
76+
}
77+
if data.childData[0].NullN() > 0 {
78+
panic(fmt.Errorf("%w: arrow/array: run ends array cannot contain nulls", arrow.ErrInvalid))
79+
}
80+
81+
r.array.setData(data)
82+
83+
r.ends = MakeFromData(r.data.childData[0])
84+
r.values = MakeFromData(r.data.childData[1])
85+
}
86+
87+
func (r *RunEndEncoded) GetPhysicalOffset() int {
88+
return encoded.FindPhysicalOffset(r.data)
89+
}
90+
91+
func (r *RunEndEncoded) GetPhysicalLength() int {
92+
return encoded.GetPhysicalLength(r.data)
93+
}
94+
95+
func (r *RunEndEncoded) String() string {
96+
var buf bytes.Buffer
97+
buf.WriteByte('[')
98+
for i := 0; i < r.ends.Len(); i++ {
99+
if i != 0 {
100+
buf.WriteByte(',')
101+
}
102+
fmt.Fprintf(&buf, "{%v -> %v}",
103+
r.ends.(arraymarshal).getOneForMarshal(i),
104+
r.values.(arraymarshal).getOneForMarshal(i))
105+
}
106+
107+
buf.WriteByte(']')
108+
return buf.String()
109+
}
110+
111+
func (r *RunEndEncoded) getOneForMarshal(i int) interface{} {
112+
return [2]interface{}{r.ends.(arraymarshal).getOneForMarshal(i),
113+
r.values.(arraymarshal).getOneForMarshal(i)}
114+
}
115+
116+
func (r *RunEndEncoded) MarshalJSON() ([]byte, error) {
117+
var buf bytes.Buffer
118+
enc := json.NewEncoder(&buf)
119+
buf.WriteByte('[')
120+
for i := 0; i < r.ends.Len(); i++ {
121+
if i != 0 {
122+
buf.WriteByte(',')
123+
}
124+
if err := enc.Encode(r.getOneForMarshal(i)); err != nil {
125+
return nil, err
126+
}
127+
}
128+
buf.WriteByte(']')
129+
return buf.Bytes(), nil
130+
}
131+
132+
func arrayRunEndEncodedEqual(l, r *RunEndEncoded) bool {
133+
// types were already checked before getting here, so we know
134+
// the encoded types are equal
135+
mr := encoded.NewMergedRuns([2]arrow.Array{l, r})
136+
for mr.Next() {
137+
lIndex := mr.IndexIntoArray(0)
138+
rIndex := mr.IndexIntoArray(1)
139+
if !SliceEqual(l.values, lIndex, lIndex+1, r.values, rIndex, rIndex+1) {
140+
return false
141+
}
142+
}
143+
return true
144+
}
145+
146+
func arrayRunEndEncodedApproxEqual(l, r *RunEndEncoded, opt equalOption) bool {
147+
// types were already checked before getting here, so we know
148+
// the encoded types are equal
149+
mr := encoded.NewMergedRuns([2]arrow.Array{l, r})
150+
for mr.Next() {
151+
lIndex := mr.IndexIntoArray(0)
152+
rIndex := mr.IndexIntoArray(1)
153+
if !sliceApproxEqual(l.values, lIndex, lIndex+1, r.values, rIndex, rIndex+1, opt) {
154+
return false
155+
}
156+
}
157+
return true
158+
}

0 commit comments

Comments
 (0)