@@ -20,12 +20,14 @@ import (
2020 "bytes"
2121 "fmt"
2222 "math"
23+ "reflect"
2324 "sync/atomic"
2425
2526 "github.com/apache/arrow/go/v12/arrow"
2627 "github.com/apache/arrow/go/v12/arrow/encoded"
2728 "github.com/apache/arrow/go/v12/arrow/internal/debug"
2829 "github.com/apache/arrow/go/v12/arrow/memory"
30+ "github.com/apache/arrow/go/v12/internal/utils"
2931 "github.com/goccy/go-json"
3032)
3133
@@ -68,6 +70,102 @@ func (r *RunEndEncoded) Release() {
6870 r .ends .Release ()
6971}
7072
73+ // LogicalValuesArray returns an array holding the values of each
74+ // run, only over the range of run values inside the logical offset/length
75+ // range of the parent array.
76+ //
77+ // Example
78+ //
79+ // For this array:
80+ // RunEndEncoded: { Offset: 150, Length: 1500 }
81+ // RunEnds: [ 1, 2, 4, 6, 10, 1000, 1750, 2000 ]
82+ // Values: [ "a", "b", "c", "d", "e", "f", "g", "h" ]
83+ //
84+ // LogicalValuesArray will return the following array:
85+ // [ "f", "g" ]
86+ //
87+ // This is because the offset of 150 tells it to skip the values until
88+ // "f" which corresponds with the logical offset (the run from 10 - 1000),
89+ // and stops after "g" because the length + offset goes to 1650 which is
90+ // within the run from 1000 - 1750, corresponding to the "g" value.
91+ //
92+ // Note
93+ //
94+ // The return from this needs to be Released.
95+ func (r * RunEndEncoded ) LogicalValuesArray () arrow.Array {
96+ physOffset := r .GetPhysicalOffset ()
97+ physLength := r .GetPhysicalLength ()
98+ data := NewSliceData (r .data .Children ()[1 ], int64 (physOffset ), int64 (physOffset + physLength ))
99+ defer data .Release ()
100+ return MakeFromData (data )
101+ }
102+
103+ // LogicalRunEndsArray returns an array holding the logical indexes
104+ // of each run end, only over the range of run end values relative
105+ // to the logical offset/length range of the parent array.
106+ //
107+ // For arrays with an offset, this is not a slice of the existing
108+ // internal run ends array. Instead a new array is created with run-ends
109+ // that are adjusted so the new array can have an offset of 0. As a result
110+ // this method can be expensive to call for an array with a non-zero offset.
111+ //
112+ // Example
113+ //
114+ // For this array:
115+ // RunEndEncoded: { Offset: 150, Length: 1500 }
116+ // RunEnds: [ 1, 2, 4, 6, 10, 1000, 1750, 2000 ]
117+ // Values: [ "a", "b", "c", "d", "e", "f", "g", "h" ]
118+ //
119+ // LogicalRunEndsArray will return the following array:
120+ // [ 850, 1500 ]
121+ //
122+ // This is because the offset of 150 tells us to skip all run-ends less
123+ // than 150 (by finding the physical offset), and we adjust the run-ends
124+ // accordingly (1000 - 150 = 850). The logical length of the array is 1500,
125+ // so we know we don't want to go past the 1750 run end. Thus the last
126+ // run-end is determined by doing: min(1750 - 150, 1500) = 1500.
127+ //
128+ // Note
129+ //
130+ // The return from this needs to be Released
131+ func (r * RunEndEncoded ) LogicalRunEndsArray (mem memory.Allocator ) arrow.Array {
132+ physOffset := r .GetPhysicalOffset ()
133+ physLength := r .GetPhysicalLength ()
134+
135+ if r .data .offset == 0 {
136+ data := NewSliceData (r .data .childData [0 ], 0 , int64 (physLength ))
137+ defer data .Release ()
138+ return MakeFromData (data )
139+ }
140+
141+ bldr := NewBuilder (mem , r .data .childData [0 ].DataType ())
142+ defer bldr .Release ()
143+ bldr .Resize (physLength )
144+
145+ switch e := r .ends .(type ) {
146+ case * Int16 :
147+ for _ , v := range e .Int16Values ()[physOffset : physOffset + physLength ] {
148+ v -= int16 (r .data .offset )
149+ v = int16 (utils .MinInt (int (v ), r .data .length ))
150+ bldr .(* Int16Builder ).Append (v )
151+ }
152+ case * Int32 :
153+ for _ , v := range e .Int32Values ()[physOffset : physOffset + physLength ] {
154+ v -= int32 (r .data .offset )
155+ v = int32 (utils .MinInt (int (v ), r .data .length ))
156+ bldr .(* Int32Builder ).Append (v )
157+ }
158+ case * Int64 :
159+ for _ , v := range e .Int64Values ()[physOffset : physOffset + physLength ] {
160+ v -= int64 (r .data .offset )
161+ v = int64 (utils .MinInt (int (v ), r .data .length ))
162+ bldr .(* Int64Builder ).Append (v )
163+ }
164+ }
165+
166+ return bldr .NewArray ()
167+ }
168+
71169func (r * RunEndEncoded ) setData (data * Data ) {
72170 if len (data .childData ) != 2 {
73171 panic (fmt .Errorf ("%w: arrow/array: RLE array must have exactly 2 children" , arrow .ErrInvalid ))
@@ -101,25 +199,30 @@ func (r *RunEndEncoded) String() string {
101199 if i != 0 {
102200 buf .WriteByte (',' )
103201 }
104- fmt .Fprintf (& buf , "{%v -> %v}" ,
202+
203+ value := r .values .(arraymarshal ).getOneForMarshal (i )
204+ if byts , ok := value .(json.RawMessage ); ok {
205+ value = string (byts )
206+ }
207+ fmt .Fprintf (& buf , "{%d -> %v}" ,
105208 r .ends .(arraymarshal ).getOneForMarshal (i ),
106- r . values .( arraymarshal ). getOneForMarshal ( i ) )
209+ value )
107210 }
108211
109212 buf .WriteByte (']' )
110213 return buf .String ()
111214}
112215
113216func (r * RunEndEncoded ) getOneForMarshal (i int ) interface {} {
114- return [ 2 ] interface {}{ r . ends .( arraymarshal ). getOneForMarshal ( i ),
115- r .values .(arraymarshal ).getOneForMarshal (i )}
217+ physIndex := encoded . FindPhysicalIndex ( r . data , i + r . data . offset )
218+ return r .values .(arraymarshal ).getOneForMarshal (physIndex )
116219}
117220
118221func (r * RunEndEncoded ) MarshalJSON () ([]byte , error ) {
119222 var buf bytes.Buffer
120223 enc := json .NewEncoder (& buf )
121224 buf .WriteByte ('[' )
122- for i := 0 ; i < r .ends . Len (); i ++ {
225+ for i := 0 ; i < r .Len (); i ++ {
123226 if i != 0 {
124227 buf .WriteByte (',' )
125228 }
@@ -166,6 +269,8 @@ type RunEndEncodedBuilder struct {
166269 runEnds Builder
167270 values Builder
168271 maxRunEnd uint64
272+
273+ lastUnmarshalled interface {}
169274}
170275
171276func NewRunEndEncodedBuilder (mem memory.Allocator , runEnds , encoded arrow.DataType ) * RunEndEncodedBuilder {
@@ -184,11 +289,12 @@ func NewRunEndEncodedBuilder(mem memory.Allocator, runEnds, encoded arrow.DataTy
184289 maxEnd = math .MaxInt64
185290 }
186291 return & RunEndEncodedBuilder {
187- builder : builder {refCount : 1 , mem : mem },
188- dt : dt ,
189- runEnds : NewBuilder (mem , runEnds ),
190- values : NewBuilder (mem , encoded ),
191- maxRunEnd : maxEnd ,
292+ builder : builder {refCount : 1 , mem : mem },
293+ dt : dt ,
294+ runEnds : NewBuilder (mem , runEnds ),
295+ values : NewBuilder (mem , encoded ),
296+ maxRunEnd : maxEnd ,
297+ lastUnmarshalled : nil ,
192298 }
193299}
194300
@@ -214,6 +320,7 @@ func (b *RunEndEncodedBuilder) addLength(n uint64) {
214320}
215321
216322func (b * RunEndEncodedBuilder ) finishRun () {
323+ b .lastUnmarshalled = nil
217324 if b .length == 0 {
218325 return
219326 }
@@ -233,6 +340,12 @@ func (b *RunEndEncodedBuilder) Append(n uint64) {
233340 b .finishRun ()
234341 b .addLength (n )
235342}
343+ func (b * RunEndEncodedBuilder ) AppendRuns (runs []uint64 ) {
344+ for _ , r := range runs {
345+ b .finishRun ()
346+ b .addLength (r )
347+ }
348+ }
236349func (b * RunEndEncodedBuilder ) ContinueRun (n uint64 ) {
237350 b .addLength (n )
238351}
@@ -285,10 +398,35 @@ func (b *RunEndEncodedBuilder) newData() (data *Data) {
285398}
286399
287400func (b * RunEndEncodedBuilder ) unmarshalOne (dec * json.Decoder ) error {
288- return arrow .ErrNotImplemented
401+ var value interface {}
402+ if err := dec .Decode (& value ); err != nil {
403+ return err
404+ }
405+
406+ // if we unmarshalled the same value as the previous one, we want to
407+ // continue the run. However, there's an edge case. At the start of
408+ // unmarshalling, lastUnmarshalled will be nil, but we might get
409+ // nil as the first value we unmarshal. In that case we want to
410+ // make sure we add a new run instead. We can detect that case by
411+ // checking that the number of runEnds matches the number of values
412+ // we have, which means no matter what we have to start a new run
413+ if reflect .DeepEqual (value , b .lastUnmarshalled ) && (value != nil || b .runEnds .Len () != b .values .Len ()) {
414+ b .ContinueRun (1 )
415+ return nil
416+ }
417+
418+ data , err := json .Marshal (value )
419+ if err != nil {
420+ return err
421+ }
422+
423+ b .Append (1 )
424+ b .lastUnmarshalled = value
425+ return b .ValueBuilder ().unmarshalOne (json .NewDecoder (bytes .NewReader (data )))
289426}
290427
291428func (b * RunEndEncodedBuilder ) unmarshal (dec * json.Decoder ) error {
429+ b .finishRun ()
292430 for dec .More () {
293431 if err := b .unmarshalOne (dec ); err != nil {
294432 return err
0 commit comments