Skip to content

Commit cd23529

Browse files
zeroshadeArgusLi
authored andcommitted
apacheGH-32950: [Go] REE Benchmarks (apache#34666)
### Rationale for this change Adding benchmarks for `run_end_encode` and `run_end_decode` to track and hopefully improve the performance of these kernels. * Closes: apache#32950 Authored-by: Matt Topol <[email protected]> Signed-off-by: Matt Topol <[email protected]>
1 parent c10944e commit cd23529

3 files changed

Lines changed: 143 additions & 9 deletions

File tree

‎go/arrow/compute/internal/kernels/vector_run_end_encode.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,8 @@ func (re *runEndEncodeLoopFixedWidth[R, V]) WriteEncodedRuns(out *exec.ExecResul
9292
readOffset++
9393

9494
var writeOffset int64
95+
var value V
9596
for readOffset < re.inputOffset+re.inputLen {
96-
var value V
9797
valid := re.readValue(re.inputValidity, re.inputValues, readOffset, &value)
9898
if valid != curRunValid || value != currentRun {
9999
// close the current run by writing it out
@@ -122,8 +122,8 @@ func (re *runEndEncodeLoopFixedWidth[R, V]) CountNumberOfRuns() (numValid, numOu
122122
}
123123
numOutput = 1
124124

125+
var value V
125126
for offset < re.inputOffset+re.inputLen {
126-
var value V
127127
valid := re.readValue(re.inputValidity, re.inputValues, offset, &value)
128128
offset++
129129
// new run

‎go/arrow/compute/vector_run_end_test.go‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,17 @@ package compute_test
2020

2121
import (
2222
"context"
23+
"fmt"
24+
"math"
2325
"strings"
2426
"testing"
2527

2628
"github.com/apache/arrow/go/v12/arrow"
2729
"github.com/apache/arrow/go/v12/arrow/array"
2830
"github.com/apache/arrow/go/v12/arrow/bitutil"
2931
"github.com/apache/arrow/go/v12/arrow/compute"
32+
"github.com/apache/arrow/go/v12/arrow/compute/internal/exec"
33+
"github.com/apache/arrow/go/v12/arrow/internal/testing/gen"
3034
"github.com/apache/arrow/go/v12/arrow/memory"
3135
"github.com/stretchr/testify/suite"
3236
)
@@ -295,3 +299,125 @@ func TestRunEndFunctions(t *testing.T) {
295299
})
296300
}
297301
}
302+
303+
func benchRunEndEncode(b *testing.B, sz int, nullProb float64, runEndType, valueType arrow.DataType) {
304+
b.Run("encode", func(b *testing.B) {
305+
var (
306+
mem = memory.NewCheckedAllocator(memory.DefaultAllocator)
307+
rng = gen.NewRandomArrayGenerator(seed, mem)
308+
)
309+
310+
values := rng.ArrayOf(valueType.ID(), int64(sz), nullProb)
311+
b.Cleanup(func() {
312+
values.Release()
313+
})
314+
315+
var (
316+
res compute.Datum
317+
err error
318+
ctx = compute.WithAllocator(context.Background(), mem)
319+
input = &compute.ArrayDatum{Value: values.Data()}
320+
opts = compute.RunEndEncodeOptions{RunEndType: runEndType}
321+
322+
byts int64
323+
)
324+
325+
for _, buf := range values.Data().Buffers() {
326+
if buf != nil {
327+
byts += int64(buf.Len())
328+
}
329+
}
330+
331+
b.SetBytes(byts)
332+
b.ResetTimer()
333+
for n := 0; n < b.N; n++ {
334+
res, err = compute.RunEndEncode(ctx, opts, input)
335+
b.StopTimer()
336+
if err != nil {
337+
b.Fatal(err)
338+
}
339+
res.Release()
340+
b.StartTimer()
341+
}
342+
})
343+
}
344+
345+
func benchRunEndDecode(b *testing.B, sz int, nullProb float64, runEndType, valueType arrow.DataType) {
346+
b.Run("decode", func(b *testing.B) {
347+
var (
348+
mem = memory.NewCheckedAllocator(memory.DefaultAllocator)
349+
rng = gen.NewRandomArrayGenerator(seed, mem)
350+
)
351+
352+
values := rng.ArrayOf(valueType.ID(), int64(sz), nullProb)
353+
b.Cleanup(func() {
354+
values.Release()
355+
})
356+
357+
var (
358+
res compute.Datum
359+
ctx = compute.WithAllocator(context.Background(), mem)
360+
opts = compute.RunEndEncodeOptions{RunEndType: runEndType}
361+
input, err = compute.RunEndEncode(ctx, opts, &compute.ArrayDatum{Value: values.Data()})
362+
byts int64
363+
)
364+
365+
if err != nil {
366+
b.Fatal(err)
367+
}
368+
369+
for _, buf := range values.Data().Buffers() {
370+
if buf != nil {
371+
byts += int64(buf.Len())
372+
}
373+
}
374+
375+
b.SetBytes(byts)
376+
b.ResetTimer()
377+
for n := 0; n < b.N; n++ {
378+
res, err = compute.RunEndDecode(ctx, input)
379+
b.StopTimer()
380+
if err != nil {
381+
b.Fatal(err)
382+
}
383+
res.Release()
384+
b.StartTimer()
385+
}
386+
})
387+
}
388+
389+
func BenchmarkRunEndKernels(b *testing.B) {
390+
args := []struct {
391+
sz int
392+
nullProb float64
393+
}{
394+
{CpuCacheSizes[2], 0},
395+
{CpuCacheSizes[2], 0.5},
396+
{CpuCacheSizes[2], 1},
397+
}
398+
399+
runEnds := []struct {
400+
dt arrow.DataType
401+
maxLen int
402+
}{
403+
{arrow.PrimitiveTypes.Int16, math.MaxInt16},
404+
{arrow.PrimitiveTypes.Int32, math.MaxInt32},
405+
{arrow.PrimitiveTypes.Int64, math.MaxInt64},
406+
}
407+
408+
for _, a := range args {
409+
b.Run(fmt.Sprintf("nullprob=%.1f", a.nullProb), func(b *testing.B) {
410+
for _, runEndType := range runEnds {
411+
sz := exec.Min(a.sz, runEndType.maxLen)
412+
b.Run("run_ends_type="+runEndType.dt.String(), func(b *testing.B) {
413+
for _, valType := range append(numericTypes, arrow.BinaryTypes.String, arrow.FixedWidthTypes.Boolean) {
414+
b.Run("value_type="+valType.String(), func(b *testing.B) {
415+
benchRunEndEncode(b, sz, a.nullProb, runEndType.dt, valType)
416+
benchRunEndDecode(b, sz, a.nullProb, runEndType.dt, valType)
417+
})
418+
}
419+
})
420+
}
421+
})
422+
}
423+
}

‎go/arrow/internal/testing/gen/random_array_gen.go‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ func (r *RandomArrayGenerator) Uint8(size int64, min, max uint8, prob float64) a
126126
dist := rand.New(rand.NewSource(r.seed + r.extra))
127127
out := arrow.Uint8Traits.CastFromBytes(buffers[1].Bytes())
128128
for i := int64(0); i < size; i++ {
129-
out[i] = uint8(dist.Intn(int(max-min+1))) + min
129+
out[i] = uint8(dist.Intn(int(max)-int(min)+1)) + min
130130
}
131131

132132
data := array.NewData(arrow.PrimitiveTypes.Uint8, int(size), buffers, nil, int(nullcount), 0)
@@ -144,7 +144,7 @@ func (r *RandomArrayGenerator) Int16(size int64, min, max int16, prob float64) a
144144
dist := rand.New(rand.NewSource(r.seed + r.extra))
145145
out := arrow.Int16Traits.CastFromBytes(buffers[1].Bytes())
146146
for i := int64(0); i < size; i++ {
147-
out[i] = int16(dist.Intn(int(max-min+1))) + min
147+
out[i] = int16(dist.Intn(int(max)-int(min)+1)) + min
148148
}
149149

150150
data := array.NewData(arrow.PrimitiveTypes.Int16, int(size), buffers, nil, int(nullcount), 0)
@@ -162,7 +162,7 @@ func (r *RandomArrayGenerator) Uint16(size int64, min, max uint16, prob float64)
162162
dist := rand.New(rand.NewSource(r.seed + r.extra))
163163
out := arrow.Uint16Traits.CastFromBytes(buffers[1].Bytes())
164164
for i := int64(0); i < size; i++ {
165-
out[i] = uint16(dist.Intn(int(max-min+1))) + min
165+
out[i] = uint16(dist.Intn(int(max)-int(min)+1)) + min
166166
}
167167

168168
data := array.NewData(arrow.PrimitiveTypes.Uint16, int(size), buffers, nil, int(nullcount), 0)
@@ -180,7 +180,7 @@ func (r *RandomArrayGenerator) Int32(size int64, min, max int32, prob float64) a
180180
dist := rand.New(rand.NewSource(r.seed + r.extra))
181181
out := arrow.Int32Traits.CastFromBytes(buffers[1].Bytes())
182182
for i := int64(0); i < size; i++ {
183-
out[i] = dist.Int31n(max-min+1) + min
183+
out[i] = int32(dist.Intn(int(max)-int(min)+1)) + min
184184
}
185185

186186
data := array.NewData(arrow.PrimitiveTypes.Int32, int(size), buffers, nil, int(nullcount), 0)
@@ -198,7 +198,7 @@ func (r *RandomArrayGenerator) Uint32(size int64, min, max uint32, prob float64)
198198
dist := rand.New(rand.NewSource(r.seed + r.extra))
199199
out := arrow.Uint32Traits.CastFromBytes(buffers[1].Bytes())
200200
for i := int64(0); i < size; i++ {
201-
out[i] = uint32(dist.Uint64n(uint64(max-min+1))) + min
201+
out[i] = uint32(dist.Uint64n(uint64(max)-uint64(min)+1)) + min
202202
}
203203

204204
data := array.NewData(arrow.PrimitiveTypes.Uint32, int(size), buffers, nil, int(nullcount), 0)
@@ -239,8 +239,14 @@ func (r *RandomArrayGenerator) Uint64(size int64, min, max uint64, prob float64)
239239
r.extra++
240240
dist := rand.New(rand.NewSource(r.seed + r.extra))
241241
out := arrow.Uint64Traits.CastFromBytes(buffers[1].Bytes())
242-
for i := int64(0); i < size; i++ {
243-
out[i] = dist.Uint64n(max-min+1) + min
242+
if max == math.MaxUint64 {
243+
for i := int64(0); i < size; i++ {
244+
out[i] = dist.Uint64() + min
245+
}
246+
} else {
247+
for i := int64(0); i < size; i++ {
248+
out[i] = dist.Uint64n(max-min+1) + min
249+
}
244250
}
245251

246252
data := array.NewData(arrow.PrimitiveTypes.Uint64, int(size), buffers, nil, int(nullcount), 0)
@@ -372,6 +378,8 @@ func (r *RandomArrayGenerator) Numeric(dt arrow.Type, size int64, min, max int64
372378

373379
func (r *RandomArrayGenerator) ArrayOf(dt arrow.Type, size int64, nullprob float64) arrow.Array {
374380
switch dt {
381+
case arrow.BOOL:
382+
return r.Boolean(size, 0.50, nullprob)
375383
case arrow.STRING:
376384
return r.String(size, 0, 20, nullprob)
377385
case arrow.LARGE_STRING:

0 commit comments

Comments
 (0)