Skip to content

Commit 8449c55

Browse files
authored
GH-33840: [Go] Improve SQLite Flight SQL Example and provide mainprog (#33841)
### Rationale for this change Better facilitate Flight SQL testing by providing a simple mainprog which can be easily installed via `go get` or `go install` for testing and for running CI against. ### What changes are included in this PR? Improvements to the FlightSQL SQLite Go example and the creation of a mainprog at github.com/apache/arrow/go/v11/arrow/flight/flightsql/cmd/sqlite_flightsql_server ### Are these changes tested? Covered by existing tests in sqlite_server_test.go * Closes: #33840 Authored-by: Matt Topol <[email protected]> Signed-off-by: Matt Topol <[email protected]>
1 parent 7672523 commit 8449c55

10 files changed

Lines changed: 416 additions & 69 deletions

File tree

‎go/arrow/cdata/cdata_exports.go‎

Lines changed: 42 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import (
3838
"fmt"
3939
"reflect"
4040
"runtime/cgo"
41+
"strconv"
4142
"strings"
4243
"unsafe"
4344

@@ -152,6 +153,8 @@ func (exp *schemaExporter) exportFormat(dt arrow.DataType) string {
152153
return fmt.Sprintf("w:%d", dt.ByteWidth)
153154
case *arrow.Decimal128Type:
154155
return fmt.Sprintf("d:%d,%d", dt.Precision, dt.Scale)
156+
case *arrow.Decimal256Type:
157+
return fmt.Sprintf("d:%d,%d,256", dt.Precision, dt.Scale)
155158
case *arrow.BinaryType:
156159
return "z"
157160
case *arrow.LargeBinaryType:
@@ -235,6 +238,20 @@ func (exp *schemaExporter) exportFormat(dt arrow.DataType) string {
235238
exp.flags |= C.ARROW_FLAG_DICTIONARY_ORDERED
236239
}
237240
return exp.exportFormat(dt.IndexType)
241+
case arrow.UnionType:
242+
var b strings.Builder
243+
if dt.Mode() == arrow.SparseMode {
244+
b.WriteString("+us:")
245+
} else {
246+
b.WriteString("+ud:")
247+
}
248+
for i, c := range dt.TypeCodes() {
249+
if i != 0 {
250+
b.WriteByte(',')
251+
}
252+
b.WriteString(strconv.Itoa(int(c)))
253+
}
254+
return b.String()
238255
}
239256
panic("unsupported data type for export")
240257
}
@@ -250,23 +267,11 @@ func (exp *schemaExporter) export(field arrow.Field) {
250267
case *arrow.DictionaryType:
251268
exp.dict = new(schemaExporter)
252269
exp.dict.export(arrow.Field{Type: dt.ValueType})
253-
case *arrow.ListType:
254-
exp.children = make([]schemaExporter, 1)
255-
exp.children[0].export(dt.ElemField())
256-
case *arrow.LargeListType:
257-
exp.children = make([]schemaExporter, 1)
258-
exp.children[0].export(dt.ElemField())
259-
case *arrow.StructType:
270+
case arrow.NestedType:
260271
exp.children = make([]schemaExporter, len(dt.Fields()))
261272
for i, f := range dt.Fields() {
262273
exp.children[i].export(f)
263274
}
264-
case *arrow.MapType:
265-
exp.children = make([]schemaExporter, 1)
266-
exp.children[0].export(dt.ValueField())
267-
case *arrow.FixedSizeListType:
268-
exp.children = make([]schemaExporter, 1)
269-
exp.children[0].export(dt.ElemField())
270275
}
271276

272277
exp.exportMeta(&field.Metadata)
@@ -364,9 +369,21 @@ func exportArray(arr arrow.Array, out *CArrowArray, outSchema *CArrowSchema) {
364369
out.n_buffers = C.int64_t(len(arr.Data().Buffers()))
365370

366371
if out.n_buffers > 0 {
367-
buffers := allocateBufferPtrArr(len(arr.Data().Buffers()))
368-
for i := range arr.Data().Buffers() {
369-
buf := arr.Data().Buffers()[i]
372+
var (
373+
nbuffers = len(arr.Data().Buffers())
374+
bufs = arr.Data().Buffers()
375+
)
376+
// unions don't have validity bitmaps, but we keep them shifted
377+
// to make processing easier in other contexts. This means that
378+
// we have to adjust for union arrays
379+
if arr.DataType().ID() == arrow.DENSE_UNION || arr.DataType().ID() == arrow.SPARSE_UNION {
380+
out.n_buffers--
381+
nbuffers--
382+
bufs = bufs[1:]
383+
}
384+
buffers := allocateBufferPtrArr(nbuffers)
385+
for i := range bufs {
386+
buf := bufs[i]
370387
if buf == nil || buf.Len() == 0 {
371388
buffers[i] = nil
372389
continue
@@ -408,6 +425,15 @@ func exportArray(arr arrow.Array, out *CArrowArray, outSchema *CArrowSchema) {
408425
case *array.Dictionary:
409426
out.dictionary = (*CArrowArray)(C.malloc(C.sizeof_struct_ArrowArray))
410427
exportArray(arr.Dictionary(), out.dictionary, nil)
428+
case array.Union:
429+
out.n_children = C.int64_t(arr.NumFields())
430+
childPtrs := allocateArrowArrayPtrArr(arr.NumFields())
431+
children := allocateArrowArrayArr(arr.NumFields())
432+
for i := 0; i < arr.NumFields(); i++ {
433+
exportArray(arr.Field(i), &children[i], nil)
434+
childPtrs[i] = &children[i]
435+
}
436+
out.children = (**CArrowArray)(unsafe.Pointer(&childPtrs[0]))
411437
default:
412438
out.n_children = 0
413439
out.children = nil

‎go/arrow/cdata/cdata_fulltest.c‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,27 @@ struct ArrowSchema** test_map(const char** fmts, const char** names, int64_t* fl
244244
return schemas;
245245
}
246246

247+
struct ArrowSchema** test_union(const char** fmts, const char** names, int64_t* flags, const int n) {
248+
struct ArrowSchema** schemas = malloc(sizeof(struct ArrowSchema*)*n);
249+
for (int i = 0; i < n; ++i) {
250+
schemas[i] = malloc(sizeof(struct ArrowSchema));
251+
*schemas[i] = (struct ArrowSchema) {
252+
.format = fmts[i],
253+
.name = names[i],
254+
.metadata = NULL,
255+
.flags = flags[i],
256+
.children = NULL,
257+
.n_children = 0,
258+
.dictionary = NULL,
259+
.release = &release_nested_dynamic,
260+
};
261+
}
262+
263+
schemas[0]->n_children = n-1;
264+
schemas[0]->children = &schemas[1];
265+
return schemas;
266+
}
267+
247268
struct streamcounter {
248269
int n;
249270
int max;

‎go/arrow/cdata/cdata_test.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -579,6 +579,50 @@ func createTestMapArr() arrow.Array {
579579
return bld.NewArray()
580580
}
581581

582+
func createTestSparseUnion() arrow.Array {
583+
return createTestUnionArr(arrow.SparseMode)
584+
}
585+
586+
func createTestDenseUnion() arrow.Array {
587+
return createTestUnionArr(arrow.DenseMode)
588+
}
589+
590+
func createTestUnionArr(mode arrow.UnionMode) arrow.Array {
591+
fields := []arrow.Field{
592+
arrow.Field{Name: "u0", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
593+
arrow.Field{Name: "u1", Type: arrow.PrimitiveTypes.Uint8, Nullable: true},
594+
}
595+
typeCodes := []arrow.UnionTypeCode{5, 10}
596+
bld := array.NewBuilder(memory.DefaultAllocator, arrow.UnionOf(mode, fields, typeCodes)).(array.UnionBuilder)
597+
defer bld.Release()
598+
599+
u0Bld := bld.Child(0).(*array.Int32Builder)
600+
u1Bld := bld.Child(1).(*array.Uint8Builder)
601+
602+
bld.Append(5)
603+
if mode == arrow.SparseMode {
604+
u1Bld.AppendNull()
605+
}
606+
u0Bld.Append(128)
607+
bld.Append(5)
608+
if mode == arrow.SparseMode {
609+
u1Bld.AppendNull()
610+
}
611+
u0Bld.Append(256)
612+
bld.Append(10)
613+
if mode == arrow.SparseMode {
614+
u0Bld.AppendNull()
615+
}
616+
u1Bld.Append(127)
617+
bld.Append(10)
618+
if mode == arrow.SparseMode {
619+
u0Bld.AppendNull()
620+
}
621+
u1Bld.Append(25)
622+
623+
return bld.NewArray()
624+
}
625+
582626
func TestNestedArrays(t *testing.T) {
583627
tests := []struct {
584628
name string
@@ -589,6 +633,8 @@ func TestNestedArrays(t *testing.T) {
589633
{"fixed size list", createTestFixedSizeList},
590634
{"struct", createTestStructArr},
591635
{"map", createTestMapArr},
636+
{"sparse union", createTestSparseUnion},
637+
{"dense union", createTestDenseUnion},
592638
}
593639

594640
for _, tt := range tests {

‎go/arrow/cdata/cdata_test_framework.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ package cdata
5353
// struct ArrowSchema** test_struct(const char** fmts, const char** names, int64_t* flags, const int n);
5454
// struct ArrowSchema** test_map(const char** fmts, const char** names, int64_t* flags, const int n);
5555
// struct ArrowSchema** test_schema(const char** fmts, const char** names, int64_t* flags, const int n);
56+
// struct ArrowSchema** test_union(const char** fmts, const char** names, int64_t* flags, const int n);
5657
// int test_exported_stream(struct ArrowArrayStream* stream);
5758
import "C"
5859
import (
@@ -182,6 +183,24 @@ func testMap(fmts, names []string, flags []int64) **CArrowSchema {
182183
return C.test_map((**C.char)(unsafe.Pointer(&cfmts[0])), (**C.char)(unsafe.Pointer(&cnames[0])), (*C.int64_t)(unsafe.Pointer(&cflags[0])), C.int(len(fmts)))
183184
}
184185

186+
func testUnion(fmts, names []string, flags []int64) **CArrowSchema {
187+
if len(fmts) != len(names) || len(names) != len(flags) {
188+
panic("testing unions must all have the same size slices in args")
189+
}
190+
191+
cfmts := make([]*C.char, len(fmts))
192+
cnames := make([]*C.char, len(names))
193+
cflags := make([]C.int64_t, len(flags))
194+
195+
for i := range fmts {
196+
cfmts[i] = C.CString(fmts[i])
197+
cnames[i] = C.CString(names[i])
198+
cflags[i] = C.int64_t(flags[i])
199+
}
200+
201+
return C.test_union((**C.char)(unsafe.Pointer(&cfmts[0])), (**C.char)(unsafe.Pointer(&cnames[0])), (*C.int64_t)(unsafe.Pointer(&cflags[0])), C.int(len(fmts)))
202+
}
203+
185204
func testSchema(fmts, names []string, flags []int64) **CArrowSchema {
186205
if len(fmts) != len(names) || len(names) != len(flags) {
187206
panic("testing structs must all have the same size slices in args")
@@ -235,6 +254,13 @@ func createCArr(arr arrow.Array) *CArrowArray {
235254
clist := []*CArrowArray{createCArr(arr.ListValues())}
236255
children = (**CArrowArray)(unsafe.Pointer(&clist[0]))
237256
nchildren += 1
257+
case array.Union:
258+
clist := []*CArrowArray{}
259+
for i := 0; i < arr.NumFields(); i++ {
260+
clist = append(clist, createCArr(arr.Field(i)))
261+
nchildren += 1
262+
}
263+
children = (**CArrowArray)(unsafe.Pointer(&clist[0]))
238264
}
239265

240266
carr.children = children

‎go/arrow/datatype_nested.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -596,7 +596,7 @@ func (t *SparseUnionType) Fingerprint() string {
596596
return typeFingerprint(t) + "[s" + t.fingerprint()
597597
}
598598
func (SparseUnionType) Layout() DataTypeLayout {
599-
return DataTypeLayout{Buffers: []BufferSpec{SpecAlwaysNull(), SpecFixedWidth(Uint8SizeBytes)}}
599+
return DataTypeLayout{Buffers: []BufferSpec{SpecFixedWidth(Uint8SizeBytes)}}
600600
}
601601
func (t *SparseUnionType) String() string {
602602
return t.Name() + t.unionType.String()
@@ -659,7 +659,7 @@ func (t *DenseUnionType) Fingerprint() string {
659659
}
660660

661661
func (DenseUnionType) Layout() DataTypeLayout {
662-
return DataTypeLayout{Buffers: []BufferSpec{SpecAlwaysNull(), SpecFixedWidth(Uint8SizeBytes), SpecFixedWidth(Int32SizeBytes)}}
662+
return DataTypeLayout{Buffers: []BufferSpec{SpecFixedWidth(Uint8SizeBytes), SpecFixedWidth(Int32SizeBytes)}}
663663
}
664664

665665
func (DenseUnionType) OffsetTypeTraits() OffsetTraits { return Int32Traits }
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing, software
12+
// distributed under the License is distributed on an "AS IS" BASIS,
13+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
// See the License for the specific language governing permissions and
15+
// limitations under the License.
16+
17+
//go:build go1.18
18+
// +build go1.18
19+
20+
package main
21+
22+
import (
23+
"flag"
24+
"fmt"
25+
"log"
26+
"net"
27+
"os"
28+
"strconv"
29+
30+
"github.com/apache/arrow/go/v11/arrow/flight"
31+
"github.com/apache/arrow/go/v11/arrow/flight/flightsql"
32+
"github.com/apache/arrow/go/v11/arrow/flight/flightsql/example"
33+
)
34+
35+
func main() {
36+
var (
37+
host = flag.String("host", "localhost", "hostname to bind to")
38+
port = flag.Int("port", 0, "port to bind to")
39+
)
40+
41+
flag.Parse()
42+
43+
srv, err := example.NewSQLiteFlightSQLServer()
44+
if err != nil {
45+
log.Fatal(err)
46+
}
47+
48+
server := flight.NewServerWithMiddleware(nil)
49+
server.RegisterFlightService(flightsql.NewFlightServer(srv))
50+
server.Init(net.JoinHostPort(*host, strconv.Itoa(*port)))
51+
server.SetShutdownOnSignals(os.Interrupt, os.Kill)
52+
53+
fmt.Println("Starting SQLite Flight SQL Server on", server.Addr(), "...")
54+
55+
if err := server.Serve(); err != nil {
56+
log.Fatal(err)
57+
}
58+
}

0 commit comments

Comments
 (0)