@@ -86,6 +86,7 @@ type Writer struct {
8686 mapper dictutils.Mapper
8787 codec flatbuf.CompressionType
8888 compressNP int
89+ compressors []compressor
8990 minSpaceSavings * float64
9091
9192 // map of the last written dictionaries by id
@@ -107,6 +108,7 @@ func NewWriterWithPayloadWriter(pw PayloadWriter, opts ...Option) *Writer {
107108 compressNP : cfg .compressNP ,
108109 minSpaceSavings : cfg .minSpaceSavings ,
109110 emitDictDeltas : cfg .emitDictDeltas ,
111+ compressors : make ([]compressor , cfg .compressNP ),
110112 }
111113}
112114
@@ -120,6 +122,8 @@ func NewWriter(w io.Writer, opts ...Option) *Writer {
120122 schema : cfg .schema ,
121123 codec : cfg .codec ,
122124 emitDictDeltas : cfg .emitDictDeltas ,
125+ compressNP : cfg .compressNP ,
126+ compressors : make ([]compressor , cfg .compressNP ),
123127 }
124128}
125129
@@ -170,7 +174,16 @@ func (w *Writer) Write(rec arrow.Record) (err error) {
170174 const allow64b = true
171175 var (
172176 data = Payload {msg : MessageRecordBatch }
173- enc = newRecordEncoder (w .mem , 0 , kMaxNestingDepth , allow64b , w .codec , w .compressNP , w .minSpaceSavings )
177+ enc = newRecordEncoder (
178+ w .mem ,
179+ 0 ,
180+ kMaxNestingDepth ,
181+ allow64b ,
182+ w .codec ,
183+ w .compressNP ,
184+ w .minSpaceSavings ,
185+ w .compressors ,
186+ )
174187 )
175188 defer data .Release ()
176189
@@ -310,17 +323,28 @@ type recordEncoder struct {
310323 allow64b bool
311324 codec flatbuf.CompressionType
312325 compressNP int
326+ compressors []compressor
313327 minSpaceSavings * float64
314328}
315329
316- func newRecordEncoder (mem memory.Allocator , startOffset , maxDepth int64 , allow64b bool , codec flatbuf.CompressionType , compressNP int , minSpaceSavings * float64 ) * recordEncoder {
330+ func newRecordEncoder (
331+ mem memory.Allocator ,
332+ startOffset ,
333+ maxDepth int64 ,
334+ allow64b bool ,
335+ codec flatbuf.CompressionType ,
336+ compressNP int ,
337+ minSpaceSavings * float64 ,
338+ compressors []compressor ,
339+ ) * recordEncoder {
317340 return & recordEncoder {
318341 mem : mem ,
319342 start : startOffset ,
320343 depth : maxDepth ,
321344 allow64b : allow64b ,
322345 codec : codec ,
323346 compressNP : compressNP ,
347+ compressors : compressors ,
324348 minSpaceSavings : minSpaceSavings ,
325349 }
326350}
@@ -340,6 +364,13 @@ func (w *recordEncoder) reset() {
340364 w .fields = make ([]fieldMetadata , 0 )
341365}
342366
367+ func (w * recordEncoder ) getCompressor (id int ) compressor {
368+ if w .compressors [id ] == nil {
369+ w .compressors [id ] = getCompressor (w .codec )
370+ }
371+ return w .compressors [id ]
372+ }
373+
343374func (w * recordEncoder ) compressBodyBuffers (p * Payload ) error {
344375 compress := func (idx int , codec compressor ) error {
345376 if p .body [idx ] == nil || p .body [idx ].Len () == 0 {
@@ -378,7 +409,7 @@ func (w *recordEncoder) compressBodyBuffers(p *Payload) error {
378409 }
379410
380411 if w .compressNP <= 1 {
381- codec := getCompressor ( w . codec )
412+ codec := w . getCompressor ( 0 )
382413 for idx := range p .body {
383414 if err := compress (idx , codec ); err != nil {
384415 return err
@@ -395,11 +426,11 @@ func (w *recordEncoder) compressBodyBuffers(p *Payload) error {
395426 )
396427 defer cancel ()
397428
398- for i := 0 ; i < w .compressNP ; i ++ {
429+ for workerID := 0 ; workerID < w .compressNP ; workerID ++ {
399430 wg .Add (1 )
400- go func () {
431+ go func (id int ) {
401432 defer wg .Done ()
402- codec := getCompressor ( w . codec )
433+ codec := w . getCompressor ( id )
403434 for {
404435 select {
405436 case idx , ok := <- ch :
@@ -418,7 +449,7 @@ func (w *recordEncoder) compressBodyBuffers(p *Payload) error {
418449 return
419450 }
420451 }
421- }()
452+ }(workerID )
422453 }
423454
424455 for idx := range p .body {
0 commit comments