Skip to content
This repository was archived by the owner on Jan 18, 2026. It is now read-only.

Commit fa1080c

Browse files
vpappvpapp
authored
feat: limit decimal encoding to values allowed by schema (#565)
1 parent 3a21909 commit fa1080c

File tree

8 files changed

+115
-15
lines changed

8 files changed

+115
-15
lines changed

‎codec_fixed.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,16 @@ func (c *fixedDecimalCodec) Encode(ptr unsafe.Pointer, w *Writer) {
145145
i := (&big.Int{}).Mul(r.Num(), scale)
146146
i = i.Div(i, r.Denom())
147147

148+
if numDigits, ok := checkDecimalPrecision(i, c.prec); !ok {
149+
w.Error = fmt.Errorf(
150+
"avro: cannot encode %v as Avro fixed.decimal with precision=%d, has %d significant digits",
151+
r.FloatString(c.scale),
152+
c.prec,
153+
numDigits,
154+
)
155+
return
156+
}
157+
148158
var b []byte
149159
switch i.Sign() {
150160
case 0:
@@ -165,6 +175,16 @@ func (c *fixedDecimalCodec) Encode(ptr unsafe.Pointer, w *Writer) {
165175
b = i.Add(i, (&big.Int{}).Lsh(one, uint(c.size*8))).Bytes()
166176
}
167177

178+
if len(b) != c.size {
179+
w.Error = fmt.Errorf(
180+
"avro: cannot encode %v as Avro fixed.decimal with size=%d, encodes to %d bytes",
181+
r.FloatString(c.scale),
182+
c.size,
183+
len(b),
184+
)
185+
return
186+
}
187+
168188
_, _ = w.Write(b)
169189
}
170190

‎codec_native.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -610,6 +610,16 @@ func (c *bytesDecimalCodec) Encode(ptr unsafe.Pointer, w *Writer) {
610610
i := (&big.Int{}).Mul(r.Num(), scale)
611611
i = i.Div(i, r.Denom())
612612

613+
if numDigits, ok := checkDecimalPrecision(i, c.prec); !ok {
614+
w.Error = fmt.Errorf(
615+
"avro: cannot encode %v as Avro bytes.decimal with precision=%d, has %d significant digits",
616+
r.FloatString(c.scale),
617+
c.prec,
618+
numDigits,
619+
)
620+
return
621+
}
622+
613623
var b []byte
614624
switch i.Sign() {
615625
case 0:
@@ -647,6 +657,16 @@ func (c *bytesDecimalPtrCodec) Encode(ptr unsafe.Pointer, w *Writer) {
647657
i := (&big.Int{}).Mul(r.Num(), scale)
648658
i = i.Div(i, r.Denom())
649659

660+
if numDigits, ok := checkDecimalPrecision(i, c.prec); !ok {
661+
w.Error = fmt.Errorf(
662+
"avro: cannot encode %v as Avro bytes.decimal with precision=%d, has %d significant digits",
663+
r.FloatString(c.scale),
664+
c.prec,
665+
numDigits,
666+
)
667+
return
668+
}
669+
650670
var b []byte
651671
switch i.Sign() {
652672
case 0:

‎decimal.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package avro
2+
3+
import (
4+
"math/big"
5+
)
6+
7+
// checkDecimalPrecision checks if the value exceeds the specified precision.
8+
// returns the number of digits and whether it is valid.
9+
func checkDecimalPrecision(value *big.Int, prec int) (int, bool) {
10+
unscaledAbsStr := new(big.Int).Abs(value).String()
11+
numDigits := len(unscaledAbsStr)
12+
13+
if len(unscaledAbsStr) > prec {
14+
return numDigits, false
15+
}
16+
17+
return numDigits, true
18+
}

‎encoder_fixed_test.go‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ func TestEncoder_Fixed(t *testing.T) {
4242
func TestEncoder_FixedRat_Positive(t *testing.T) {
4343
defer ConfigTeardown()
4444

45-
schema := `{"type":"fixed", "name": "test", "size": 6,"logicalType":"decimal","precision":4,"scale":2}`
45+
schema := `{"type":"fixed", "name": "test", "size": 6,"logicalType":"decimal","precision":5,"scale":2}`
4646
buf := bytes.NewBuffer([]byte{})
4747
enc, err := avro.NewEncoder(schema, buf)
4848
require.NoError(t, err)
@@ -56,7 +56,7 @@ func TestEncoder_FixedRat_Positive(t *testing.T) {
5656
func TestEncoder_FixedRat_Negative(t *testing.T) {
5757
defer ConfigTeardown()
5858

59-
schema := `{"type":"fixed", "name": "test", "size": 6, "logicalType":"decimal","precision":4,"scale":2}`
59+
schema := `{"type":"fixed", "name": "test", "size": 6, "logicalType":"decimal","precision":5,"scale":2}`
6060
buf := bytes.NewBuffer([]byte{})
6161
enc, err := avro.NewEncoder(schema, buf)
6262
require.NoError(t, err)
@@ -70,7 +70,7 @@ func TestEncoder_FixedRat_Negative(t *testing.T) {
7070
func TestEncoder_FixedRat_Zero(t *testing.T) {
7171
defer ConfigTeardown()
7272

73-
schema := `{"type":"fixed", "name": "test", "size": 6,"logicalType":"decimal","precision":4,"scale":2}`
73+
schema := `{"type":"fixed", "name": "test", "size": 6,"logicalType":"decimal","precision":5,"scale":2}`
7474
buf := bytes.NewBuffer([]byte{})
7575
enc, err := avro.NewEncoder(schema, buf)
7676
require.NoError(t, err)
@@ -81,6 +81,20 @@ func TestEncoder_FixedRat_Zero(t *testing.T) {
8181
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, buf.Bytes())
8282
}
8383

84+
func TestEncoder_FixedRat_TooManyDigits(t *testing.T) {
85+
defer ConfigTeardown()
86+
87+
schema := `{"type":"fixed", "name": "test", "size": 6,"logicalType":"decimal","precision":3,"scale":2}`
88+
buf := bytes.NewBuffer([]byte{})
89+
enc, err := avro.NewEncoder(schema, buf)
90+
require.NoError(t, err)
91+
92+
err = enc.Encode(big.NewRat(1734, 5))
93+
94+
assert.ErrorContains(t, err, "avro: cannot encode 346.80 as Avro fixed.decimal with precision=3, has 5 significant digits")
95+
assert.Empty(t, buf.Bytes())
96+
}
97+
8498
func TestEncoder_FixedRatInvalidLogicalSchema(t *testing.T) {
8599
defer ConfigTeardown()
86100

‎encoder_native_test.go‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -669,7 +669,7 @@ func TestEncoder_DurationInvalidSchema(t *testing.T) {
669669
func TestEncoder_BytesRat_Positive(t *testing.T) {
670670
defer ConfigTeardown()
671671

672-
schema := `{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}`
672+
schema := `{"type":"bytes","logicalType":"decimal","precision":5,"scale":2}`
673673
buf := bytes.NewBuffer([]byte{})
674674
enc, err := avro.NewEncoder(schema, buf)
675675
require.NoError(t, err)
@@ -683,7 +683,7 @@ func TestEncoder_BytesRat_Positive(t *testing.T) {
683683
func TestEncoder_BytesRat_Negative(t *testing.T) {
684684
defer ConfigTeardown()
685685

686-
schema := `{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}`
686+
schema := `{"type":"bytes","logicalType":"decimal","precision":5,"scale":2}`
687687
buf := bytes.NewBuffer([]byte{})
688688
enc, err := avro.NewEncoder(schema, buf)
689689
require.NoError(t, err)
@@ -697,7 +697,7 @@ func TestEncoder_BytesRat_Negative(t *testing.T) {
697697
func TestEncoder_BytesRat_Zero(t *testing.T) {
698698
defer ConfigTeardown()
699699

700-
schema := `{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}`
700+
schema := `{"type":"bytes","logicalType":"decimal","precision":5,"scale":2}`
701701
buf := bytes.NewBuffer([]byte{})
702702
enc, err := avro.NewEncoder(schema, buf)
703703
require.NoError(t, err)
@@ -708,10 +708,24 @@ func TestEncoder_BytesRat_Zero(t *testing.T) {
708708
assert.Equal(t, []byte{0x02, 0x00}, buf.Bytes())
709709
}
710710

711+
func TestEncoder_BytesRat_TooManyDigits(t *testing.T) {
712+
defer ConfigTeardown()
713+
714+
schema := `{"type":"bytes","logicalType":"decimal","precision":3,"scale":2}`
715+
buf := bytes.NewBuffer([]byte{})
716+
enc, err := avro.NewEncoder(schema, buf)
717+
require.NoError(t, err)
718+
719+
err = enc.Encode(big.NewRat(1734, 5))
720+
721+
assert.ErrorContains(t, err, "avro: cannot encode 346.80 as Avro bytes.decimal with precision=3, has 5 significant digits")
722+
assert.Empty(t, buf.Bytes())
723+
}
724+
711725
func TestEncoder_BytesRatNonPtr_Positive(t *testing.T) {
712726
defer ConfigTeardown()
713727

714-
schema := `{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}`
728+
schema := `{"type":"bytes","logicalType":"decimal","precision":5,"scale":2}`
715729
buf := bytes.NewBuffer([]byte{})
716730
enc, err := avro.NewEncoder(schema, buf)
717731
require.NoError(t, err)
@@ -725,7 +739,7 @@ func TestEncoder_BytesRatNonPtr_Positive(t *testing.T) {
725739
func TestEncoder_BytesRatNonPtr_Negative(t *testing.T) {
726740
defer ConfigTeardown()
727741

728-
schema := `{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}`
742+
schema := `{"type":"bytes","logicalType":"decimal","precision":5,"scale":2}`
729743
buf := bytes.NewBuffer([]byte{})
730744
enc, err := avro.NewEncoder(schema, buf)
731745
require.NoError(t, err)
@@ -739,7 +753,7 @@ func TestEncoder_BytesRatNonPtr_Negative(t *testing.T) {
739753
func TestEncoder_BytesRatNonPtr_Zero(t *testing.T) {
740754
defer ConfigTeardown()
741755

742-
schema := `{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}`
756+
schema := `{"type":"bytes","logicalType":"decimal","precision":5,"scale":2}`
743757
buf := bytes.NewBuffer([]byte{})
744758
enc, err := avro.NewEncoder(schema, buf)
745759
require.NoError(t, err)
@@ -750,6 +764,20 @@ func TestEncoder_BytesRatNonPtr_Zero(t *testing.T) {
750764
assert.Equal(t, []byte{0x02, 0x00}, buf.Bytes())
751765
}
752766

767+
func TestEncoder_BytesRatNonPtr_TooManyDigits(t *testing.T) {
768+
defer ConfigTeardown()
769+
770+
schema := `{"type":"bytes","logicalType":"decimal","precision":3,"scale":2}`
771+
buf := bytes.NewBuffer([]byte{})
772+
enc, err := avro.NewEncoder(schema, buf)
773+
require.NoError(t, err)
774+
775+
err = enc.Encode(*big.NewRat(1734, 5))
776+
777+
assert.ErrorContains(t, err, "avro: cannot encode 346.80 as Avro bytes.decimal with precision=3, has 5 significant digits")
778+
assert.Empty(t, buf.Bytes())
779+
}
780+
753781
func TestEncoder_BytesRatInvalidSchema(t *testing.T) {
754782
defer ConfigTeardown()
755783

‎encoder_union_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ func TestEncoder_UnionMapWithDecimal(t *testing.T) {
126126
defer ConfigTeardown()
127127

128128
t.Run("low scale", func(t *testing.T) {
129-
schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
129+
schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 5, "scale": 2}]`
130130
buf := bytes.NewBuffer([]byte{})
131131
enc, err := avro.NewEncoder(schema, buf)
132132
require.NoError(t, err)
@@ -582,7 +582,7 @@ func TestEncoder_UnionInterfaceWithDecimal(t *testing.T) {
582582
defer ConfigTeardown()
583583

584584
t.Run("low scale", func(t *testing.T) {
585-
schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 2}]`
585+
schema := `["null", {"type": "bytes", "logicalType": "decimal", "precision": 5, "scale": 2}]`
586586
buf := bytes.NewBuffer([]byte{})
587587
enc, err := avro.NewEncoder(schema, buf)
588588
require.NoError(t, err)
@@ -758,7 +758,7 @@ func TestEncoder_UnionResolver(t *testing.T) {
758758
},
759759
{
760760
name: "Go big.Rat as Avro bytes.decimal",
761-
schema: `["null",{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}]`,
761+
schema: `["null",{"type":"bytes","logicalType":"decimal","precision":5,"scale":2}]`,
762762
value: big.NewRat(1734, 5),
763763
want: []byte{0x2, 0x6, 0x00, 0x87, 0x78},
764764
},

‎schema_parse.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -550,7 +550,7 @@ func newDecimalLogicalType(size, prec, scale int) LogicalSchema {
550550
}
551551

552552
if size > 0 {
553-
maxPrecision := int(math.Round(math.Floor(math.Log10(2) * (8*float64(size) - 1))))
553+
maxPrecision := int(math.Floor(math.Log10(math.Pow(2, 8*float64(size)) - 1)))
554554
if prec > maxPrecision {
555555
return nil
556556
}

‎typeconverter_encode_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ func TestEncoderTypeConverter_MapRecordMapUnion(t *testing.T) {
129129
func TestEncoderTypeConverter_MapRecordUnionFixedDecimal(t *testing.T) {
130130
defer ConfigTeardown()
131131

132-
schema := `{"type":"record", "name":"test", "fields":[{"name":"a", "type": {"type":"fixed", "name":"fixed_decimal", "size":6, "logicalType":"decimal", "precision":4, "scale":2}}]}`
132+
schema := `{"type":"record", "name":"test", "fields":[{"name":"a", "type": {"type":"fixed", "name":"fixed_decimal", "size":6, "logicalType":"decimal", "precision":5, "scale":2}}]}`
133133
buf := &bytes.Buffer{}
134134
enc, err := avro.NewEncoder(schema, buf)
135135
require.NoError(t, err)
@@ -217,7 +217,7 @@ func TestEncoderTypeConverter_StructRecordUnionResolved(t *testing.T) {
217217
func TestEncoderTypeConverter_StructRecordFixedDecimal(t *testing.T) {
218218
defer ConfigTeardown()
219219

220-
schema := `{"type":"record", "name":"test", "fields":[{"name":"a", "type": {"type":"fixed", "name":"fixed_decimal", "size":6, "logicalType":"decimal", "precision":4, "scale":2}}]}`
220+
schema := `{"type":"record", "name":"test", "fields":[{"name":"a", "type": {"type":"fixed", "name":"fixed_decimal", "size":6, "logicalType":"decimal", "precision":5, "scale":2}}]}`
221221
buf := &bytes.Buffer{}
222222
enc, err := avro.NewEncoder(schema, buf)
223223
require.NoError(t, err)

0 commit comments

Comments
 (0)