Skip to content

Commit 2e5a973

Browse files
klauspostklauspost
authored
zstd: Check FSE init values (#772)
* zstd: Check FSE init values If `br.init(s.br.unread())` fails, it may decode bogus data if previous block returned without reading everything from the bit reader. This is used to feed the huff0 table for literal decoding. Return error correctly. Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=56870 Add parsing of OSS reported input. * Don't use file (yet) Fail on error nilness mismatch * Revert useless file change
1 parent 3588812 commit 2e5a973

File tree

12 files changed

+76
-39
lines changed

12 files changed

+76
-39
lines changed

‎flate/fuzz_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ func TestMain(m *testing.M) {
2626
}
2727

2828
func FuzzEncoding(f *testing.F) {
29-
fuzz.AddFromZip(f, "testdata/regression.zip", true, false)
30-
fuzz.AddFromZip(f, "testdata/fuzz/encode-raw-corpus.zip", true, testing.Short())
31-
fuzz.AddFromZip(f, "testdata/fuzz/FuzzEncoding.zip", false, testing.Short())
29+
fuzz.AddFromZip(f, "testdata/regression.zip", fuzz.TypeRaw, false)
30+
fuzz.AddFromZip(f, "testdata/fuzz/encode-raw-corpus.zip", fuzz.TypeRaw, testing.Short())
31+
fuzz.AddFromZip(f, "testdata/fuzz/FuzzEncoding.zip", fuzz.TypeGoFuzz, testing.Short())
3232

3333
startFuzz := *fuzzStartF
3434
endFuzz := *fuzzEndF

‎fse/decompress.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,9 @@ func (s *Scratch) buildDtable() error {
260260
// If the buffer is over-read an error is returned.
261261
func (s *Scratch) decompress() error {
262262
br := &s.bits
263-
br.init(s.br.unread())
263+
if err := br.init(s.br.unread()); err != nil {
264+
return err
265+
}
264266

265267
var s1, s2 decoder
266268
// Initialize and decode first state and symbol.

‎fse/fse_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ var decTestfiles = []struct {
6262
{name: "crash4", fn: func() ([]byte, error) { return os.ReadFile("../testdata/crash4.bin") }, err: "symbolLen (1) too small"},
6363
{name: "crash5", fn: func() ([]byte, error) { return os.ReadFile("../testdata/crash5.bin") }, err: "symbolLen (1) too small"},
6464
{name: "crash6", fn: func() ([]byte, error) { return os.ReadFile("../testdata/dec-crash6.bin") }, err: "newState (32768) outside table size (32768)"},
65-
{name: "something", fn: func() ([]byte, error) { return os.ReadFile("../testdata/fse-artifact3.bin") }, err: "output size (1048576) > DecompressLimit (1048576)"},
65+
{name: "something", fn: func() ([]byte, error) { return os.ReadFile("../testdata/fse-artifact3.bin") }, err: "corrupt stream, did not find end of stream"},
6666
}
6767

6868
func TestCompress(t *testing.T) {

‎internal/fuzz/helpers.go‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package fuzz
66
import (
77
"archive/zip"
88
"bytes"
9+
"encoding/binary"
910
"fmt"
1011
"go/ast"
1112
"go/parser"
@@ -16,8 +17,20 @@ import (
1617
"testing"
1718
)
1819

20+
type InputType uint8
21+
22+
const (
23+
// TypeRaw indicates that files are raw bytes.
24+
TypeRaw InputType = iota
25+
// TypeGoFuzz indicates files are from Go Fuzzer.
26+
TypeGoFuzz
27+
// TypeOSSFuzz indicates that files are from OSS fuzzer with size before data.
28+
TypeOSSFuzz
29+
)
30+
1931
// AddFromZip will read the supplied zip and add all as corpus for f.
20-
func AddFromZip(f *testing.F, filename string, raw, short bool) {
32+
// Byte slices only.
33+
func AddFromZip(f *testing.F, filename string, t InputType, short bool) {
2134
file, err := os.Open(filename)
2235
if err != nil {
2336
f.Fatal(err)
@@ -44,11 +57,25 @@ func AddFromZip(f *testing.F, filename string, raw, short bool) {
4457
f.Fatal(err)
4558
}
4659
rc.Close()
47-
raw := raw
60+
t := t
61+
if t == TypeOSSFuzz {
62+
t = TypeRaw // Fallback
63+
if len(b) >= 4 {
64+
sz := binary.BigEndian.Uint32(b)
65+
if sz == uint32(len(b))-4 {
66+
f.Add(b[4:])
67+
continue
68+
}
69+
}
70+
}
71+
4872
if bytes.HasPrefix(b, []byte("go test fuzz")) {
49-
raw = false
73+
t = TypeGoFuzz
74+
} else {
75+
t = TypeRaw
5076
}
51-
if raw {
77+
78+
if t == TypeRaw {
5279
f.Add(b)
5380
continue
5481
}

‎s2/dict_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -404,9 +404,9 @@ func TestDictSize(t *testing.T) {
404404
}
405405

406406
func FuzzDictBlocks(f *testing.F) {
407-
fuzz.AddFromZip(f, "testdata/enc_regressions.zip", true, false)
408-
fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-raw.zip", true, testing.Short())
409-
fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-enc.zip", false, testing.Short())
407+
fuzz.AddFromZip(f, "testdata/enc_regressions.zip", fuzz.TypeRaw, false)
408+
fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-raw.zip", fuzz.TypeRaw, testing.Short())
409+
fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-enc.zip", fuzz.TypeGoFuzz, testing.Short())
410410

411411
// Fuzzing tweaks:
412412
const (

‎s2/fuzz_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ import (
1313
)
1414

1515
func FuzzEncodingBlocks(f *testing.F) {
16-
fuzz.AddFromZip(f, "testdata/enc_regressions.zip", true, false)
17-
fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-raw.zip", true, testing.Short())
18-
fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-enc.zip", false, testing.Short())
16+
fuzz.AddFromZip(f, "testdata/enc_regressions.zip", fuzz.TypeRaw, false)
17+
fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-raw.zip", fuzz.TypeRaw, testing.Short())
18+
fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-enc.zip", fuzz.TypeGoFuzz, testing.Short())
1919

2020
// Fuzzing tweaks:
2121
const (

‎s2/lz4convert_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -352,8 +352,8 @@ func BenchmarkCompressBlockReference(b *testing.B) {
352352
}
353353

354354
func FuzzLZ4Block(f *testing.F) {
355-
fuzz.AddFromZip(f, "testdata/fuzz/lz4-convert-corpus-raw.zip", true, false)
356-
fuzz.AddFromZip(f, "testdata/fuzz/FuzzLZ4Block.zip", false, false)
355+
fuzz.AddFromZip(f, "testdata/fuzz/lz4-convert-corpus-raw.zip", fuzz.TypeRaw, false)
356+
fuzz.AddFromZip(f, "testdata/fuzz/FuzzLZ4Block.zip", fuzz.TypeGoFuzz, false)
357357
// Fuzzing tweaks:
358358
const (
359359
// Max input size:

‎zip/fuzz_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ func FuzzReader(f *testing.F) {
3030
}
3131
f.Add(b)
3232
}
33-
fuzz.AddFromZip(f, "testdata/FuzzReader-raw.zip", true, testing.Short())
34-
fuzz.AddFromZip(f, "testdata/FuzzReader-enc.zip", false, testing.Short())
33+
fuzz.AddFromZip(f, "testdata/FuzzReader-raw.zip", fuzz.TypeRaw, testing.Short())
34+
fuzz.AddFromZip(f, "testdata/FuzzReader-enc.zip", fuzz.TypeGoFuzz, testing.Short())
3535

3636
f.Fuzz(func(t *testing.T, b []byte) {
3737
r, err := NewReader(bytes.NewReader(b), int64(len(b)))

‎zstd/blockdec.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"encoding/binary"
1010
"errors"
1111
"fmt"
12+
"hash/crc32"
1213
"io"
1314
"os"
1415
"path/filepath"
@@ -442,6 +443,9 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err
442443
}
443444
}
444445
var err error
446+
if debugDecoder {
447+
println("huff table input:", len(literals), "CRC:", crc32.ChecksumIEEE(literals))
448+
}
445449
huff, literals, err = huff0.ReadTable(literals, huff)
446450
if err != nil {
447451
println("reading huffman table:", err)

‎zstd/fuzz_test.go‎

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,9 @@ import (
1515
)
1616

1717
func FuzzDecodeAll(f *testing.F) {
18-
fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", true, testing.Short())
19-
fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", false, testing.Short())
20-
decLow, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderConcurrency(2), WithDecoderMaxMemory(20<<20), WithDecoderMaxWindow(1<<20), IgnoreChecksum(true))
21-
if err != nil {
22-
f.Fatal(err)
23-
}
24-
defer decLow.Close()
25-
decHi, err := NewReader(nil, WithDecoderLowmem(false), WithDecoderConcurrency(2), WithDecoderMaxMemory(20<<20), WithDecoderMaxWindow(1<<20), IgnoreChecksum(true))
26-
if err != nil {
27-
f.Fatal(err)
28-
}
29-
defer decHi.Close()
18+
fuzz.AddFromZip(f, "testdata/decode-regression.zip", fuzz.TypeRaw, false)
19+
fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", fuzz.TypeRaw, testing.Short())
20+
fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", fuzz.TypeGoFuzz, testing.Short())
3021

3122
f.Fuzz(func(t *testing.T, b []byte) {
3223
// Just test if we crash...
@@ -36,10 +27,23 @@ func FuzzDecodeAll(f *testing.F) {
3627
t.Fatal(r)
3728
}
3829
}()
39-
b1, err1 := decLow.DecodeAll(b, nil)
40-
b2, err2 := decHi.DecodeAll(b, nil)
30+
31+
decLow, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderConcurrency(2), WithDecoderMaxMemory(20<<20), WithDecoderMaxWindow(1<<20), IgnoreChecksum(true))
32+
if err != nil {
33+
f.Fatal(err)
34+
}
35+
defer decLow.Close()
36+
decHi, err := NewReader(nil, WithDecoderLowmem(false), WithDecoderConcurrency(2), WithDecoderMaxMemory(20<<20), WithDecoderMaxWindow(1<<20), IgnoreChecksum(true))
37+
if err != nil {
38+
f.Fatal(err)
39+
}
40+
defer decHi.Close()
41+
b1, err1 := decLow.DecodeAll(b, make([]byte, 0, len(b)))
42+
b2, err2 := decHi.DecodeAll(b, make([]byte, 0, len(b)))
4143
if err1 != err2 {
42-
t.Log(err1, err2)
44+
if (err1 == nil) != (err2 == nil) {
45+
t.Errorf("err low: %v, hi: %v", err1, err2)
46+
}
4347
}
4448
if err1 != nil {
4549
b1, b2 = b1[:0], b2[:0]
@@ -60,8 +64,8 @@ func FuzzDecAllNoBMI2(f *testing.F) {
6064
}
6165

6266
func FuzzDecoder(f *testing.F) {
63-
fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", true, testing.Short())
64-
fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", false, testing.Short())
67+
fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-raw.zip", fuzz.TypeRaw, testing.Short())
68+
fuzz.AddFromZip(f, "testdata/fuzz/decode-corpus-encoded.zip", fuzz.TypeGoFuzz, testing.Short())
6569

6670
brLow := newBytesReader(nil)
6771
brHi := newBytesReader(nil)
@@ -112,9 +116,9 @@ func FuzzNoBMI2Dec(f *testing.F) {
112116
}
113117

114118
func FuzzEncoding(f *testing.F) {
115-
fuzz.AddFromZip(f, "testdata/fuzz/encode-corpus-raw.zip", true, testing.Short())
116-
fuzz.AddFromZip(f, "testdata/comp-crashers.zip", true, false)
117-
fuzz.AddFromZip(f, "testdata/fuzz/encode-corpus-encoded.zip", false, testing.Short())
119+
fuzz.AddFromZip(f, "testdata/fuzz/encode-corpus-raw.zip", fuzz.TypeRaw, testing.Short())
120+
fuzz.AddFromZip(f, "testdata/comp-crashers.zip", fuzz.TypeRaw, false)
121+
fuzz.AddFromZip(f, "testdata/fuzz/encode-corpus-encoded.zip", fuzz.TypeGoFuzz, testing.Short())
118122
// Fuzzing tweaks:
119123
const (
120124
// Test a subset of encoders.

0 commit comments

Comments
 (0)